From 193d25b05198f7fcf28b5263717c645d1815a152 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Mon, 10 Oct 2022 13:57:56 +0100 Subject: [PATCH 001/114] refactor: Launcher to interface & native.Launcher In preparation for sylabs/singularity#1021, where we will add a placeholder OCI runtime launcher, refactor the existing code so that: * The internal/pkg/runtime/launcher package contains common option handling, utility functions, and a Launcher interface. * There is a launcher.native package containing the existing launch code for the native singularity runtime. Signed-off-by: Edita Kizinevic --- cmd/internal/cli/actions.go | 104 +++++++------ internal/pkg/runtime/launcher/launcher.go | 33 ++++ .../native}/launcher_linux.go | 64 +++----- .../runtime/{launch => launcher}/options.go | 143 ++++++++---------- internal/pkg/runtime/launcher/util.go | 50 ++++++ 5 files changed, 221 insertions(+), 173 deletions(-) create mode 100644 internal/pkg/runtime/launcher/launcher.go rename internal/pkg/runtime/{launch => launcher/native}/launcher_linux.go (97%) rename internal/pkg/runtime/{launch => launcher}/options.go (81%) create mode 100644 internal/pkg/runtime/launcher/util.go diff --git a/cmd/internal/cli/actions.go b/cmd/internal/cli/actions.go index 16763e9219..ffc3afbcaf 100644 --- a/cmd/internal/cli/actions.go +++ b/cmd/internal/cli/actions.go @@ -24,7 +24,8 @@ import ( "github.com/apptainer/apptainer/internal/pkg/client/oci" "github.com/apptainer/apptainer/internal/pkg/client/oras" "github.com/apptainer/apptainer/internal/pkg/client/shub" - "github.com/apptainer/apptainer/internal/pkg/runtime/launch" + "github.com/apptainer/apptainer/internal/pkg/runtime/launcher" + "github.com/apptainer/apptainer/internal/pkg/runtime/launcher/native" "github.com/apptainer/apptainer/internal/pkg/util/env" "github.com/apptainer/apptainer/internal/pkg/util/uri" "github.com/apptainer/apptainer/pkg/sylog" @@ -287,7 +288,7 @@ var TestCmd = &cobra.Command{ } func launchContainer(cmd *cobra.Command, image string, args []string, instanceName string) error { - ns := launch.Namespaces{ + ns := launcher.Namespaces{ User: userNamespace, UTS: utsNamespace, PID: pidNamespace, @@ -309,60 +310,63 @@ func launchContainer(cmd *cobra.Command, image string, args []string, instanceNa return err } - opts := []launch.Option{ - launch.OptWritable(isWritable), - launch.OptWritableTmpfs(isWritableTmpfs), - launch.OptOverlayPaths(overlayPath), - launch.OptScratchDirs(scratchPath), - launch.OptWorkDir(workdirPath), - launch.OptHome( + opts := []launcher.Option{ + launcher.OptWritable(isWritable), + launcher.OptWritableTmpfs(isWritableTmpfs), + launcher.OptOverlayPaths(overlayPath), + launcher.OptScratchDirs(scratchPath), + launcher.OptWorkDir(workdirPath), + launcher.OptHome( homePath, cmd.Flag(actionHomeFlag.Name).Changed, noHome, ), - launch.OptMounts(bindPaths, mounts, fuseMount), - launch.OptNoMount(noMount), - launch.OptNvidia(nvidia, nvCCLI), - launch.OptNoNvidia(noNvidia), - launch.OptRocm(rocm), - launch.OptNoRocm(noRocm), - launch.OptContainLibs(containLibsPath), - launch.OptEnv(apptainerEnv, apptainerEnvFile, isCleanEnv), - launch.OptNoEval(noEval), - launch.OptNamespaces(ns), - launch.OptNetwork(network, networkArgs), - launch.OptHostname(hostname), - launch.OptDNS(dns), - launch.OptCaps(addCaps, dropCaps), - launch.OptAllowSUID(allowSUID), - launch.OptKeepPrivs(keepPrivs), - launch.OptNoPrivs(noPrivs), - launch.OptSecurity(security), - launch.OptNoUmask(noUmask), - launch.OptCgroupsJSON(cgJSON), - launch.OptConfigFile(configurationFile), - launch.OptShellPath(shellPath), - launch.OptCwdPath(cwdPath), - launch.OptFakeroot(isFakeroot), - launch.OptBoot(isBoot), - launch.OptNoInit(noInit), - launch.OptContain(isContained), - launch.OptContainAll(isContainAll), - launch.OptAppName(appName), - launch.OptKeyInfo(ki), - launch.OptCacheDisabled(disableCache), - launch.OptDMTCPLaunch(dmtcpLaunch), - launch.OptDMTCPRestart(dmtcpRestart), - launch.OptUnsquash(unsquash), - launch.OptIgnoreSubuid(ignoreSubuid), - launch.OptIgnoreFakerootCmd(ignoreFakerootCmd), - launch.OptIgnoreUserns(ignoreUserns), - launch.OptUseBuildConfig(useBuildConfig), - launch.OptTmpDir(tmpDir), - launch.OptUnderlay(underlay), + launcher.OptMounts(bindPaths, mounts, fuseMount), + launcher.OptNoMount(noMount), + launcher.OptNvidia(nvidia, nvCCLI), + launcher.OptNoNvidia(noNvidia), + launcher.OptRocm(rocm), + launcher.OptNoRocm(noRocm), + launcher.OptContainLibs(containLibsPath), + launcher.OptEnv(apptainerEnv, apptainerEnvFile, isCleanEnv), + launcher.OptNoEval(noEval), + launcher.OptNamespaces(ns), + launcher.OptNetwork(network, networkArgs), + launcher.OptHostname(hostname), + launcher.OptDNS(dns), + launcher.OptCaps(addCaps, dropCaps), + launcher.OptAllowSUID(allowSUID), + launcher.OptKeepPrivs(keepPrivs), + launcher.OptNoPrivs(noPrivs), + launcher.OptSecurity(security), + launcher.OptNoUmask(noUmask), + launcher.OptCgroupsJSON(cgJSON), + launcher.OptConfigFile(configurationFile), + launcher.OptShellPath(shellPath), + launcher.OptCwdPath(cwdPath), + launcher.OptFakeroot(isFakeroot), + launcher.OptBoot(isBoot), + launcher.OptNoInit(noInit), + launcher.OptContain(isContained), + launcher.OptContainAll(isContainAll), + launcher.OptAppName(appName), + launcher.OptKeyInfo(ki), + launcher.OptCacheDisabled(disableCache), + launcher.OptDMTCPLaunch(dmtcpLaunch), + launcher.OptDMTCPRestart(dmtcpRestart), + launcher.OptUnsquash(unsquash), + launcher.OptIgnoreSubuid(ignoreSubuid), + launcher.OptIgnoreFakerootCmd(ignoreFakerootCmd), + launcher.OptIgnoreUserns(ignoreUserns), + launcher.OptUseBuildConfig(useBuildConfig), + launcher.OptTmpDir(tmpDir), + launcher.OptUnderlay(underlay), } - l, err := launch.NewLauncher(opts...) + // Explicitly use the interface type here, as we will add alternative launchers later... + var l launcher.Launcher + + l, err = native.NewLauncher(opts...) if err != nil { return fmt.Errorf("while configuring container: %s", err) } diff --git a/internal/pkg/runtime/launcher/launcher.go b/internal/pkg/runtime/launcher/launcher.go new file mode 100644 index 0000000000..1a79a55889 --- /dev/null +++ b/internal/pkg/runtime/launcher/launcher.go @@ -0,0 +1,33 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2022, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +// Package launcher is responsible for implementing launchers, which can start a +// container, with configuration passed from the CLI layer. +// +// The package currently implements a single native.Launcher, with an Exec +// method that constructs a runtime configuration and calls the Apptainer +// runtime starter binary to start the container. +// +// TODO - the launcher package will be extended to support launching containers +// via the OCI runc/crun runtime, in addition to the current Apptainer runtime +// starter, by adding an oci.Launcher. +package launcher + +import "context" + +// Launcher is responsible for configuring and launching a container image. +// It will execute a runtime, such as Apptainer's native runtime (via the starter +// binary), or an external OCI runtime (e.g. runc). +type Launcher interface { + // Exec will execute the container image 'image', passing arguments 'args' + // the container#s initial process. If instanceName is specified, the + // container must be launched as a background instance, otherwist it must + // run interactively, attached to the console. + Exec(ctx context.Context, image string, args []string, instanceName string) error +} diff --git a/internal/pkg/runtime/launch/launcher_linux.go b/internal/pkg/runtime/launcher/native/launcher_linux.go similarity index 97% rename from internal/pkg/runtime/launch/launcher_linux.go rename to internal/pkg/runtime/launcher/native/launcher_linux.go index 42fb49a1b4..3d706ad6b3 100644 --- a/internal/pkg/runtime/launch/launcher_linux.go +++ b/internal/pkg/runtime/launcher/native/launcher_linux.go @@ -7,7 +7,9 @@ // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. -package launch +// Package native implements a Launcher that will configure and launch a +// container with Apptainer's own (native) runtime. +package native import ( "context" @@ -31,6 +33,7 @@ import ( "github.com/apptainer/apptainer/internal/pkg/plugin" "github.com/apptainer/apptainer/internal/pkg/runtime/engine/config/oci" "github.com/apptainer/apptainer/internal/pkg/runtime/engine/config/oci/generate" + "github.com/apptainer/apptainer/internal/pkg/runtime/launcher" "github.com/apptainer/apptainer/internal/pkg/security" "github.com/apptainer/apptainer/internal/pkg/util/bin" "github.com/apptainer/apptainer/internal/pkg/util/env" @@ -57,8 +60,19 @@ import ( "golang.org/x/sys/unix" ) -func NewLauncher(opts ...Option) (*Launcher, error) { - lo := launchOptions{} +// Launcher will holds configuration for, and will launch a container using +// Apptainer's own (native) runtime. +type Launcher struct { + uid uint32 + gid uint32 + cfg launcher.Options + engineConfig *apptainerConfig.EngineConfig + generator *generate.Generator +} + +// NewLauncher returns a native.Launcher with an initial configuration set by opts. +func NewLauncher(opts ...launcher.Option) (*Launcher, error) { + lo := launcher.Options{} for _, opt := range opts { if err := opt(&lo); err != nil { return nil, fmt.Errorf("%w", err) @@ -289,7 +303,7 @@ func (l *Launcher) Exec(ctx context.Context, image string, args []string, instan l.engineConfig.SetUseBuildConfig(l.cfg.UseBuildConfig) // When running as root, the user can optionally allow setuid with container. - err = withPrivilege(l.uid, l.cfg.AllowSUID, "--allow-setuid", func() error { + err = launcher.WithPrivilege(l.uid, l.cfg.AllowSUID, "--allow-setuid", func() error { l.engineConfig.SetAllowSUID(l.cfg.AllowSUID) return nil }) @@ -298,7 +312,7 @@ func (l *Launcher) Exec(ctx context.Context, image string, args []string, instan } // When running as root, the user can optionally keep all privs in the container. - err = withPrivilege(l.uid, l.cfg.KeepPrivs, "--keep-privs", func() error { + err = launcher.WithPrivilege(l.uid, l.cfg.KeepPrivs, "--keep-privs", func() error { l.engineConfig.SetKeepPrivs(l.cfg.KeepPrivs) return nil }) @@ -327,7 +341,7 @@ func (l *Launcher) Exec(ctx context.Context, image string, args []string, instan l.setCgroups(instanceName) // --boot flag requires privilege, so check for this. - err = withPrivilege(l.uid, l.cfg.Boot, "--boot", func() error { return nil }) + err = launcher.WithPrivilege(l.uid, l.cfg.Boot, "--boot", func() error { return nil }) if err != nil { sylog.Fatalf("Could not configure --boot: %s", err) } @@ -350,7 +364,7 @@ func (l *Launcher) Exec(ctx context.Context, image string, args []string, instan l.engineConfig.SetInstance(true) l.engineConfig.SetBootInstance(l.cfg.Boot) - if useSuid && !l.cfg.Namespaces.User && hidepidProc() { + if useSuid && !l.cfg.Namespaces.User && launcher.HidepidProc() { return fmt.Errorf("hidepid option set on /proc mount, require 'hidepid=0' to start instance with setuid workflow") } @@ -452,7 +466,7 @@ func (l *Launcher) setTargetIDs(useSuid bool) (err error) { } // If a target uid was requested, and we are root or non-suid, handle that. - err = withPrivilege(pseudoRoot, uidParam != "", "uid security feature with suid mode", func() error { + err = launcher.WithPrivilege(pseudoRoot, uidParam != "", "uid security feature with suid mode", func() error { u, err := strconv.ParseUint(uidParam, 10, 32) if err != nil { return fmt.Errorf("failed to parse provided UID: %w", err) @@ -468,7 +482,7 @@ func (l *Launcher) setTargetIDs(useSuid bool) (err error) { } // If any target gids were requested, and we are root or non-suid, handle that. - err = withPrivilege(pseudoRoot, gidParam != "", "gid security feature with suid mode", func() error { + err = launcher.WithPrivilege(pseudoRoot, gidParam != "", "gid security feature with suid mode", func() error { gids := strings.Split(gidParam, ":") for _, id := range gids { g, err := strconv.ParseUint(id, 10, 32) @@ -1227,38 +1241,6 @@ func runPluginCallbacks(cfg *config.Common) error { return nil } -// withPrivilege calls fn if cond is satisfied, and we are uid 0 -func withPrivilege(uid uint32, cond bool, desc string, fn func() error) error { - if !cond { - return nil - } - if uid != 0 { - return fmt.Errorf("%s requires root privileges", desc) - } - return fn() -} - -// hidepidProc checks if hidepid is set on /proc mount point, when this -// option is an instance started with setuid workflow could not even be -// joined later or stopped correctly. -func hidepidProc() bool { - entries, err := proc.GetMountInfoEntry("/proc/self/mountinfo") - if err != nil { - sylog.Warningf("while reading /proc/self/mountinfo: %s", err) - return false - } - for _, e := range entries { - if e.Point == "/proc" { - for _, o := range e.SuperOptions { - if strings.HasPrefix(o, "hidepid=") { - return true - } - } - } - } - return false -} - // convertImage extracts the image found at filename to directory dir within a temporary directory // tempDir. If the unsquashfs binary is not located, the binary at unsquashfsPath is used. It is // the caller's responsibility to remove rootfsDir when no longer needed. diff --git a/internal/pkg/runtime/launch/options.go b/internal/pkg/runtime/launcher/options.go similarity index 81% rename from internal/pkg/runtime/launch/options.go rename to internal/pkg/runtime/launcher/options.go index 5e57689ed6..1af7785bbc 100644 --- a/internal/pkg/runtime/launch/options.go +++ b/internal/pkg/runtime/launcher/options.go @@ -7,28 +7,25 @@ // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. -// Package launcher is responsible for starting a container, with configuration -// passed to it from the CLI layer. -// -// The package currently implements a single Launcher, with an Exec method that -// constructs a runtime configuration and calls the Apptainer runtime starter -// binary to start the container. -// -// TODO - the launcher package will be extended to support launching containers -// via the OCI runc/crun runtime, in addition to the current Apptainer runtime -// starter. -package launch +package launcher import ( - "github.com/apptainer/apptainer/internal/pkg/runtime/engine/config/oci/generate" - apptainerConfig "github.com/apptainer/apptainer/pkg/runtime/engine/apptainer/config" "github.com/apptainer/apptainer/pkg/util/cryptkey" ) -// launchOptions accumulates configuration from passed functional options. Note -// that the launchOptions is modified heavily by logic during the Exec function -// call. -type launchOptions struct { +// Namespaces holds flags for the optional (non-mount) namespaces that can be +// requested for a container launch. +type Namespaces struct { + User bool + UTS bool + PID bool + IPC bool + Net bool +} + +// Options accumulates launch configuration from passed functional options. Note +// that the Options is modified heavily by logic during the Exec function call. +type Options struct { // Writable marks the container image itself as writable. Writable bool // WriteableTmpfs applies an ephemeral writable overlay to the container. @@ -153,29 +150,11 @@ type launchOptions struct { Underlay bool // whether prefer underlay over overlay } -type Launcher struct { - uid uint32 - gid uint32 - cfg launchOptions - engineConfig *apptainerConfig.EngineConfig - generator *generate.Generator -} - -// Namespaces holds flags for the optional (non-mount) namespaces that can be -// requested for a container launch. -type Namespaces struct { - User bool - UTS bool - PID bool - IPC bool - Net bool -} - -type Option func(co *launchOptions) error +type Option func(co *Options) error // OptWritable sets the container image to be writable. func OptWritable(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.Writable = b return nil } @@ -183,7 +162,7 @@ func OptWritable(b bool) Option { // OptWritableTmpFs applies an ephemeral writable overlay to the container. func OptWritableTmpfs(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.WritableTmpfs = b return nil } @@ -191,7 +170,7 @@ func OptWritableTmpfs(b bool) Option { // OptOverlayPaths sets overlay images and directories to apply to the container. func OptOverlayPaths(op []string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.OverlayPaths = op return nil } @@ -199,7 +178,7 @@ func OptOverlayPaths(op []string) Option { // OptScratchDirs sets temporary host directories to create and bind into the container. func OptScratchDirs(sd []string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.ScratchDirs = sd return nil } @@ -207,7 +186,7 @@ func OptScratchDirs(sd []string) Option { // OptWorkDir sets the parent path for scratch directories, and contained home/tmp on the host. func OptWorkDir(wd string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.WorkDir = wd return nil } @@ -219,7 +198,7 @@ func OptWorkDir(wd string) Option { // custom is a marker that this is user supplied, and must not be overridden. // disable will disable the home mount entirely, ignoring other options. func OptHome(homeDir string, custom bool, disable bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.HomeDir = homeDir lo.CustomHome = custom lo.NoHome = disable @@ -233,7 +212,7 @@ func OptHome(homeDir string, custom bool, disable bool) Option { // mounts lists bind mount specifications in Docker CSV processed format. // fuseMounts list FUSE mounts in : format. func OptMounts(binds []string, mounts []string, fuseMounts []string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.BindPaths = binds lo.Mounts = mounts lo.FuseMount = fuseMounts @@ -243,7 +222,7 @@ func OptMounts(binds []string, mounts []string, fuseMounts []string) Option { // OptNoMount disables the specified bind mounts. func OptNoMount(nm []string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.NoMount = nm return nil } @@ -253,7 +232,7 @@ func OptNoMount(nm []string) Option { // // nvccli sets whether to use the nvidia-container-runtime (true), or legacy bind mounts (false). func OptNvidia(nv bool, nvccli bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.Nvidia = nv || nvccli lo.NvCCLI = nvccli return nil @@ -262,7 +241,7 @@ func OptNvidia(nv bool, nvccli bool) Option { // OptNoNvidia disables NVIDIA GPU support, even if enabled via apptainer.conf. func OptNoNvidia(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.NoNvidia = b return nil } @@ -270,7 +249,7 @@ func OptNoNvidia(b bool) Option { // OptRocm enable Rocm GPU support. func OptRocm(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.Rocm = b return nil } @@ -278,7 +257,7 @@ func OptRocm(b bool) Option { // OptNoRocm disables Rocm GPU support, even if enabled via apptainer.conf. func OptNoRocm(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.NoRocm = b return nil } @@ -286,7 +265,7 @@ func OptNoRocm(b bool) Option { // OptContainLibs mounts specified libraries into the container .singularity.d/libs dir. func OptContainLibs(cl []string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.ContainLibs = cl return nil } @@ -298,7 +277,7 @@ func OptContainLibs(cl []string) Option { // env is a map of name=value env vars to set. // clean removes host variables from the container environment. func OptEnv(env map[string]string, envFile string, clean bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.Env = env lo.EnvFile = envFile lo.CleanEnv = clean @@ -308,7 +287,7 @@ func OptEnv(env map[string]string, envFile string, clean bool) Option { // OptNoEval disables shell evaluation of args and env vars. func OptNoEval(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.NoEval = b return nil } @@ -316,7 +295,7 @@ func OptNoEval(b bool) Option { // OptNamespaces enable the individual kernel-support namespaces for the container. func OptNamespaces(n Namespaces) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.Namespaces = n return nil } @@ -327,7 +306,7 @@ func OptNamespaces(n Namespaces) Option { // network is the name of the CNI configuration to enable. // args are arguments to pass to the CNI plugin. func OptNetwork(network string, args []string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.Network = network lo.NetworkArgs = args return nil @@ -336,7 +315,7 @@ func OptNetwork(network string, args []string) Option { // OptHostname sets a hostname for the container (infers/requires UTS namespace). func OptHostname(h string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.Hostname = h return nil } @@ -344,7 +323,7 @@ func OptHostname(h string) Option { // OptDNS sets a DNS entry for the container resolv.conf. func OptDNS(d string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.DNS = d return nil } @@ -352,7 +331,7 @@ func OptDNS(d string) Option { // OptCaps sets capabilities to add and drop. func OptCaps(add, drop string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.AddCaps = add lo.DropCaps = drop return nil @@ -361,7 +340,7 @@ func OptCaps(add, drop string) Option { // OptAllowSUID permits setuid executables inside a container started by the root user. func OptAllowSUID(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.AllowSUID = b return nil } @@ -369,7 +348,7 @@ func OptAllowSUID(b bool) Option { // OptKeepPrivs keeps all privileges inside a container started by the root user. func OptKeepPrivs(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.KeepPrivs = b return nil } @@ -377,7 +356,7 @@ func OptKeepPrivs(b bool) Option { // OptNoPrivs drops all privileges inside a container. func OptNoPrivs(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.NoPrivs = b return nil } @@ -385,7 +364,7 @@ func OptNoPrivs(b bool) Option { // OptSecurity supplies a list of security options (selinux, apparmor, seccomp) to apply. func OptSecurity(s []string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.SecurityOpts = s return nil } @@ -393,7 +372,7 @@ func OptSecurity(s []string) Option { // OptNoUmask disables propagation of the host umask into the container, using a default 0022. func OptNoUmask(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.NoUmask = b return nil } @@ -401,7 +380,7 @@ func OptNoUmask(b bool) Option { // OptCgroupsJSON sets a Cgroups resource limit configuration to apply to the container. func OptCgroupsJSON(cj string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.CGroupsJSON = cj return nil } @@ -409,7 +388,7 @@ func OptCgroupsJSON(cj string) Option { // OptConfigFile specifies an alternate apptainer.conf that will be used by unprivileged installations only. func OptConfigFile(c string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.ConfigFile = c return nil } @@ -417,7 +396,7 @@ func OptConfigFile(c string) Option { // OptShellPath specifies a custom shell executable to be launched in the container. func OptShellPath(s string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.ShellPath = s return nil } @@ -425,7 +404,7 @@ func OptShellPath(s string) Option { // OptCwdPath specifies the initial working directory in the container. func OptCwdPath(p string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.CwdPath = p return nil } @@ -433,7 +412,7 @@ func OptCwdPath(p string) Option { // OptFakeroot enables the fake root mode, using user namespaces and subuid / subgid mapping. func OptFakeroot(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.Fakeroot = b return nil } @@ -441,7 +420,7 @@ func OptFakeroot(b bool) Option { // OptBoot enables execution of /sbin/init on startup of an instance container. func OptBoot(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.Boot = b return nil } @@ -449,7 +428,7 @@ func OptBoot(b bool) Option { // OptNoInit disables shim process when PID namespace is used. func OptNoInit(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.NoInit = b return nil } @@ -457,7 +436,7 @@ func OptNoInit(b bool) Option { // OptContain starts the container with minimal /dev and empty home/tmp mounts. func OptContain(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.Contain = b return nil } @@ -465,7 +444,7 @@ func OptContain(b bool) Option { // OptContainAll infers Contain, and adds PID, IPC namespaces, and CleanEnv. func OptContainAll(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.ContainAll = b return nil } @@ -473,7 +452,7 @@ func OptContainAll(b bool) Option { // OptAppName sets a SCIF application name to run. func OptAppName(a string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.AppName = a return nil } @@ -481,7 +460,7 @@ func OptAppName(a string) Option { // OptKeyInfo sets encryption key material to use when accessing an encrypted container image. func OptKeyInfo(ki *cryptkey.KeyInfo) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.KeyInfo = ki return nil } @@ -489,7 +468,7 @@ func OptKeyInfo(ki *cryptkey.KeyInfo) Option { // CacheDisabled indicates caching of images was disabled in the CLI. func OptCacheDisabled(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.CacheDisabled = b return nil } @@ -497,7 +476,7 @@ func OptCacheDisabled(b bool) Option { // OptDMTCPLaunch func OptDMTCPLaunch(a string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.DMTCPLaunch = a return nil } @@ -505,7 +484,7 @@ func OptDMTCPLaunch(a string) Option { // OptDMTCPRestart func OptDMTCPRestart(a string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.DMTCPRestart = a return nil } @@ -513,7 +492,7 @@ func OptDMTCPRestart(a string) Option { // OptUnsquash func OptUnsquash(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.Unsquash = b return nil } @@ -521,7 +500,7 @@ func OptUnsquash(b bool) Option { // OptIgnoreSubuid func OptIgnoreSubuid(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.IgnoreSubuid = b return nil } @@ -529,7 +508,7 @@ func OptIgnoreSubuid(b bool) Option { // OptIgnoreFakerootCmd func OptIgnoreFakerootCmd(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.IgnoreFakerootCmd = b return nil } @@ -537,7 +516,7 @@ func OptIgnoreFakerootCmd(b bool) Option { // OptIgnoreUserns func OptIgnoreUserns(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.IgnoreUserns = b return nil } @@ -545,7 +524,7 @@ func OptIgnoreUserns(b bool) Option { // OptUseBuildConfig func OptUseBuildConfig(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.UseBuildConfig = b return nil } @@ -553,7 +532,7 @@ func OptUseBuildConfig(b bool) Option { // OptTmpDir func OptTmpDir(a string) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.TmpDir = a return nil } @@ -561,7 +540,7 @@ func OptTmpDir(a string) Option { // OptUnderlay func OptUnderlay(b bool) Option { - return func(lo *launchOptions) error { + return func(lo *Options) error { lo.Underlay = b return nil } diff --git a/internal/pkg/runtime/launcher/util.go b/internal/pkg/runtime/launcher/util.go new file mode 100644 index 0000000000..f294df8fa3 --- /dev/null +++ b/internal/pkg/runtime/launcher/util.go @@ -0,0 +1,50 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2022, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package launcher + +import ( + "fmt" + "strings" + + "github.com/apptainer/apptainer/pkg/sylog" + "github.com/apptainer/apptainer/pkg/util/fs/proc" +) + +// WithPrivilege calls fn if cond is satisfied, and we are uid 0 +func WithPrivilege(uid uint32, cond bool, desc string, fn func() error) error { + if !cond { + return nil + } + if uid != 0 { + return fmt.Errorf("%s requires root privileges", desc) + } + return fn() +} + +// HidepidProc checks if hidepid is set on the /proc mount point. +// +// If this is set then an instance started in the with setuid workflow cannot be +func HidepidProc() bool { + entries, err := proc.GetMountInfoEntry("/proc/self/mountinfo") + if err != nil { + sylog.Warningf("while reading /proc/self/mountinfo: %s", err) + return false + } + for _, e := range entries { + if e.Point == "/proc" { + for _, o := range e.SuperOptions { + if strings.HasPrefix(o, "hidepid=") { + return true + } + } + } + } + return false +} From c4548b57725b11688176795718305627649c1c9c Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Mon, 10 Oct 2022 14:55:15 +0100 Subject: [PATCH 002/114] launcher: add placeholder OCI launcher Accepts no options, fails if options provided. Implements an Exec method which does nothing. Fixes sylabs/singularity#1021 Signed-off-by: Edita Kizinevic --- .../runtime/launcher/oci/launcher_linux.go | 234 ++++++++++++++++++ .../launcher/oci/launcher_linux_test.go | 71 ++++++ 2 files changed, 305 insertions(+) create mode 100644 internal/pkg/runtime/launcher/oci/launcher_linux.go create mode 100644 internal/pkg/runtime/launcher/oci/launcher_linux_test.go diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go new file mode 100644 index 0000000000..eb8cc56f7e --- /dev/null +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -0,0 +1,234 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2022, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +// Package oci implements a Launcher that will configure and launch a container +// with an OCI runtime. +package oci + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/apptainer/apptainer/internal/pkg/runtime/launcher" +) + +var ( + ErrUnsupportedOption = errors.New("not supported by OCI launcher") + ErrNotImplemented = errors.New("not implemented") +) + +// Launcher will holds configuration for, and will launch a container using an +// OCI runtime. +type Launcher struct { + cfg launcher.Options +} + +// NewLauncher returns a oci.Launcher with an initial configuration set by opts. +func NewLauncher(opts ...launcher.Option) (*Launcher, error) { + lo := launcher.Options{} + for _, opt := range opts { + if err := opt(&lo); err != nil { + return nil, fmt.Errorf("%w", err) + } + } + + if err := checkOpts(lo); err != nil { + return nil, err + } + return &Launcher{lo}, nil +} + +// checkOpts ensures that options set are supported by the oci.Launcher. +// +// nolint:maintidx +func checkOpts(lo launcher.Options) error { + badOpt := []string{} + + if lo.Writable { + badOpt = append(badOpt, "Writable") + } + if lo.WritableTmpfs { + badOpt = append(badOpt, "WritableTmpfs") + } + if lo.OverlayPaths != nil { + badOpt = append(badOpt, "OverlayPaths") + } + if lo.ScratchDirs != nil { + badOpt = append(badOpt, "ScratchDirs") + } + if lo.WorkDir != "" { + badOpt = append(badOpt, "WorkDir") + } + + // Home is always sent from the CLI, and must be valid as an option, but + // CustomHome signifies if it was a user specified value which we don't + // support (yet). + if lo.CustomHome { + badOpt = append(badOpt, "CustomHome") + } + if lo.NoHome { + badOpt = append(badOpt, "NoHome") + } + + if lo.BindPaths != nil { + badOpt = append(badOpt, "BindPaths") + } + if lo.FuseMount != nil { + badOpt = append(badOpt, "FuseMount") + } + if lo.Mounts != nil { + badOpt = append(badOpt, "Mounts") + } + if lo.NoMount != nil { + badOpt = append(badOpt, "NoMount") + } + + if lo.Nvidia { + badOpt = append(badOpt, "Nvidia") + } + if lo.NvCCLI { + badOpt = append(badOpt, "NvCCLI") + } + if lo.NoNvidia { + badOpt = append(badOpt, "NoNvidia") + } + if lo.Rocm { + badOpt = append(badOpt, "Rocm") + } + if lo.NoRocm { + badOpt = append(badOpt, "NoRocm") + } + + if lo.ContainLibs != nil { + badOpt = append(badOpt, "ContainLibs") + } + + if lo.Env != nil { + badOpt = append(badOpt, "Env") + } + if lo.EnvFile != "" { + badOpt = append(badOpt, "EnvFile") + } + if lo.CleanEnv { + badOpt = append(badOpt, "CleanEnv") + } + if lo.NoEval { + badOpt = append(badOpt, "NoEval") + } + + if lo.Namespaces.IPC { + badOpt = append(badOpt, "Namespaces.IPC") + } + if lo.Namespaces.Net { + badOpt = append(badOpt, "Namespaces.Net") + } + if lo.Namespaces.PID { + badOpt = append(badOpt, "Namespaces.PID") + } + if lo.Namespaces.UTS { + badOpt = append(badOpt, "Namespaces.UTS") + } + if lo.Namespaces.User { + badOpt = append(badOpt, "Namespaces.User") + } + + if lo.Network != "" { + badOpt = append(badOpt, "Network") + } + if lo.NetworkArgs != nil { + badOpt = append(badOpt, "NetworkArgs") + } + if lo.Hostname != "" { + badOpt = append(badOpt, "Hostname") + } + if lo.DNS != "" { + badOpt = append(badOpt, "DNS") + } + + if lo.AddCaps != "" { + badOpt = append(badOpt, "AddCaps") + } + if lo.DropCaps != "" { + badOpt = append(badOpt, "DropCaps") + } + if lo.AllowSUID { + badOpt = append(badOpt, "AllowSUID") + } + if lo.KeepPrivs { + badOpt = append(badOpt, "KeepPrivs") + } + if lo.NoPrivs { + badOpt = append(badOpt, "NoPrivs") + } + if lo.SecurityOpts != nil { + badOpt = append(badOpt, "SecurityOpts") + } + if lo.NoUmask { + badOpt = append(badOpt, "NoUmask") + } + + if lo.CGroupsJSON != "" { + badOpt = append(badOpt, "CGroupsJSON") + } + + if lo.ConfigFile != "" { + badOpt = append(badOpt, "ConfigFile") + } + + if lo.ShellPath != "" { + badOpt = append(badOpt, "ShellPath") + } + if lo.PwdPath != "" { + badOpt = append(badOpt, "PwdPath") + } + + if lo.Fakeroot { + badOpt = append(badOpt, "Fakeroot") + } + if lo.Boot { + badOpt = append(badOpt, "Boot") + } + if lo.NoInit { + badOpt = append(badOpt, "NoInit") + } + if lo.Contain { + badOpt = append(badOpt, "Contain") + } + if lo.ContainAll { + badOpt = append(badOpt, "ContainAll") + } + + if lo.AppName != "" { + badOpt = append(badOpt, "AppName") + } + + if lo.KeyInfo != nil { + badOpt = append(badOpt, "KeyInfo") + } + + if lo.SIFFUSE { + badOpt = append(badOpt, "SIFFUSE") + } + if lo.CacheDisabled { + badOpt = append(badOpt, "CacheDisabled") + } + + if len(badOpt) > 0 { + return fmt.Errorf("%w: %s", ErrUnsupportedOption, strings.Join(badOpt, ",")) + } + + return nil +} + +// Exec is not yet implemented. +func (l *Launcher) Exec(ctx context.Context, image string, args []string, instanceName string) error { + return ErrNotImplemented +} diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux_test.go b/internal/pkg/runtime/launcher/oci/launcher_linux_test.go new file mode 100644 index 0000000000..e75452fb41 --- /dev/null +++ b/internal/pkg/runtime/launcher/oci/launcher_linux_test.go @@ -0,0 +1,71 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2022, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package oci + +import ( + "context" + "reflect" + "testing" + + "github.com/apptainer/apptainer/internal/pkg/runtime/launcher" +) + +func TestNewLauncher(t *testing.T) { + tests := []struct { + name string + opts []launcher.Option + want *Launcher + wantErr bool + }{ + { + name: "default", + want: &Launcher{}, + wantErr: false, + }, + { + name: "validOption", + opts: []launcher.Option{ + launcher.OptHome("/home/test", false, false), + }, + want: &Launcher{cfg: launcher.Options{HomeDir: "/home/test"}}, + }, + { + name: "unsupportedOption", + opts: []launcher.Option{ + launcher.OptCacheDisabled(true), + }, + want: nil, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := NewLauncher(tt.opts...) + if (err != nil) != tt.wantErr { + t.Errorf("NewLauncher() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("NewLauncher() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestExec(t *testing.T) { + l, err := NewLauncher([]launcher.Option{}...) + if err != nil { + t.Errorf("Couldn't initialize launcher: %s", err) + } + + if err := l.Exec(context.Background(), "", []string{}, ""); err != ErrNotImplemented { + t.Errorf("Expected %v, got %v", ErrNotImplemented, err) + } +} From dffd2c8ff983e8d73db7f15b42ba66b17eba1cd1 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Tue, 11 Oct 2022 11:18:31 +0100 Subject: [PATCH 003/114] actions: Add --oci flag to call OCI launcher Signed-off-by: Edita Kizinevic --- cmd/internal/cli/action_flags.go | 13 +++++ cmd/internal/cli/actions.go | 17 ++++-- e2e/actions/actions.go | 23 +++++++- e2e/internal/e2e/apptainercmd.go | 4 +- e2e/internal/e2e/profile.go | 91 +++++++++++++++++++++++++++----- 5 files changed, 128 insertions(+), 20 deletions(-) diff --git a/cmd/internal/cli/action_flags.go b/cmd/internal/cli/action_flags.go index 6947c35b66..60b38f1243 100644 --- a/cmd/internal/cli/action_flags.go +++ b/cmd/internal/cli/action_flags.go @@ -95,6 +95,8 @@ var ( ignoreUserns bool underlay bool // whether using underlay instead of overlay + + ociRuntime bool ) // --app @@ -875,6 +877,16 @@ var actionUnderlayFlag = cmdline.Flag{ Hidden: false, } +// --oci +var actionOCIFlag = cmdline.Flag{ + ID: "actionOCI", + Value: &ociRuntime, + DefaultValue: false, + Name: "oci", + Usage: "Launch container with OCI runtime (experimental)", + EnvKeys: []string{"OCI"}, +} + func init() { addCmdInit(func(cmdManager *cmdline.CommandManager) { cmdManager.RegisterCmd(ExecCmd) @@ -972,5 +984,6 @@ func init() { cmdManager.RegisterFlagForCmd(&actionIgnoreFakerootCommand, actionsInstanceCmd...) cmdManager.RegisterFlagForCmd(&actionIgnoreUsernsFlag, actionsInstanceCmd...) cmdManager.RegisterFlagForCmd(&actionUnderlayFlag, actionsInstanceCmd...) + cmdManager.RegisterFlagForCmd(&actionOCIFlag, actionsCmd...) }) } diff --git a/cmd/internal/cli/actions.go b/cmd/internal/cli/actions.go index ffc3afbcaf..448b49c6b7 100644 --- a/cmd/internal/cli/actions.go +++ b/cmd/internal/cli/actions.go @@ -26,6 +26,7 @@ import ( "github.com/apptainer/apptainer/internal/pkg/client/shub" "github.com/apptainer/apptainer/internal/pkg/runtime/launcher" "github.com/apptainer/apptainer/internal/pkg/runtime/launcher/native" + ocilauncher "github.com/apptainer/apptainer/internal/pkg/runtime/launcher/oci" "github.com/apptainer/apptainer/internal/pkg/util/env" "github.com/apptainer/apptainer/internal/pkg/util/uri" "github.com/apptainer/apptainer/pkg/sylog" @@ -363,12 +364,20 @@ func launchContainer(cmd *cobra.Command, image string, args []string, instanceNa launcher.OptUnderlay(underlay), } - // Explicitly use the interface type here, as we will add alternative launchers later... var l launcher.Launcher - l, err = native.NewLauncher(opts...) - if err != nil { - return fmt.Errorf("while configuring container: %s", err) + if ociRuntime { + sylog.Debugf("Using OCI runtime launcher.") + l, err = ocilauncher.NewLauncher(opts...) + if err != nil { + return fmt.Errorf("while configuring container: %s", err) + } + } else { + sylog.Debugf("Using native runtime launcher.") + l, err = native.NewLauncher(opts...) + if err != nil { + return fmt.Errorf("while configuring container: %s", err) + } } return l.Exec(cmd.Context(), image, args, instanceName) diff --git a/e2e/actions/actions.go b/e2e/actions/actions.go index 5a87a6b7c6..866bc2a50e 100644 --- a/e2e/actions/actions.go +++ b/e2e/actions/actions.go @@ -929,7 +929,7 @@ func (c actionTests) actionBasicProfiles(t *testing.T) { }, } - for _, profile := range e2e.Profiles { + for _, profile := range e2e.NativeProfiles { profile := profile t.Run(profile.String(), func(t *testing.T) { @@ -1850,7 +1850,7 @@ func (c actionTests) actionBinds(t *testing.T) { }, } - for _, profile := range e2e.Profiles { + for _, profile := range e2e.NativeProfiles { profile := profile createWorkspaceDirs(t) @@ -2861,6 +2861,24 @@ func (c actionTests) relWorkdirScratch(t *testing.T) { } } +func (c actionTests) ociRuntime(t *testing.T) { + e2e.EnsureImage(t, c.env) + + for _, p := range []e2e.Profile{e2e.OCIUserProfile, e2e.OCIRootProfile} { + c.env.RunApptainer( + t, + e2e.AsSubtest(p.String()), + e2e.WithProfile(p), + e2e.WithCommand("exec"), + e2e.WithArgs(c.env.ImagePath, "/bin/true"), + e2e.ExpectExit( + 255, + e2e.ExpectError(e2e.ContainMatch, "not implemented"), + ), + ) + } +} + // E2ETests is the main func to trigger the test suite func E2ETests(env e2e.TestEnv) testhelper.Tests { c := actionTests{ @@ -2906,6 +2924,7 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { "no-mount": c.actionNoMount, // test --no-mount "compat": np(c.actionCompat), // test --compat "umask": np(c.actionUmask), // test umask propagation + "ociRuntime": c.ociRuntime, // test --oci (unimplemented) "invalidRemote": np(c.invalidRemote), // GHSA-5mv9-q7fq-9394 "fakeroot home": c.actionFakerootHome, // test home dir in fakeroot "relWorkdirScratch": np(c.relWorkdirScratch), // test relative --workdir with --scratch diff --git a/e2e/internal/e2e/apptainercmd.go b/e2e/internal/e2e/apptainercmd.go index 3b36416a04..47af2c1dfa 100644 --- a/e2e/internal/e2e/apptainercmd.go +++ b/e2e/internal/e2e/apptainercmd.go @@ -496,8 +496,8 @@ func (env TestEnv) RunApptainer(t *testing.T, cmdOps ...ApptainerCmdOp) { // a profile is required if s.profile.name == "" { i := 0 - availableProfiles := make([]string, len(Profiles)) - for profile := range Profiles { + availableProfiles := make([]string, len(NativeProfiles)) + for profile := range NativeProfiles { availableProfiles[i] = profile i++ } diff --git a/e2e/internal/e2e/profile.go b/e2e/internal/e2e/profile.go index 3fe4868ce8..ce1cbce315 100644 --- a/e2e/internal/e2e/profile.go +++ b/e2e/internal/e2e/profile.go @@ -24,19 +24,29 @@ const ( fakerootProfile = "FakerootProfile" userNamespaceProfile = "UserNamespaceProfile" rootUserNamespaceProfile = "RootUserNamespaceProfile" + ociUserProfile = "OCIUserProfile" + ociRootProfile = "OCIRootProfile" + ociFakerootProfile = "OCIFakerootProfile" ) var ( - // UserProfile is the execution profile for a regular user. - UserProfile = Profiles[userProfile] - // RootProfile is the execution profile for root. - RootProfile = Profiles[rootProfile] - // FakerootProfile is the execution profile for fakeroot. - FakerootProfile = Profiles[fakerootProfile] - // UserNamespaceProfile is the execution profile for a regular user and a user namespace. - UserNamespaceProfile = Profiles[userNamespaceProfile] - // RootUserNamespaceProfile is the execution profile for root and a user namespace. - RootUserNamespaceProfile = Profiles[rootUserNamespaceProfile] + // UserProfile is the execution profile for a regular user, using the Apptainer native runtime. + UserProfile = NativeProfiles[userProfile] + // RootProfile is the execution profile for root, using the Apptainer native runtime. + RootProfile = NativeProfiles[rootProfile] + // FakerootProfile is the execution profile for fakeroot, using the Apptainer native runtime. + FakerootProfile = NativeProfiles[fakerootProfile] + // UserNamespaceProfile is the execution profile for a regular user and a user namespace, using the Apptainer native runtime. + UserNamespaceProfile = NativeProfiles[userNamespaceProfile] + // RootUserNamespaceProfile is the execution profile for root and a user namespace, using the Apptainer native runtime. + RootUserNamespaceProfile = NativeProfiles[rootUserNamespaceProfile] + // OCIUserProfile is the execution profile for a regular user, using the Apptainer native runtime. + OCIUserProfile = OCIProfiles[ociUserProfile] + // RootProfile is the execution profile for root, using the Apptainer native runtime. + OCIRootProfile = OCIProfiles[ociRootProfile] + // FakerootProfile is the execution profile for fakeroot, using the Apptainer native runtime. + OCIFakerootProfile = OCIProfiles[ociFakerootProfile] + // UserNamespaceProfile is the execution profile for a regular user and a user namespace, using the Apptainer native runtime. ) // Profile represents various properties required to run an E2E test @@ -60,8 +70,8 @@ type Profile struct { optionForCommands []string // apptainer commands concerned by the option to be added } -// Profiles defines all available profiles. -var Profiles = map[string]Profile{ +// NativeProfiles defines all available profiles for the native apptainer runtime +var NativeProfiles = map[string]Profile{ userProfile: { name: "User", privileged: false, @@ -114,6 +124,40 @@ var Profiles = map[string]Profile{ }, } +// OCIProfiles defines all available profiles for the OCI runtime +var OCIProfiles = map[string]Profile{ + ociUserProfile: { + name: "OCIUser", + privileged: false, + hostUID: origUID, + containerUID: origUID, + defaultCwd: "", + requirementsFn: ociRequirements, + apptainerOption: "--oci", + optionForCommands: []string{"shell", "exec", "run", "test", "instance start"}, + }, + ociRootProfile: { + name: "OCIRoot", + privileged: true, + hostUID: 0, + containerUID: 0, + defaultCwd: "", + requirementsFn: ociRequirements, + apptainerOption: "--oci", + optionForCommands: []string{"shell", "exec", "run", "test", "instance start"}, + }, + ociFakerootProfile: { + name: "OCIFakeroot", + privileged: false, + hostUID: origUID, + containerUID: 0, + defaultCwd: "", + requirementsFn: ociRequirements, + apptainerOption: "--oci --fakeroot", + optionForCommands: []string{"shell", "exec", "run", "test", "instance start"}, + }, +} + // Privileged returns whether the test should be executed with // elevated privileges or not. func (p Profile) Privileged() bool { @@ -205,3 +249,26 @@ func fakerootRequirements(t *testing.T) { t.Fatalf("fakeroot configuration error: %s", err) } } + +// ociRequirements ensures requirements are satisfied to correctly execute +// commands with the OCI runtime / profile. +func ociRequirements(t *testing.T) { + require.UserNamespace(t) + require.Command(t, "runc") + + uid := uint32(origUID) + + // check that current user has valid mappings in /etc/subuid + if _, err := fakeroot.GetIDRange(fakeroot.SubUIDFile, uid); err != nil { + t.Fatalf("fakeroot configuration error: %s", err) + } + + // check that current user has valid mappings in /etc/subgid; + // since that file contains the group mappings for a given user + // *name*, it is keyed by user name, not by group name. This + // means that even if we are requesting the *group* mappings, we + // need to pass the *user* ID. + if _, err := fakeroot.GetIDRange(fakeroot.SubGIDFile, uid); err != nil { + t.Fatalf("fakeroot configuration error: %s", err) + } +} From d62fe20259407e77f37e3ac5530bad889d908534 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Tue, 11 Oct 2022 11:30:25 +0100 Subject: [PATCH 004/114] launcher: fix OCI launcher supported option checks Handle empty structs (not just nils). Handle always-set network, config file. Signed-off-by: Edita Kizinevic --- .../runtime/launcher/oci/launcher_linux.go | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index eb8cc56f7e..ca4151be86 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -17,12 +17,13 @@ import ( "fmt" "strings" + "github.com/apptainer/apptainer/internal/pkg/buildcfg" "github.com/apptainer/apptainer/internal/pkg/runtime/launcher" ) var ( ErrUnsupportedOption = errors.New("not supported by OCI launcher") - ErrNotImplemented = errors.New("not implemented") + ErrNotImplemented = errors.New("not implemented by OCI launcher") ) // Launcher will holds configuration for, and will launch a container using an @@ -58,10 +59,10 @@ func checkOpts(lo launcher.Options) error { if lo.WritableTmpfs { badOpt = append(badOpt, "WritableTmpfs") } - if lo.OverlayPaths != nil { + if len(lo.OverlayPaths) > 0 { badOpt = append(badOpt, "OverlayPaths") } - if lo.ScratchDirs != nil { + if len(lo.ScratchDirs) > 0 { badOpt = append(badOpt, "ScratchDirs") } if lo.WorkDir != "" { @@ -78,16 +79,16 @@ func checkOpts(lo launcher.Options) error { badOpt = append(badOpt, "NoHome") } - if lo.BindPaths != nil { + if len(lo.BindPaths) > 0 { badOpt = append(badOpt, "BindPaths") } - if lo.FuseMount != nil { + if len(lo.FuseMount) > 0 { badOpt = append(badOpt, "FuseMount") } - if lo.Mounts != nil { + if len(lo.Mounts) > 0 { badOpt = append(badOpt, "Mounts") } - if lo.NoMount != nil { + if len(lo.NoMount) > 0 { badOpt = append(badOpt, "NoMount") } @@ -107,11 +108,11 @@ func checkOpts(lo launcher.Options) error { badOpt = append(badOpt, "NoRocm") } - if lo.ContainLibs != nil { + if len(lo.ContainLibs) > 0 { badOpt = append(badOpt, "ContainLibs") } - if lo.Env != nil { + if len(lo.Env) > 0 { badOpt = append(badOpt, "Env") } if lo.EnvFile != "" { @@ -140,10 +141,12 @@ func checkOpts(lo launcher.Options) error { badOpt = append(badOpt, "Namespaces.User") } - if lo.Network != "" { + // Network always set in CLI layer even if network namespace not requested. + if lo.Namespaces.Net && lo.Network != "" { badOpt = append(badOpt, "Network") } - if lo.NetworkArgs != nil { + + if len(lo.NetworkArgs) > 0 { badOpt = append(badOpt, "NetworkArgs") } if lo.Hostname != "" { @@ -168,7 +171,7 @@ func checkOpts(lo launcher.Options) error { if lo.NoPrivs { badOpt = append(badOpt, "NoPrivs") } - if lo.SecurityOpts != nil { + if len(lo.SecurityOpts) > 0 { badOpt = append(badOpt, "SecurityOpts") } if lo.NoUmask { @@ -179,7 +182,8 @@ func checkOpts(lo launcher.Options) error { badOpt = append(badOpt, "CGroupsJSON") } - if lo.ConfigFile != "" { + // ConfigFile always set by CLI. We should support only the default from build time. + if lo.ConfigFile != "" && lo.ConfigFile != buildcfg.APPTAINER_CONF_FILE { badOpt = append(badOpt, "ConfigFile") } From 99f68e82e64fd1e18044b7fbfd0d024b5b8de809 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Thu, 24 Feb 2022 13:56:12 -0600 Subject: [PATCH 005/114] oci: switch to runc for oci commands Replace invocations of SingularityCE's OCI runtime engine with wrapped invocations of runc. Signed-off-by: Edita Kizinevic --- CHANGELOG.md | 3 + cmd/internal/cli/oci_linux.go | 43 +------ internal/app/apptainer/oci_attach_linux.go | 140 +-------------------- internal/app/apptainer/oci_create_linux.go | 63 ++-------- internal/app/apptainer/oci_delete_linux.go | 42 ++----- internal/app/apptainer/oci_exec_linux.go | 33 ++--- internal/app/apptainer/oci_kill_linux.go | 61 ++------- internal/app/apptainer/oci_linux.go | 87 ++----------- internal/app/apptainer/oci_pause_linux.go | 73 ++++------- internal/app/apptainer/oci_run_linux.go | 99 +++------------ internal/app/apptainer/oci_start_linux.go | 46 ++----- internal/app/apptainer/oci_state_linux.go | 30 ++--- internal/app/apptainer/oci_update_linux.go | 54 ++------ 13 files changed, 131 insertions(+), 643 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f72ae6ca1f..22601e6bd6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,9 @@ For older changes see the [archived Singularity change log](https://github.com/a working directory, though `--pwd` is still supported for compatibility. - When building RPM, we will now use `/var/lib/apptainer` (rather than `/var/apptainer`) to store local state files. +- The `apptainer oci` command group now uses `runc` to manage containers. +- The `apptainer oci` flags `--sync-socket`, `--empty-process`, and + `--timeout` have been removed. ### New Features & Functionality diff --git a/cmd/internal/cli/oci_linux.go b/cmd/internal/cli/oci_linux.go index 4113e85feb..d38d139c24 100644 --- a/cmd/internal/cli/oci_linux.go +++ b/cmd/internal/cli/oci_linux.go @@ -32,28 +32,6 @@ var ociBundleFlag = cmdline.Flag{ EnvKeys: []string{"BUNDLE"}, } -// -s|--sync-socket -var ociSyncSocketFlag = cmdline.Flag{ - ID: "ociSyncSocketFlag", - Value: &ociArgs.SyncSocketPath, - DefaultValue: "", - Name: "sync-socket", - ShortHand: "s", - Usage: "specify the path to unix socket for state synchronization", - Tag: "", - EnvKeys: []string{"SYNC_SOCKET"}, -} - -// --empty-process -var ociCreateEmptyProcessFlag = cmdline.Flag{ - ID: "ociCreateEmptyProcessFlag", - Value: &ociArgs.EmptyProcess, - DefaultValue: false, - Name: "empty-process", - Usage: "run container without executing container process (eg: for POD container)", - EnvKeys: []string{"EMPTY_PROCESS"}, -} - // -l|--log-path var ociLogPathFlag = cmdline.Flag{ ID: "ociLogPathFlag", @@ -111,16 +89,6 @@ var ociKillForceFlag = cmdline.Flag{ EnvKeys: []string{"FORCE"}, } -// -t|--timeout -var ociKillTimeoutFlag = cmdline.Flag{ - ID: "ociKillTimeoutFlag", - Value: &ociArgs.KillTimeout, - DefaultValue: uint32(0), - Name: "timeout", - ShortHand: "t", - Usage: "timeout in second before killing container", -} - // -f|--from-file var ociUpdateFromFileFlag = cmdline.Flag{ ID: "ociUpdateFromFileFlag", @@ -153,16 +121,12 @@ func init() { createRunCmd := cmdManager.GetCmdGroup("create_run") cmdManager.RegisterFlagForCmd(&ociBundleFlag, createRunCmd...) - cmdManager.RegisterFlagForCmd(&ociSyncSocketFlag, createRunCmd...) cmdManager.RegisterFlagForCmd(&ociLogPathFlag, createRunCmd...) cmdManager.RegisterFlagForCmd(&ociLogFormatFlag, createRunCmd...) cmdManager.RegisterFlagForCmd(&ociPidFileFlag, createRunCmd...) - cmdManager.RegisterFlagForCmd(&ociCreateEmptyProcessFlag, OciCreateCmd) cmdManager.RegisterFlagForCmd(&ociKillForceFlag, OciKillCmd) cmdManager.RegisterFlagForCmd(&ociKillSignalFlag, OciKillCmd) - cmdManager.RegisterFlagForCmd(&ociKillTimeoutFlag, OciKillCmd) cmdManager.RegisterFlagForCmd(&ociUpdateFromFileFlag, OciUpdateCmd) - cmdManager.RegisterFlagForCmd(&ociSyncSocketFlag, OciStateCmd) }) } @@ -236,7 +200,6 @@ var OciKillCmd = &cobra.Command{ DisableFlagsInUseLine: true, PreRun: CheckRoot, Run: func(cmd *cobra.Command, args []string) { - timeout := int(ociArgs.KillTimeout) killSignal := "" if len(args) > 1 && args[1] != "" { killSignal = args[1] @@ -246,7 +209,7 @@ var OciKillCmd = &cobra.Command{ if ociArgs.ForceKill { killSignal = "SIGKILL" } - if err := apptainer.OciKill(args[0], killSignal, timeout); err != nil { + if err := apptainer.OciKill(args[0], killSignal); err != nil { sylog.Fatalf("%s", err) } }, @@ -326,7 +289,7 @@ var OciPauseCmd = &cobra.Command{ DisableFlagsInUseLine: true, PreRun: CheckRoot, Run: func(cmd *cobra.Command, args []string) { - if err := apptainer.OciPauseResume(args[0], true); err != nil { + if err := apptainer.OciPause(args[0]); err != nil { sylog.Fatalf("%s", err) } }, @@ -342,7 +305,7 @@ var OciResumeCmd = &cobra.Command{ DisableFlagsInUseLine: true, PreRun: CheckRoot, Run: func(cmd *cobra.Command, args []string) { - if err := apptainer.OciPauseResume(args[0], false); err != nil { + if err := apptainer.OciResume(args[0]); err != nil { sylog.Fatalf("%s", err) } }, diff --git a/internal/app/apptainer/oci_attach_linux.go b/internal/app/apptainer/oci_attach_linux.go index c800ebc233..f5086842bc 100644 --- a/internal/app/apptainer/oci_attach_linux.go +++ b/internal/app/apptainer/oci_attach_linux.go @@ -11,148 +11,10 @@ package apptainer import ( "context" - "encoding/json" "fmt" - "io" - "net" - "os" - osignal "os/signal" - "sync" - "syscall" - - "github.com/creack/pty" - - "github.com/apptainer/apptainer/internal/pkg/runtime/engine/oci" - "github.com/apptainer/apptainer/pkg/ociruntime" - "github.com/apptainer/apptainer/pkg/sylog" - "github.com/apptainer/apptainer/pkg/util/unix" - specs "github.com/opencontainers/runtime-spec/specs-go" - "golang.org/x/term" ) -func resize(controlSocket string, oversized bool) { - ctrl := &ociruntime.Control{} - ctrl.ConsoleSize = &specs.Box{} - - c, err := unix.Dial(controlSocket) - if err != nil { - sylog.Errorf("failed to connect to control socket") - return - } - defer c.Close() - - rows, cols, err := pty.Getsize(os.Stdin) - if err != nil { - sylog.Errorf("terminal resize error: %s", err) - return - } - - ctrl.ConsoleSize.Height = uint(rows) - ctrl.ConsoleSize.Width = uint(cols) - - if oversized { - ctrl.ConsoleSize.Height++ - ctrl.ConsoleSize.Width++ - } - - enc := json.NewEncoder(c) - if enc == nil { - sylog.Errorf("cannot instantiate JSON encoder") - return - } - - if err := enc.Encode(ctrl); err != nil { - sylog.Errorf("%s", err) - return - } -} - -func attach(engineConfig *oci.EngineConfig, run bool) error { - var ostate *term.State - var conn net.Conn - var wg sync.WaitGroup - - state := &engineConfig.State - - if state.AttachSocket == "" { - return fmt.Errorf("attach socket not available, container state: %s", state.Status) - } - if state.ControlSocket == "" { - return fmt.Errorf("control socket not available, container state: %s", state.Status) - } - - hasTerminal := engineConfig.OciConfig.Process.Terminal - if hasTerminal && !term.IsTerminal(0) { - return fmt.Errorf("attach requires a terminal when terminal config is set to true") - } - - var err error - conn, err = unix.Dial(state.AttachSocket) - if err != nil { - return err - } - defer conn.Close() - - if hasTerminal { - ostate, _ = term.MakeRaw(0) - resize(state.ControlSocket, true) - resize(state.ControlSocket, false) - } - - wg.Add(1) - - go func() { - // catch SIGWINCH signal for terminal resize - signals := make(chan os.Signal, 1) - pid := state.Pid - osignal.Notify(signals) - - for { - s := <-signals - switch s { - case syscall.SIGWINCH: - if hasTerminal { - resize(state.ControlSocket, false) - } - default: - syscall.Kill(pid, s.(syscall.Signal)) - } - } - }() - - if hasTerminal || !run { - // Pipe session to bash and visa-versa - go func() { - io.Copy(os.Stdout, conn) - wg.Done() - }() - go func() { - io.Copy(conn, os.Stdin) - }() - wg.Wait() - - if hasTerminal { - fmt.Printf("\r") - return term.Restore(0, ostate) - } - return nil - } - - io.Copy(io.Discard, conn) - return nil -} - // OciAttach attaches console to a running container func OciAttach(ctx context.Context, containerID string) error { - engineConfig, err := getEngineConfig(containerID) - if err != nil { - return err - } - if engineConfig.GetState().Status != ociruntime.Running { - return fmt.Errorf("could not attach to %s: not in running state", containerID) - } - - defer exitContainer(ctx, containerID, false) - - return attach(engineConfig, false) + return fmt.Errorf("TODO - NOT IMPLEMENTED") } diff --git a/internal/app/apptainer/oci_create_linux.go b/internal/app/apptainer/oci_create_linux.go index d19c6b4c7b..c204e51f83 100644 --- a/internal/app/apptainer/oci_create_linux.go +++ b/internal/app/apptainer/oci_create_linux.go @@ -10,27 +10,16 @@ package apptainer import ( - "encoding/json" "fmt" - "io" "os" "path/filepath" + "syscall" - "github.com/apptainer/apptainer/internal/pkg/runtime/engine/config/oci/generate" - "github.com/apptainer/apptainer/internal/pkg/runtime/engine/oci" - "github.com/apptainer/apptainer/internal/pkg/util/starter" - "github.com/apptainer/apptainer/pkg/runtime/engine/config" + "github.com/apptainer/apptainer/pkg/sylog" ) // OciCreate creates a container from an OCI bundle func OciCreate(containerID string, args *OciArgs) error { - _, err := getState(containerID) - if err == nil { - return fmt.Errorf("%s already exists", containerID) - } - - os.Clearenv() - absBundle, err := filepath.Abs(args.BundlePath) if err != nil { return fmt.Errorf("failed to determine bundle absolute path: %s", err) @@ -40,46 +29,20 @@ func OciCreate(containerID string, args *OciArgs) error { return fmt.Errorf("failed to change directory to %s: %s", absBundle, err) } - engineConfig := oci.NewConfig() - generator := generate.New(&engineConfig.OciConfig.Spec) - engineConfig.SetBundlePath(absBundle) - engineConfig.SetLogPath(args.LogPath) - engineConfig.SetLogFormat(args.LogFormat) - engineConfig.SetPidFile(args.PidFile) - - // load config.json from bundle path - configJSON := filepath.Join(absBundle, "config.json") - fb, err := os.Open(configJSON) - if err != nil { - return fmt.Errorf("oci specification file %q is missing or cannot be read", configJSON) + cmdArgs := []string{ + "--root=" + OciStateDir, + "create", + "-b", absBundle, } - - data, err := io.ReadAll(fb) - if err != nil { - return fmt.Errorf("failed to read OCI specification file %s: %s", configJSON, err) + if args.PidFile != "" { + cmdArgs = append(cmdArgs, "--pid-file="+args.PidFile) } + cmdArgs = append(cmdArgs, containerID) - fb.Close() - - if err := json.Unmarshal(data, generator.Config); err != nil { - return fmt.Errorf("failed to parse OCI specification file %s: %s", configJSON, err) - } - - engineConfig.EmptyProcess = args.EmptyProcess - engineConfig.SyncSocket = args.SyncSocketPath - - commonConfig := &config.Common{ - ContainerID: containerID, - EngineName: oci.Name, - EngineConfig: engineConfig, + sylog.Debugf("Calling runc with args %v", cmdArgs) + if err := syscall.Exec(runc, cmdArgs, []string{}); err != nil { + return fmt.Errorf("while calling runc: %w", err) } - procName := fmt.Sprintf("Apptainer OCI %s", containerID) - return starter.Run( - procName, - commonConfig, - starter.WithStdin(os.Stdin), - starter.WithStderr(os.Stderr), - starter.WithStdout(os.Stdout), - ) + return nil } diff --git a/internal/app/apptainer/oci_delete_linux.go b/internal/app/apptainer/oci_delete_linux.go index 4f59396f99..47f8ff8b92 100644 --- a/internal/app/apptainer/oci_delete_linux.go +++ b/internal/app/apptainer/oci_delete_linux.go @@ -12,47 +12,23 @@ package apptainer import ( "context" "fmt" + "syscall" - "github.com/apptainer/apptainer/internal/pkg/instance" - "github.com/apptainer/apptainer/internal/pkg/util/exec" - "github.com/apptainer/apptainer/pkg/ociruntime" "github.com/apptainer/apptainer/pkg/sylog" ) // OciDelete deletes container resources func OciDelete(ctx context.Context, containerID string) error { - engineConfig, err := getEngineConfig(containerID) - if err != nil { - return err + runcArgs := []string{ + "--root=" + OciStateDir, + "delete", + containerID, } - switch engineConfig.State.Status { - case ociruntime.Running: - return fmt.Errorf("cannot delete '%s', the state of the container must be created or stopped", containerID) - case ociruntime.Stopped: - case ociruntime.Created: - if err := OciKill(containerID, "SIGTERM", 2); err != nil { - return err - } - engineConfig, err = getEngineConfig(containerID) - if err != nil { - return err - } + sylog.Debugf("Calling runc with args %v", runcArgs) + if err := syscall.Exec(runc, runcArgs, []string{}); err != nil { + return fmt.Errorf("while calling runc: %w", err) } - hooks := engineConfig.OciConfig.Hooks - if hooks != nil { - for _, h := range hooks.Poststop { - if err := exec.Hook(ctx, &h, &engineConfig.State.State); err != nil { - sylog.Warningf("%s", err) - } - } - } - - // remove instance files - file, err := instance.Get(containerID, instance.OciSubDir) - if err != nil { - return err - } - return file.Delete() + return nil } diff --git a/internal/app/apptainer/oci_exec_linux.go b/internal/app/apptainer/oci_exec_linux.go index a94b404d52..8886a9bb91 100644 --- a/internal/app/apptainer/oci_exec_linux.go +++ b/internal/app/apptainer/oci_exec_linux.go @@ -11,35 +11,24 @@ package apptainer import ( "fmt" - "os" - "strings" + "syscall" - "github.com/apptainer/apptainer/internal/pkg/runtime/engine/oci" - "github.com/apptainer/apptainer/internal/pkg/util/starter" - "github.com/apptainer/apptainer/pkg/ociruntime" + "github.com/apptainer/apptainer/pkg/sylog" ) // OciExec executes a command in a container func OciExec(containerID string, cmdArgs []string) error { - commonConfig, err := getCommonConfig(containerID) - if err != nil { - return fmt.Errorf("%s doesn't exist", containerID) + runcArgs := []string{ + "--root=" + OciStateDir, + "exec", + containerID, } + runcArgs = append(runcArgs, cmdArgs...) - engineConfig := commonConfig.EngineConfig.(*oci.EngineConfig) - - switch engineConfig.GetState().Status { - case ociruntime.Running, ociruntime.Paused: - default: - args := strings.Join(cmdArgs, " ") - return fmt.Errorf("cannot execute command %q, container '%s' is not running", args, containerID) + sylog.Debugf("Calling runc with args %v", runcArgs) + if err := syscall.Exec(runc, runcArgs, []string{}); err != nil { + return fmt.Errorf("while calling runc: %w", err) } - engineConfig.Exec = true - engineConfig.OciConfig.SetProcessArgs(cmdArgs) - - os.Clearenv() - - procName := fmt.Sprintf("Apptainer OCI %s", containerID) - return starter.Exec(procName, commonConfig) + return nil } diff --git a/internal/app/apptainer/oci_kill_linux.go b/internal/app/apptainer/oci_kill_linux.go index ae1fbdd743..daafb9b419 100644 --- a/internal/app/apptainer/oci_kill_linux.go +++ b/internal/app/apptainer/oci_kill_linux.go @@ -11,64 +11,23 @@ package apptainer import ( "fmt" - "io" "syscall" - "time" - "github.com/apptainer/apptainer/internal/pkg/util/signal" - "github.com/apptainer/apptainer/pkg/ociruntime" - "github.com/apptainer/apptainer/pkg/util/unix" + "github.com/apptainer/apptainer/pkg/sylog" ) // OciKill kills container process -func OciKill(containerID string, killSignal string, killTimeout int) error { - // send signal to the instance - state, err := getState(containerID) - if err != nil { - return err +func OciKill(containerID string, killSignal string) error { + runcArgs := []string{ + "--root=" + OciStateDir, + "kill", + containerID, + killSignal, } - if state.Status != ociruntime.Created && state.Status != ociruntime.Running { - return fmt.Errorf("cannot kill '%s', the state of the container must be created or running", containerID) - } - - sig := syscall.SIGTERM - - if killSignal != "" { - sig, err = signal.Convert(killSignal) - if err != nil { - return err - } - } - - if killTimeout > 0 { - c, err := unix.Dial(state.ControlSocket) - if err != nil { - return fmt.Errorf("failed to connect to control socket") - } - defer c.Close() - - killed := make(chan bool, 1) - - go func() { - // wait runtime close socket connection for ACK - d := make([]byte, 1) - if _, err := c.Read(d); err == io.EOF { - killed <- true - } - }() - - if err := syscall.Kill(state.Pid, sig); err != nil { - return err - } - - select { - case <-killed: - case <-time.After(time.Duration(killTimeout) * time.Second): - return syscall.Kill(state.Pid, syscall.SIGKILL) - } - } else { - return syscall.Kill(state.Pid, sig) + sylog.Debugf("Calling runc with args %v", runcArgs) + if err := syscall.Exec(runc, runcArgs, []string{}); err != nil { + return fmt.Errorf("while calling runc: %w", err) } return nil diff --git a/internal/app/apptainer/oci_linux.go b/internal/app/apptainer/oci_linux.go index f603ddc97a..3a83ca10a6 100644 --- a/internal/app/apptainer/oci_linux.go +++ b/internal/app/apptainer/oci_linux.go @@ -9,83 +9,20 @@ package apptainer -import ( - "context" - "encoding/json" - "fmt" - "os" - - "github.com/apptainer/apptainer/internal/pkg/instance" - "github.com/apptainer/apptainer/internal/pkg/runtime/engine/oci" - "github.com/apptainer/apptainer/pkg/ociruntime" - "github.com/apptainer/apptainer/pkg/runtime/engine/config" - "github.com/apptainer/apptainer/pkg/sylog" +const ( + OciStateDir = "/run/apptainer-oci" + runc = "/usr/bin/runc" ) // OciArgs contains CLI arguments type OciArgs struct { - BundlePath string - LogPath string - LogFormat string - SyncSocketPath string - PidFile string - FromFile string - KillSignal string - KillTimeout uint32 - EmptyProcess bool - ForceKill bool -} - -func getCommonConfig(containerID string) (*config.Common, error) { - commonConfig := config.Common{ - EngineConfig: &oci.EngineConfig{}, - } - - file, err := instance.Get(containerID, instance.OciSubDir) - if err != nil { - return nil, fmt.Errorf("no container found with name %s", containerID) - } - - if err := json.Unmarshal(file.Config, &commonConfig); err != nil { - return nil, fmt.Errorf("failed to read %s container configuration: %s", containerID, err) - } - - return &commonConfig, nil -} - -func getEngineConfig(containerID string) (*oci.EngineConfig, error) { - commonConfig, err := getCommonConfig(containerID) - if err != nil { - return nil, err - } - return commonConfig.EngineConfig.(*oci.EngineConfig), nil -} - -func getState(containerID string) (*ociruntime.State, error) { - engineConfig, err := getEngineConfig(containerID) - if err != nil { - return nil, err - } - return &engineConfig.State, nil -} - -func exitContainer(ctx context.Context, containerID string, delete bool) { - state, err := getState(containerID) - if err != nil { - if !delete { - sylog.Errorf("%s", err) - os.Exit(1) - } - return - } - - if state.ExitCode != nil { - defer os.Exit(*state.ExitCode) - } - - if delete { - if err := OciDelete(ctx, containerID); err != nil { - sylog.Errorf("%s", err) - } - } + BundlePath string + LogPath string + LogFormat string + PidFile string + FromFile string + KillSignal string + KillTimeout uint32 + EmptyProcess bool + ForceKill bool } diff --git a/internal/app/apptainer/oci_pause_linux.go b/internal/app/apptainer/oci_pause_linux.go index 74a529fd81..3358b2f255 100644 --- a/internal/app/apptainer/oci_pause_linux.go +++ b/internal/app/apptainer/oci_pause_linux.go @@ -10,68 +10,39 @@ package apptainer import ( - "encoding/json" "fmt" - "io" + "syscall" - "github.com/apptainer/apptainer/pkg/ociruntime" - "github.com/apptainer/apptainer/pkg/util/unix" + "github.com/apptainer/apptainer/pkg/sylog" ) -// OciPauseResume pauses/resumes processes in a container -func OciPauseResume(containerID string, pause bool) error { - state, err := getState(containerID) - if err != nil { - return err +// OciPause pauses processes in a container +func OciPause(containerID string) error { + runcArgs := []string{ + "--root=" + OciStateDir, + "pause", + containerID, } - if state.ControlSocket == "" { - return fmt.Errorf("can't find control socket") + sylog.Debugf("Calling runc with args %v", runcArgs) + if err := syscall.Exec(runc, runcArgs, []string{}); err != nil { + return fmt.Errorf("while calling runc: %w", err) } - if pause && state.Status != ociruntime.Running { - return fmt.Errorf("container %s is not running", containerID) - } else if !pause && state.Status != ociruntime.Paused { - return fmt.Errorf("container %s is not paused", containerID) - } - - ctrl := &ociruntime.Control{} - if pause { - ctrl.Pause = true - } else { - ctrl.Resume = true - } - - c, err := unix.Dial(state.ControlSocket) - if err != nil { - return fmt.Errorf("failed to connect to control socket") - } - defer c.Close() - - enc := json.NewEncoder(c) - if enc == nil { - return fmt.Errorf("cannot instantiate new JSON encoder") - } - - if err := enc.Encode(ctrl); err != nil { - return err - } + return nil +} - // wait runtime close socket connection for ACK - d := make([]byte, 1) - if _, err := c.Read(d); err != io.EOF { - return err +// OciResume pauses processes in a container +func OciResume(containerID string) error { + runcArgs := []string{ + "--root=" + OciStateDir, + "resume", + containerID, } - // check status - state, err = getState(containerID) - if err != nil { - return err - } - if pause && state.Status != ociruntime.Paused { - return fmt.Errorf("bad status %s returned instead of paused", state.Status) - } else if !pause && state.Status != ociruntime.Running { - return fmt.Errorf("bad status %s returned instead of running", state.Status) + sylog.Debugf("Calling runc with args %v", runcArgs) + if err := syscall.Exec(runc, runcArgs, []string{}); err != nil { + return fmt.Errorf("while calling runc: %w", err) } return nil diff --git a/internal/app/apptainer/oci_run_linux.go b/internal/app/apptainer/oci_run_linux.go index 1869b756f0..593b61b839 100644 --- a/internal/app/apptainer/oci_run_linux.go +++ b/internal/app/apptainer/oci_run_linux.go @@ -2,7 +2,7 @@ // Apptainer a Series of LF Projects LLC. // For website terms of use, trademark policy, privacy policy and other // project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2020, Sylabs Inc. All rights reserved. +// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -11,105 +11,38 @@ package apptainer import ( "context" - "encoding/json" "fmt" "os" "path/filepath" + "syscall" - "github.com/apptainer/apptainer/internal/pkg/instance" - "github.com/apptainer/apptainer/pkg/ociruntime" "github.com/apptainer/apptainer/pkg/sylog" - "github.com/apptainer/apptainer/pkg/util/unix" - specs "github.com/opencontainers/runtime-spec/specs-go" ) // OciRun runs a container (equivalent to create/start/delete) func OciRun(ctx context.Context, containerID string, args *OciArgs) error { - dir, err := instance.GetDir(containerID, instance.OciSubDir) + absBundle, err := filepath.Abs(args.BundlePath) if err != nil { - return err - } - if err := os.MkdirAll(dir, 0o700); err != nil { - return err - } - args.SyncSocketPath = filepath.Join(dir, "run.sock") - - l, err := unix.CreateSocket(args.SyncSocketPath) - if err != nil { - os.Remove(args.SyncSocketPath) - return err - } - - defer l.Close() - - status := make(chan string, 1) - - if err := OciCreate(containerID, args); err != nil { - defer os.Remove(args.SyncSocketPath) - if _, err1 := getState(containerID); err1 != nil { - return err - } - if err := OciDelete(ctx, containerID); err != nil { - sylog.Warningf("can't delete container %s", containerID) - } - return err + return fmt.Errorf("failed to determine bundle absolute path: %s", err) } - defer exitContainer(ctx, containerID, true) - defer os.Remove(args.SyncSocketPath) - - go func() { - var state specs.State - - for { - c, err := l.Accept() - if err != nil { - status <- err.Error() - return - } - - dec := json.NewDecoder(c) - if err := dec.Decode(&state); err != nil { - status <- err.Error() - return - } - - c.Close() - - switch state.Status { - case ociruntime.Created: - // ignore error there and wait for stopped status - OciStart(containerID) - case ociruntime.Running: - status <- string(state.Status) - case ociruntime.Stopped: - status <- string(state.Status) - } - } - }() - - // wait running status - s := <-status - if s != ociruntime.Running { - return fmt.Errorf("%s", s) + if err := os.Chdir(absBundle); err != nil { + return fmt.Errorf("failed to change directory to %s: %s", absBundle, err) } - engineConfig, err := getEngineConfig(containerID) - if err != nil { - return err + runcArgs := []string{ + "--root=" + OciStateDir, + "create", + "-b", absBundle, } - - if err := attach(engineConfig, true); err != nil { - // kill container before deletion - sylog.Errorf("%s", err) - OciKill(containerID, "SIGKILL", 1) - return err + if args.PidFile != "" { + runcArgs = append(runcArgs, "--pid-file="+args.PidFile) } + runcArgs = append(runcArgs, containerID) - // wait stopped status - s = <-status - if s != ociruntime.Stopped { - return fmt.Errorf("%s", s) + sylog.Debugf("Calling runc with args %v", runcArgs) + if err := syscall.Exec(runc, runcArgs, []string{}); err != nil { + return fmt.Errorf("while calling runc: %w", err) } return nil diff --git a/internal/app/apptainer/oci_start_linux.go b/internal/app/apptainer/oci_start_linux.go index 7e324bd673..1d8b9d62d4 100644 --- a/internal/app/apptainer/oci_start_linux.go +++ b/internal/app/apptainer/oci_start_linux.go @@ -10,51 +10,23 @@ package apptainer import ( - "encoding/json" "fmt" - "io" + "syscall" - "github.com/apptainer/apptainer/pkg/ociruntime" - "github.com/apptainer/apptainer/pkg/util/unix" + "github.com/apptainer/apptainer/pkg/sylog" ) // OciStart starts a previously create container func OciStart(containerID string) error { - state, err := getState(containerID) - if err != nil { - return err + runcArgs := []string{ + "--root=" + OciStateDir, + "start", + containerID, } - if state.Status != ociruntime.Created { - return fmt.Errorf("cannot start '%s', the state of the container must be %s", containerID, ociruntime.Created) - } - - if state.ControlSocket == "" { - return fmt.Errorf("can't find control socket") - } - - ctrl := &ociruntime.Control{} - ctrl.StartContainer = true - - c, err := unix.Dial(state.ControlSocket) - if err != nil { - return fmt.Errorf("failed to connect to control socket") - } - defer c.Close() - - enc := json.NewEncoder(c) - if enc == nil { - return fmt.Errorf("cannot instantiate new JSON encoder") - } - - if err := enc.Encode(ctrl); err != nil { - return err - } - - // wait runtime close socket connection for ACK - d := make([]byte, 1) - if _, err := c.Read(d); err != io.EOF { - return err + sylog.Debugf("Calling runc with args %v", runcArgs) + if err := syscall.Exec(runc, runcArgs, []string{}); err != nil { + return fmt.Errorf("while calling runc: %w", err) } return nil diff --git a/internal/app/apptainer/oci_state_linux.go b/internal/app/apptainer/oci_state_linux.go index 8026e61168..f2483a398d 100644 --- a/internal/app/apptainer/oci_state_linux.go +++ b/internal/app/apptainer/oci_state_linux.go @@ -10,32 +10,24 @@ package apptainer import ( - "encoding/json" "fmt" + "syscall" - "github.com/apptainer/apptainer/pkg/util/unix" + "github.com/apptainer/apptainer/pkg/sylog" ) // OciState query container state func OciState(containerID string, args *OciArgs) error { - // query instance files and returns state - state, err := getState(containerID) - if err != nil { - return err + runcArgs := []string{ + "--root=" + OciStateDir, + "state", + containerID, } - if args.SyncSocketPath != "" { - data, err := json.Marshal(state) - if err != nil { - return fmt.Errorf("failed to marshal state data: %s", err) - } else if err := unix.WriteSocket(args.SyncSocketPath, data); err != nil { - return err - } - } else { - c, err := json.MarshalIndent(state, "", "\t") - if err != nil { - return err - } - fmt.Println(string(c)) + + sylog.Debugf("Calling runc with args %v", runcArgs) + if err := syscall.Exec(runc, runcArgs, []string{}); err != nil { + return fmt.Errorf("while calling runc: %w", err) } + return nil } diff --git a/internal/app/apptainer/oci_update_linux.go b/internal/app/apptainer/oci_update_linux.go index 02949b294e..b175d0bd66 100644 --- a/internal/app/apptainer/oci_update_linux.go +++ b/internal/app/apptainer/oci_update_linux.go @@ -10,57 +10,25 @@ package apptainer import ( - "encoding/json" "fmt" - "io" - "os" + "syscall" - "github.com/apptainer/apptainer/internal/pkg/cgroups" - "github.com/apptainer/apptainer/pkg/ociruntime" - specs "github.com/opencontainers/runtime-spec/specs-go" + "github.com/apptainer/apptainer/pkg/sylog" ) // OciUpdate updates container cgroups resources func OciUpdate(containerID string, args *OciArgs) error { - var reader io.Reader - - state, err := getState(containerID) - if err != nil { - return err - } - - if state.State.Status != ociruntime.Running && state.State.Status != ociruntime.Created { - return fmt.Errorf("container %s is neither running nor created", containerID) - } - - if args.FromFile == "" { - return fmt.Errorf("you must specify --from-file") - } - - resources := &specs.LinuxResources{} - manager, err := cgroups.GetManagerForPid(state.State.Pid) - if err != nil { - return fmt.Errorf("failed to get cgroups manager: %v", err) - } - - if args.FromFile == "-" { - reader = os.Stdin - } else { - f, err := os.Open(args.FromFile) - if err != nil { - return err - } - reader = f - } - - data, err := io.ReadAll(reader) - if err != nil { - return fmt.Errorf("failed to read cgroups config file: %s", err) + runcArgs := []string{ + "--root=" + OciStateDir, + "update", + "-r", args.FromFile, + containerID, } - if err := json.Unmarshal(data, resources); err != nil { - return err + sylog.Debugf("Calling runc with args %v", runcArgs) + if err := syscall.Exec(runc, runcArgs, []string{}); err != nil { + return fmt.Errorf("while calling runc: %w", err) } - return manager.UpdateFromSpec(resources) + return nil } From 00db0c7f1f082cb12cdf6c61cb3aa4f625c6ef06 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Mon, 28 Feb 2022 10:53:43 -0600 Subject: [PATCH 006/114] oci: remove singularity oci_engine Signed-off-by: Edita Kizinevic --- cmd/starter/engines/oci_linux.go | 17 - internal/pkg/instance/instance_linux.go | 2 - .../pkg/runtime/engine/oci/cleanup_linux.go | 90 -- .../pkg/runtime/engine/oci/config_linux.go | 112 -- .../pkg/runtime/engine/oci/create_linux.go | 991 ------------------ .../pkg/runtime/engine/oci/engine_linux.go | 58 - .../pkg/runtime/engine/oci/monitor_linux.go | 54 - .../pkg/runtime/engine/oci/prepare_linux.go | 272 ----- .../pkg/runtime/engine/oci/process_linux.go | 497 --------- internal/pkg/runtime/engine/oci/rpc/args.go | 15 - .../runtime/engine/oci/rpc/client/client.go | 44 - .../engine/oci/rpc/server/server_linux.go | 42 - mlocal/frags/go_common_opts.mk | 2 +- 13 files changed, 1 insertion(+), 2195 deletions(-) delete mode 100644 cmd/starter/engines/oci_linux.go delete mode 100644 internal/pkg/runtime/engine/oci/cleanup_linux.go delete mode 100644 internal/pkg/runtime/engine/oci/config_linux.go delete mode 100644 internal/pkg/runtime/engine/oci/create_linux.go delete mode 100644 internal/pkg/runtime/engine/oci/engine_linux.go delete mode 100644 internal/pkg/runtime/engine/oci/monitor_linux.go delete mode 100644 internal/pkg/runtime/engine/oci/prepare_linux.go delete mode 100644 internal/pkg/runtime/engine/oci/process_linux.go delete mode 100644 internal/pkg/runtime/engine/oci/rpc/args.go delete mode 100644 internal/pkg/runtime/engine/oci/rpc/client/client.go delete mode 100644 internal/pkg/runtime/engine/oci/rpc/server/server_linux.go diff --git a/cmd/starter/engines/oci_linux.go b/cmd/starter/engines/oci_linux.go deleted file mode 100644 index d2df39e32d..0000000000 --- a/cmd/starter/engines/oci_linux.go +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2019-2021, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -//go:build oci_engine - -package engines - -import ( - // register the oci runtime engine - _ "github.com/apptainer/apptainer/internal/pkg/runtime/engine/oci" -) diff --git a/internal/pkg/instance/instance_linux.go b/internal/pkg/instance/instance_linux.go index c44fce0196..59c560b78b 100644 --- a/internal/pkg/instance/instance_linux.go +++ b/internal/pkg/instance/instance_linux.go @@ -24,8 +24,6 @@ import ( ) const ( - // OciSubDir represents directory where OCI instance files are stored - OciSubDir = "oci" // AppSubDir represents directory where Apptainer instance files are stored AppSubDir = "app" // LogSubDir represents directory where Apptainer instance log files are stored diff --git a/internal/pkg/runtime/engine/oci/cleanup_linux.go b/internal/pkg/runtime/engine/oci/cleanup_linux.go deleted file mode 100644 index f15b5a9794..0000000000 --- a/internal/pkg/runtime/engine/oci/cleanup_linux.go +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2021, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package oci - -import ( - "context" - "fmt" - "os" - "syscall" - - "github.com/apptainer/apptainer/internal/pkg/instance" - "github.com/apptainer/apptainer/pkg/ociruntime" - "github.com/apptainer/apptainer/pkg/sylog" -) - -// CleanupContainer is called from master after the MonitorContainer returns. -// It is responsible for ensuring that the container has been properly torn down. -// -// Additional privileges may be gained when running -// in suid flow. However, when a user namespace is requested and it is not -// a hybrid workflow (e.g. fakeroot), then there is no privileged saved uid -// and thus no additional privileges can be gained. -// -// Specifically in oci engine, no additional privileges are gained here. However, -// most likely this still will be executed as root since `apptainer oci` -// command set requires privileged execution. -func (e *EngineOperations) CleanupContainer(ctx context.Context, fatal error, status syscall.WaitStatus) error { - if e.EngineConfig.Cgroups != nil { - if err := e.EngineConfig.Cgroups.Destroy(); err != nil { - sylog.Warningf("failed to remove cgroup configuration: %v", err) - } - } - - pidFile := e.EngineConfig.GetPidFile() - if pidFile != "" { - os.Remove(pidFile) - } - - // if container wasn't created, delete instance files - if e.EngineConfig.State.Status == ociruntime.Creating { - name := e.CommonConfig.ContainerID - file, err := instance.Get(name, instance.OciSubDir) - if err != nil { - sylog.Warningf("no instance files found for %s: %s", name, err) - return nil - } - if err := file.Delete(); err != nil { - sylog.Warningf("failed to delete instance files: %s", err) - } - return nil - } - - exitCode := 0 - desc := "" - - if fatal != nil { - exitCode = 255 - desc = fatal.Error() - } else if status.Signaled() { - s := status.Signal() - exitCode = int(s) + 128 - desc = fmt.Sprintf("interrupted by signal %s", s.String()) - } else { - exitCode = status.ExitStatus() - desc = fmt.Sprintf("exited with code %d", status.ExitStatus()) - } - - e.EngineConfig.State.ExitCode = &exitCode - e.EngineConfig.State.ExitDesc = desc - - if err := e.updateState(ociruntime.Stopped); err != nil { - return err - } - - if e.EngineConfig.State.AttachSocket != "" { - os.Remove(e.EngineConfig.State.AttachSocket) - } - if e.EngineConfig.State.ControlSocket != "" { - os.Remove(e.EngineConfig.State.ControlSocket) - } - - return nil -} diff --git a/internal/pkg/runtime/engine/oci/config_linux.go b/internal/pkg/runtime/engine/oci/config_linux.go deleted file mode 100644 index c88660690c..0000000000 --- a/internal/pkg/runtime/engine/oci/config_linux.go +++ /dev/null @@ -1,112 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2021, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package oci - -import ( - "sync" - - "github.com/apptainer/apptainer/internal/pkg/cgroups" - "github.com/apptainer/apptainer/internal/pkg/runtime/engine/config/oci" - "github.com/apptainer/apptainer/pkg/ociruntime" -) - -// Name of the engine. -const Name = "oci" - -// EngineConfig is the config for the OCI engine. -type EngineConfig struct { - BundlePath string `json:"bundlePath"` - LogPath string `json:"logPath"` - LogFormat string `json:"logFormat"` - PidFile string `json:"pidFile"` - OciConfig *oci.Config `json:"ociConfig"` - MasterPts int `json:"masterPts"` - SlavePts int `json:"slavePts"` - OutputStreams [2]int `json:"outputStreams"` - ErrorStreams [2]int `json:"errorStreams"` - InputStreams [2]int `json:"inputStreams"` - SyncSocket string `json:"syncSocket"` - EmptyProcess bool `json:"emptyProcess"` - Exec bool `json:"exec"` - SystemdCgroups bool `json:"systemdCgroups"` - Cgroups *cgroups.Manager `json:"-"` - - sync.Mutex `json:"-"` - State ociruntime.State `json:"state"` -} - -// NewConfig returns an oci.EngineConfig. -func NewConfig() *EngineConfig { - ret := &EngineConfig{ - OciConfig: &oci.Config{}, - } - - return ret -} - -// SetBundlePath sets the container bundle path. -func (e *EngineConfig) SetBundlePath(path string) { - e.BundlePath = path -} - -// GetBundlePath returns the container bundle path. -func (e *EngineConfig) GetBundlePath() string { - return e.BundlePath -} - -// SetState sets the container state as defined by OCI state specification. -func (e *EngineConfig) SetState(state *ociruntime.State) { - e.State = *state -} - -// GetState returns the container state as defined by OCI state specification. -func (e *EngineConfig) GetState() *ociruntime.State { - return &e.State -} - -// SetLogPath sets the container log path. -func (e *EngineConfig) SetLogPath(path string) { - e.LogPath = path -} - -// GetLogPath returns the container log path. -func (e *EngineConfig) GetLogPath() string { - return e.LogPath -} - -// SetLogFormat sets the container log format. -func (e *EngineConfig) SetLogFormat(format string) { - e.LogFormat = format -} - -// GetLogFormat returns the container log format. -func (e *EngineConfig) GetLogFormat() string { - return e.LogFormat -} - -// SetPidFile sets the pid file path. -func (e *EngineConfig) SetPidFile(path string) { - e.PidFile = path -} - -// GetPidFile gets the pid file path. -func (e *EngineConfig) GetPidFile() string { - return e.PidFile -} - -// SetSystemdCgroups sets whether to manage cgroups with systemd. -func (e *EngineConfig) SetSystemdCgroups(systemd bool) { - e.SystemdCgroups = systemd -} - -// SetSystemdCgroups gets whether to manage cgroups with systemd. -func (e *EngineConfig) GetSystemdCgroups() bool { - return e.SystemdCgroups -} diff --git a/internal/pkg/runtime/engine/oci/create_linux.go b/internal/pkg/runtime/engine/oci/create_linux.go deleted file mode 100644 index 97d3325a27..0000000000 --- a/internal/pkg/runtime/engine/oci/create_linux.go +++ /dev/null @@ -1,991 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package oci - -import ( - "bufio" - "context" - "encoding/json" - "fmt" - "net" - "net/rpc" - "os" - "path/filepath" - "strings" - "syscall" - "time" - - "github.com/apptainer/apptainer/internal/pkg/cgroups" - "github.com/apptainer/apptainer/internal/pkg/instance" - "github.com/apptainer/apptainer/internal/pkg/runtime/engine/oci/rpc/client" - "github.com/apptainer/apptainer/internal/pkg/util/fs" - "github.com/apptainer/apptainer/internal/pkg/util/fs/mount" - "github.com/apptainer/apptainer/pkg/ociruntime" - "github.com/apptainer/apptainer/pkg/sylog" - "github.com/apptainer/apptainer/pkg/util/fs/proc" - "github.com/apptainer/apptainer/pkg/util/namespaces" - "github.com/apptainer/apptainer/pkg/util/sysctl" - "github.com/apptainer/apptainer/pkg/util/unix" - specs "github.com/opencontainers/runtime-spec/specs-go" -) - -var symlinkDevices = []struct { - old string - new string -}{ - {"/proc/self/fd", "/dev/fd"}, - {"/proc/kcore", "/dev/core"}, - {"pts/ptmx", "/dev/ptmx"}, - {"/proc/self/fd/0", "/dev/stdin"}, - {"/proc/self/fd/1", "/dev/stdout"}, - {"/proc/self/fd/2", "/dev/stderr"}, -} - -type device struct { - major int64 - minor int64 - path string - mode os.FileMode - uid int - gid int -} - -var devices = []device{ - {1, 7, "/dev/full", syscall.S_IFCHR | 0o666, 0, 0}, - {1, 3, "/dev/null", syscall.S_IFCHR | 0o666, 0, 0}, - {1, 8, "/dev/random", syscall.S_IFCHR | 0o666, 0, 0}, - {5, 0, "/dev/tty", syscall.S_IFCHR | 0o666, 0, 0}, - {1, 9, "/dev/urandom", syscall.S_IFCHR | 0o666, 0, 0}, - {1, 5, "/dev/zero", syscall.S_IFCHR | 0o666, 0, 0}, -} - -var cgroupDevices = []specs.LinuxDeviceCgroup{ - { - Allow: true, - Type: "c", - Major: cgroups.Int64ptr(1), - Minor: cgroups.Int64ptr(7), - Access: "rw", - }, - { - Allow: true, - Type: "c", - Major: cgroups.Int64ptr(1), - Minor: cgroups.Int64ptr(3), - Access: "rw", - }, - { - Allow: true, - Type: "c", - Major: cgroups.Int64ptr(1), - Minor: cgroups.Int64ptr(8), - Access: "rw", - }, - { - Allow: true, - Type: "c", - Major: cgroups.Int64ptr(5), - Minor: cgroups.Int64ptr(0), - Access: "rw", - }, - { - Allow: true, - Type: "c", - Major: cgroups.Int64ptr(1), - Minor: cgroups.Int64ptr(9), - Access: "rw", - }, - { - Allow: true, - Type: "c", - Major: cgroups.Int64ptr(1), - Minor: cgroups.Int64ptr(5), - Access: "rw", - }, - { - Allow: true, - Type: "c", - Major: cgroups.Int64ptr(136), - Minor: cgroups.Int64ptr(-1), - Access: "rwm", - }, - { - Allow: true, - Type: "c", - Major: cgroups.Int64ptr(5), - Minor: cgroups.Int64ptr(1), - Access: "rw", - }, - { - Allow: true, - Type: "c", - Major: cgroups.Int64ptr(5), - Minor: cgroups.Int64ptr(2), - Access: "rw", - }, -} - -type container struct { - engine *EngineOperations - rpcOps *client.RPC - rootfs string - rpcRoot string - userNS bool - utsNS bool - mntNS bool - devIndex int - cgroupV1MountIndex int -} - -var statusChan = make(chan string, 1) - -// CreateContainer is called from master process to prepare container -// environment, e.g. perform mount operations, etc. -// -// Additional privileges required for setup may be gained when running -// in suid flow. However, when a user namespace is requested and it is not -// a hybrid workflow (e.g. fakeroot), then there is no privileged saved uid -// and thus no additional privileges can be gained. -// -// Specifically in oci engine, no additional privileges are gained. Container -// setup (e.g. mount operations) where privileges may be required is performed -// by calling RPC server methods (see internal/app/starter/rpc_linux.go for details). -// -// However, most likely this still will be executed as root since `apptainer oci` -// command set requires privileged execution. -// -//nolint:maintidx -func (e *EngineOperations) CreateContainer(ctx context.Context, pid int, rpcConn net.Conn) error { - var err error - - if e.CommonConfig.EngineName != Name { - return fmt.Errorf("engineName configuration doesn't match runtime name") - } - - rpcOps := &client.RPC{} - rpcOps.Client = rpc.NewClient(rpcConn) - rpcOps.Name = e.CommonConfig.EngineName - - if rpcOps.Client == nil { - return fmt.Errorf("failed to initialize RPC client") - } - - if err := e.createState(pid); err != nil { - return err - } - - rootfs := e.EngineConfig.OciConfig.Root.Path - - if !filepath.IsAbs(rootfs) { - rootfs = filepath.Join(e.EngineConfig.GetBundlePath(), rootfs) - } - - resolvedRootfs, err := filepath.EvalSymlinks(rootfs) - if err != nil { - return fmt.Errorf("failed to resolve %s path: %s", rootfs, err) - } - - c := &container{ - engine: e, - rpcOps: rpcOps, - rootfs: resolvedRootfs, - rpcRoot: fmt.Sprintf("/proc/%d/root", pid), - cgroupV1MountIndex: -1, - devIndex: -1, - } - - for _, ns := range e.EngineConfig.OciConfig.Linux.Namespaces { - switch ns.Type { - case specs.UserNamespace: - c.userNS = true - case specs.UTSNamespace: - c.utsNS = true - case specs.MountNamespace: - c.mntNS = true - } - } - - p := &mount.Points{} - if e.EngineConfig.OciConfig.Linux.MountLabel != "" { - if err := p.SetContext(e.EngineConfig.OciConfig.Linux.MountLabel); err != nil { - return err - } - } - - system := &mount.System{Points: p, Mount: c.mount} - - for i, point := range e.EngineConfig.OciConfig.Config.Mounts { - // A cgroup v1 mount point will be intercepted and handled separately in c.addCgroups(...) - if point.Type == "cgroup" { - c.cgroupV1MountIndex = i - continue - } - // dev creation - if point.Destination == "/dev" && point.Type == "tmpfs" { - c.devIndex = i - } - } - - if err := c.addDevices(system); err != nil { - return err - } - - if err := c.addCgroups(pid, system); err != nil { - return err - } - - // import OCI mount spec - if err := system.Points.ImportFromSpec(e.EngineConfig.OciConfig.Config.Mounts); err != nil { - return err - } - - if err := c.addRootfsMount(system); err != nil { - return err - } - - if err := system.RunAfterTag(mount.KernelTag, c.addDefaultDevices); err != nil { - return err - } - - if err := system.RunAfterTag(mount.KernelTag, c.addAllPaths); err != nil { - return err - } - - if err := proc.SetOOMScoreAdj(pid, e.EngineConfig.OciConfig.Process.OOMScoreAdj); err != nil { - return err - } - - if err := namespaces.Enter(pid, "ipc"); err != nil { - return err - } - if err := namespaces.Enter(pid, "net"); err != nil { - return err - } - - for key, value := range e.EngineConfig.OciConfig.Linux.Sysctl { - if err := sysctl.Set(key, value); err != nil { - return err - } - } - - if err := namespaces.Enter(os.Getpid(), "ipc"); err != nil { - return err - } - if err := namespaces.Enter(os.Getpid(), "net"); err != nil { - return err - } - - sylog.Debugf("Mount all") - if err := system.MountAll(); err != nil { - return err - } - - if c.utsNS && e.EngineConfig.OciConfig.Hostname != "" { - if _, err := rpcOps.SetHostname(e.EngineConfig.OciConfig.Hostname); err != nil { - return err - } - } - - // update namespaces configuration path - namespaces := []struct { - nstype string - ns specs.LinuxNamespaceType - checkEnabled bool - }{ - {"pid", specs.PIDNamespace, false}, - {"uts", specs.UTSNamespace, false}, - {"ipc", specs.IPCNamespace, false}, - {"mnt", specs.MountNamespace, false}, - {"cgroup", specs.CgroupNamespace, false}, - {"net", specs.NetworkNamespace, false}, - {"user", specs.UserNamespace, true}, - } - - path := fmt.Sprintf("/proc/%d/ns", pid) - - for _, n := range namespaces { - has, err := proc.HasNamespace(pid, n.nstype) - if err == nil && (has || n.checkEnabled) { - enabled := false - if n.checkEnabled { - if e.EngineConfig.OciConfig.Linux != nil { - for _, namespace := range e.EngineConfig.OciConfig.Linux.Namespaces { - if n.ns == namespace.Type { - enabled = true - break - } - } - } - } - if has || enabled { - nspath := filepath.Join(path, n.nstype) - e.EngineConfig.OciConfig.AddOrReplaceLinuxNamespace(n.ns, nspath) - } - } else if err != nil { - return fmt.Errorf("failed to check %s root and container namespace: %s", n.ns, err) - } - } - - method := "pivot" - if !c.mntNS { - method = "chroot" - } - - _, err = rpcOps.Chroot(c.rootfs, method) - if err != nil { - return fmt.Errorf("chroot failed: %s", err) - } - - if e.EngineConfig.SlavePts != -1 { - if err := syscall.Close(e.EngineConfig.SlavePts); err != nil { - return fmt.Errorf("failed to close slave part: %s", err) - } - } - if e.EngineConfig.OutputStreams[0] != -1 { - if err := syscall.Close(e.EngineConfig.OutputStreams[1]); err != nil { - return fmt.Errorf("failed to close write output stream: %s", err) - } - } - if e.EngineConfig.ErrorStreams[0] != -1 { - if err := syscall.Close(e.EngineConfig.ErrorStreams[1]); err != nil { - return fmt.Errorf("failed to close write error stream: %s", err) - } - } - if e.EngineConfig.InputStreams[0] != -1 { - if err := syscall.Close(e.EngineConfig.InputStreams[1]); err != nil { - return fmt.Errorf("failed to close write input stream: %s", err) - } - } - - return nil -} - -func (e *EngineOperations) createState(pid int) error { - e.EngineConfig.Lock() - defer e.EngineConfig.Unlock() - - name := e.CommonConfig.ContainerID - - file, err := instance.Add(name, instance.OciSubDir) - if err != nil { - return err - } - - e.EngineConfig.State.Version = specs.Version - e.EngineConfig.State.Bundle = e.EngineConfig.GetBundlePath() - e.EngineConfig.State.ID = e.CommonConfig.ContainerID - e.EngineConfig.State.Pid = pid - e.EngineConfig.State.Status = ociruntime.Creating - e.EngineConfig.State.Annotations = e.EngineConfig.OciConfig.Annotations - - file.Config, err = json.Marshal(e.CommonConfig) - if err != nil { - return err - } - - file.User = "root" - file.Pid = pid - file.PPid = os.Getpid() - file.Image = filepath.Join(e.EngineConfig.GetBundlePath(), e.EngineConfig.OciConfig.Root.Path) - - if err := file.Update(); err != nil { - return err - } - - socketPath := e.EngineConfig.SyncSocket - - if socketPath != "" { - data, err := json.Marshal(e.EngineConfig.State) - if err != nil { - sylog.Warningf("failed to marshal state data: %s", err) - } else if err := unix.WriteSocket(socketPath, data); err != nil { - sylog.Warningf("%s", err) - } - } - - return nil -} - -func (e *EngineOperations) updateState(status string) error { - e.EngineConfig.Lock() - defer e.EngineConfig.Unlock() - - file, err := instance.Get(e.CommonConfig.ContainerID, instance.OciSubDir) - if err != nil { - return err - } - // do nothing if already stopped - if e.EngineConfig.State.Status == ociruntime.Stopped { - return nil - } - oldStatus := e.EngineConfig.State.Status - e.EngineConfig.State.Status = specs.ContainerState(status) - - t := time.Now().UnixNano() - - switch status { - case ociruntime.Created: - if e.EngineConfig.State.CreatedAt == nil { - e.EngineConfig.State.CreatedAt = &t - } - case ociruntime.Running: - if e.EngineConfig.State.StartedAt == nil { - e.EngineConfig.State.StartedAt = &t - } - case ociruntime.Stopped: - if e.EngineConfig.State.FinishedAt == nil { - e.EngineConfig.State.FinishedAt = &t - } - } - - file.Config, err = json.Marshal(e.CommonConfig) - if err != nil { - return err - } - - if err := file.Update(); err != nil { - return err - } - - socketPath := e.EngineConfig.SyncSocket - - if socketPath != "" { - data, err := json.Marshal(e.EngineConfig.State) - if err != nil { - sylog.Warningf("failed to marshal state data: %s", err) - } else if err := unix.WriteSocket(socketPath, data); err != nil { - sylog.Warningf("%s", err) - } - } - - // send running or stopped status right after container creation - // to notify that container process started - if statusChan != nil && oldStatus == ociruntime.Created && - (status == ociruntime.Running || status == ociruntime.Stopped) { - statusChan <- status - } - return nil -} - -// one shot function to wait on running or stopped status -func (e *EngineOperations) waitStatusUpdate() { - if statusChan == nil { - return - } - // block until status update is sent - <-statusChan - // close channel and set it to nil - close(statusChan) - statusChan = nil -} - -func (c *container) addCgroups(pid int, system *mount.System) error { - name := c.engine.CommonConfig.ContainerID - resources := c.engine.EngineConfig.OciConfig.Linux.Resources - systemd := c.engine.EngineConfig.GetSystemdCgroups() - cgroupsPath := c.engine.EngineConfig.OciConfig.Linux.CgroupsPath - - if !systemd && !filepath.IsAbs(cgroupsPath) { - if cgroupsPath == "" { - cgroupsPath = filepath.Join("/apptainer-oci", name) - } else { - cgroupsPath = filepath.Join("/", cgroupsPath) - } - } - - if systemd && cgroupsPath == "" { - cgroupsPath = "system.slice:apptainer-oci:" + name - } - - c.engine.EngineConfig.OciConfig.Linux.CgroupsPath = cgroupsPath - - manager, err := cgroups.NewManagerWithSpec(resources, pid, cgroupsPath, systemd) - if err != nil { - return fmt.Errorf("failed to apply cgroups resources restriction: %s", err) - } - - // If a mount point exists for a cgroup v1 hierarchy we will handle it here. - // This is not necessary for cgroups v2 - as the unified hierarchy will be handled with a simple bind. - if c.cgroupV1MountIndex >= 0 { - m := c.engine.EngineConfig.OciConfig.Config.Mounts[c.cgroupV1MountIndex] - c.engine.EngineConfig.OciConfig.Config.Mounts = append( - c.engine.EngineConfig.OciConfig.Config.Mounts[:c.cgroupV1MountIndex], - c.engine.EngineConfig.OciConfig.Config.Mounts[c.cgroupV1MountIndex+1:]..., - ) - - cgroupRootPath, err := manager.GetCgroupRootPath() - if err != nil { - return fmt.Errorf("failed to determine cgroup root path: %w", err) - } - - flags, opt := mount.ConvertOptions(m.Options) - options := strings.Join(opt, ",") - - readOnly := false - if flags&syscall.MS_RDONLY != 0 { - readOnly = true - flags &^= uintptr(syscall.MS_RDONLY) - } - - hasMode := false - for _, o := range opt { - if strings.HasPrefix(o, "mode=") { - hasMode = true - break - } - } - if !hasMode { - options += ",mode=755" - } - - if err := system.Points.AddFS(mount.OtherTag, m.Destination, "tmpfs", flags, options); err != nil { - return err - } - - createSymlinks := func(*mount.System) error { - cgroupPath := filepath.Join(c.rpcRoot, c.rootfs, m.Destination) - if _, err := os.Stat(filepath.Join(cgroupPath, "cpu")); err != nil && os.IsNotExist(err) { - if err := c.rpcOps.Symlink("cpu,cpuacct", filepath.Join(c.rootfs, m.Destination, "cpu")); err != nil { - return err - } - if err := c.rpcOps.Symlink("cpu,cpuacct", filepath.Join(c.rootfs, m.Destination, "cpuacct")); err != nil { - return err - } - } - - if _, err := os.Stat(filepath.Join(cgroupPath, "net_cls")); err != nil && os.IsNotExist(err) { - if err := c.rpcOps.Symlink("net_cls,net_prio", filepath.Join(c.rootfs, m.Destination, "net_cls")); err != nil { - return err - } - if err := c.rpcOps.Symlink("net_cls,net_prio", filepath.Join(c.rootfs, m.Destination, "net_prio")); err != nil { - return err - } - } - return nil - } - - if err := system.RunAfterTag(mount.OtherTag, createSymlinks); err != nil { - return err - } - - f, err := os.Open(fmt.Sprintf("/proc/%d/cgroup", pid)) - if err != nil { - return err - } - defer f.Close() - - flags |= uintptr(syscall.MS_BIND) - if readOnly { - flags |= syscall.MS_RDONLY - } - - scanner := bufio.NewScanner(f) - for scanner.Scan() { - cgroupLine := strings.Split(scanner.Text(), ":") - if strings.HasPrefix(cgroupLine[1], "name=") { - cgroupLine[1] = strings.Replace(cgroupLine[1], "name=", "", 1) - } - if cgroupLine[1] != "" { - source := filepath.Join(cgroupRootPath, cgroupLine[1], cgroupLine[2]) - dest := filepath.Join(m.Destination, cgroupLine[1]) - if err := system.Points.AddBind(mount.OtherTag, source, dest, flags); err != nil { - return err - } - if readOnly { - if err := system.Points.AddRemount(mount.OtherTag, dest, flags); err != nil { - return err - } - } - } - } - - if readOnly { - if err := system.Points.AddRemount(mount.FinalTag, m.Destination, flags); err != nil { - return err - } - } - } - - c.engine.EngineConfig.Cgroups = manager - - return nil -} - -func (c *container) addAllPaths(system *mount.System) error { - // add masked path - if err := c.addMaskedPathsMount(system); err != nil { - return err - } - - // add read-only path - if !c.userNS { - if err := c.addReadonlyPathsMount(system); err != nil { - return err - } - } - - return nil -} - -func (c *container) addRootfsMount(system *mount.System) error { - flags := uintptr(syscall.MS_BIND) - - if c.engine.EngineConfig.OciConfig.Root.Readonly { - sylog.Debugf("Mounted read-only") - flags |= syscall.MS_RDONLY - } - - parentRootfs, err := proc.ParentMount(c.rootfs) - if err != nil { - return err - } - - sylog.Debugf("Parent rootfs: %s", parentRootfs) - - if err := c.rpcOps.Mount("", parentRootfs, "", syscall.MS_PRIVATE, ""); err != nil { - return err - } - if err := system.Points.AddBind(mount.RootfsTag, c.rootfs, c.rootfs, flags); err != nil { - return err - } - if flags&syscall.MS_RDONLY != 0 { - return system.Points.AddRemount(mount.FinalTag, c.rootfs, flags) - } - - return nil -} - -func (c *container) addDefaultDevices(system *mount.System) error { - oldmask := syscall.Umask(0) - defer syscall.Umask(oldmask) - - rootfsPath := filepath.Join(c.rpcRoot, c.rootfs) - - devPath := filepath.Join(rootfsPath, fs.EvalRelative("/dev", rootfsPath)) - if _, err := os.Lstat(devPath); os.IsNotExist(err) { - if err := os.Mkdir(devPath, 0o755); err != nil { - return err - } - } - - for _, symlink := range symlinkDevices { - path := filepath.Join(rootfsPath, symlink.new) - if _, err := os.Lstat(path); os.IsNotExist(err) { - if c.userNS { - path = filepath.Join(c.rootfs, symlink.new) - if err := c.rpcOps.Symlink(symlink.old, path); err != nil { - return err - } - } else { - if err := os.Symlink(symlink.old, path); err != nil { - return err - } - } - } - } - - if c.engine.EngineConfig.OciConfig.Process.Terminal { - path := filepath.Join(rootfsPath, "dev", "console") - if _, err := os.Lstat(path); os.IsNotExist(err) { - if c.userNS { - if _, err := c.rpcOps.Touch(filepath.Join(c.rootfs, "dev", "console")); err != nil { - return err - } - } else { - if err := fs.Touch(path); err != nil { - return err - } - } - path = fmt.Sprintf("/proc/self/fd/%d", c.engine.EngineConfig.SlavePts) - console, err := os.Readlink(path) - if err != nil { - return err - } - if err := system.Points.AddBind(mount.OtherTag, console, "/dev/console", syscall.MS_BIND); err != nil { - return err - } - } - } - - for _, device := range devices { - dev := int((device.major << 8) | (device.minor & 0xff) | ((device.minor & 0xfff00) << 12)) - path := filepath.Join(rootfsPath, device.path) - if _, err := os.Lstat(path); os.IsNotExist(err) { - if c.userNS { - path = filepath.Join(c.rootfs, device.path) - if _, err := os.Stat(device.path); os.IsNotExist(err) { - sylog.Debugf("skipping mount, %s doesn't exists", device.path) - continue - } - dirpath := filepath.Dir(path) - if _, err := c.rpcOps.MkdirAll(dirpath, 0o755); err != nil { - return fmt.Errorf("could not create parent directory %s: %s", dirpath, err) - } - if _, err := c.rpcOps.Touch(path); err != nil { - return fmt.Errorf("could not create file %s: %s", path, err) - } - if err := c.rpcOps.Mount(device.path, path, "", syscall.MS_BIND, ""); err != nil { - return fmt.Errorf("could not mount %s to %s: %s", device.path, path, err) - } - } else { - dirpath := filepath.Dir(path) - if err := os.MkdirAll(dirpath, 0o755); err != nil { - return fmt.Errorf("could not create parent directory %s: %s", dirpath, err) - } - if err := syscall.Mknod(path, uint32(device.mode), dev); err != nil { - return fmt.Errorf("could not create device %s: %s", path, err) - } - if device.uid != 0 || device.gid != 0 { - if err := os.Chown(path, device.uid, device.gid); err != nil { - return fmt.Errorf("could not change %s owner: %s", path, err) - } - } - } - } - } - - return nil -} - -func (c *container) addDevices(system *mount.System) error { - for _, d := range c.engine.EngineConfig.OciConfig.Linux.Devices { - var dev device - - if d.Path == "" { - return fmt.Errorf("device path required") - } - dev.path = d.Path - - if d.FileMode != nil { - dev.mode = *d.FileMode - } else { - dev.mode = 0o644 - } - - switch d.Type { - case "c", "u": - dev.mode |= syscall.S_IFCHR - dev.major = d.Major - dev.minor = d.Minor - case "b": - dev.mode |= syscall.S_IFBLK - dev.major = d.Major - dev.minor = d.Minor - case "p": - dev.mode |= syscall.S_IFIFO - default: - return fmt.Errorf("device type unknown for %s", d.Path) - } - - if d.UID != nil { - dev.uid = int(*d.UID) - } - if d.GID != nil { - dev.gid = int(*d.GID) - } - - devices = append(devices, dev) - } - - if c.devIndex >= 0 { - m := &c.engine.EngineConfig.OciConfig.Config.Mounts[c.devIndex] - - flags, _ := mount.ConvertOptions(m.Options) - - flags |= uintptr(syscall.MS_BIND) - if flags&syscall.MS_RDONLY != 0 { - if err := system.Points.AddRemount(mount.FinalTag, m.Destination, flags); err != nil { - return err - } - for i := len(m.Options) - 1; i >= 0; i-- { - if m.Options[i] == "ro" { - m.Options = append(m.Options[:i], m.Options[i+1:]...) - break - } - } - } - - if c.engine.EngineConfig.OciConfig.Linux.Resources == nil { - c.engine.EngineConfig.OciConfig.Linux.Resources = &specs.LinuxResources{} - } - - // cgroupDevices are essential for operation, so must be allowed *prior* to a configured wildcard deny. - c.engine.EngineConfig.OciConfig.Linux.Resources.Devices = append(cgroupDevices, c.engine.EngineConfig.OciConfig.Linux.Resources.Devices...) - } - - return nil -} - -func (c *container) addMaskedPathsMount(system *mount.System) error { - paths := c.engine.EngineConfig.OciConfig.Linux.MaskedPaths - - dir, err := instance.GetDir(c.engine.CommonConfig.ContainerID, instance.OciSubDir) - if err != nil { - return err - } - nullPath := filepath.Join(dir, "null") - - if _, err := os.Stat(nullPath); os.IsNotExist(err) { - oldmask := syscall.Umask(0) - defer syscall.Umask(oldmask) - - if err := os.Mkdir(nullPath, 0o755); err != nil { - return err - } - } - - for _, path := range paths { - relativePath := filepath.Join(c.rootfs, path) - rpcPath := filepath.Join(c.rpcRoot, relativePath) - fi, err := os.Stat(rpcPath) - if err != nil { - sylog.Debugf("ignoring masked path %s: %s", path, err) - continue - } - if fi.IsDir() { - if err := system.Points.AddBind(mount.OtherTag, nullPath, relativePath, syscall.MS_BIND); err != nil { - return err - } - } else if err := system.Points.AddBind(mount.OtherTag, "/dev/null", relativePath, syscall.MS_BIND); err != nil { - return err - } - } - return nil -} - -func (c *container) addReadonlyPathsMount(system *mount.System) error { - paths := c.engine.EngineConfig.OciConfig.Linux.ReadonlyPaths - - for _, path := range paths { - relativePath := filepath.Join(c.rootfs, path) - rpcPath := filepath.Join(c.rpcRoot, relativePath) - _, err := os.Stat(rpcPath) - if err != nil { - sylog.Debugf("ignoring read-only path %s: %s", path, err) - continue - } - if err := system.Points.AddBind(mount.OtherTag, relativePath, relativePath, syscall.MS_BIND|syscall.MS_RDONLY); err != nil { - return err - } - if err := system.Points.AddRemount(mount.OtherTag, relativePath, syscall.MS_BIND|syscall.MS_RDONLY); err != nil { - return err - } - } - return nil -} - -func (c *container) mount(point *mount.Point, system *mount.System) error { - source := point.Source - dest := point.Destination - flags, opts := mount.ConvertOptions(point.Options) - optsString := strings.Join(opts, ",") - ignore := false - - if flags&syscall.MS_REMOUNT != 0 { - ignore = true - } - - if !strings.HasPrefix(dest, c.rootfs) { - rootfsPath := filepath.Join(c.rpcRoot, c.rootfs) - relativeDest := fs.EvalRelative(dest, c.rootfs) - procDest := filepath.Join(rootfsPath, relativeDest) - - dest = filepath.Join(c.rootfs, relativeDest) - - sylog.Debugf("Checking if %s exists", procDest) - if _, err := os.Stat(procDest); os.IsNotExist(err) && !ignore { - oldmask := syscall.Umask(0) - defer syscall.Umask(oldmask) - - if point.Type != "" { - sylog.Debugf("Creating %s", procDest) - if c.userNS { - if _, err := c.rpcOps.MkdirAll(dest, 0o755); err != nil { - return err - } - } else { - if err := os.MkdirAll(procDest, 0o755); err != nil { - return err - } - } - } else { - var st syscall.Stat_t - - dir := filepath.Dir(procDest) - if _, err := os.Stat(dir); os.IsNotExist(err) { - sylog.Debugf("Creating parent %s", dir) - if c.userNS { - if err := c.rpcOps.Mkdir(filepath.Dir(dest), 0o755); err != nil { - return err - } - } else { - if err := os.MkdirAll(dir, 0o755); err != nil { - return err - } - } - } - - if err := syscall.Stat(source, &st); err != nil { - sylog.Debugf("Ignoring %s: %s", source, err) - return nil - } - switch st.Mode & syscall.S_IFMT { - case syscall.S_IFDIR: - sylog.Debugf("Creating dir %s", filepath.Base(procDest)) - if c.userNS { - if err := c.rpcOps.Mkdir(dest, 0o755); err != nil { - return err - } - } else { - if err := os.Mkdir(procDest, 0o755); err != nil { - return err - } - } - case syscall.S_IFREG, - syscall.S_IFBLK, - syscall.S_IFCHR, - syscall.S_IFIFO, - syscall.S_IFSOCK: - sylog.Debugf("Creating file %s", filepath.Base(procDest)) - if c.userNS { - if _, err := c.rpcOps.Touch(dest); err != nil { - return err - } - } else { - if err := fs.Touch(procDest); err != nil { - return err - } - } - } - } - } - } else { - procDest := filepath.Join(c.rpcRoot, dest) - - sylog.Debugf("Checking if %s exists", procDest) - if _, err := os.Stat(procDest); os.IsNotExist(err) { - sylog.Warningf("destination %s doesn't exist", dest) - return nil - } - } - - if ignore { - sylog.Debugf("(re)mount %s", dest) - } else { - sylog.Debugf("Mount %s to %s : %s [%s]", source, dest, point.Type, optsString) - } - - err := c.rpcOps.Mount(source, dest, point.Type, flags, optsString) - if err != nil { - sylog.Debugf("RPC mount error: %s", err) - } - - return err -} diff --git a/internal/pkg/runtime/engine/oci/engine_linux.go b/internal/pkg/runtime/engine/oci/engine_linux.go deleted file mode 100644 index cb099a4125..0000000000 --- a/internal/pkg/runtime/engine/oci/engine_linux.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2019, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package oci - -import ( - "github.com/apptainer/apptainer/internal/pkg/runtime/engine" - "github.com/apptainer/apptainer/internal/pkg/runtime/engine/apptainer/rpc/server" - ociServer "github.com/apptainer/apptainer/internal/pkg/runtime/engine/oci/rpc/server" - "github.com/apptainer/apptainer/pkg/runtime/engine/config" -) - -// EngineOperations is an Apptainer OCI runtime engine that implements engine.Operations. -// Basically, this is the core of `apptainer oci` commands. -type EngineOperations struct { - CommonConfig *config.Common `json:"-"` - EngineConfig *EngineConfig `json:"engineConfig"` -} - -// InitConfig stores the parsed config.Common inside the engine. -// -// Since this method simply stores config.Common, it does not matter -// whether or not there are any elevated privileges during this call. -func (e *EngineOperations) InitConfig(cfg *config.Common, privStageOne bool) { - e.CommonConfig = cfg -} - -// Config returns a pointer to EngineConfig literal as a config.EngineConfig -// interface. This pointer gets stored in the Engine.Common field. -// -// Since this method simply returns a zero value of the concrete -// EngineConfig, it does not matter whether or not there are any elevated -// privileges during this call. -func (e *EngineOperations) Config() config.EngineConfig { - return e.EngineConfig -} - -func init() { - engine.RegisterOperations( - Name, - &EngineOperations{ - EngineConfig: &EngineConfig{}, - }, - ) - - ocimethods := new(ociServer.Methods) - ocimethods.Methods = new(server.Methods) - engine.RegisterRPCMethods( - Name, - ocimethods, - ) -} diff --git a/internal/pkg/runtime/engine/oci/monitor_linux.go b/internal/pkg/runtime/engine/oci/monitor_linux.go deleted file mode 100644 index 41fb666151..0000000000 --- a/internal/pkg/runtime/engine/oci/monitor_linux.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package oci - -import ( - "fmt" - "os" - "syscall" -) - -// MonitorContainer is called from master once the container has -// been spawned. It will block until the container exists. -// -// Additional privileges may be gained when running -// in suid flow. However, when a user namespace is requested and it is not -// a hybrid workflow (e.g. fakeroot), then there is no privileged saved uid -// and thus no additional privileges can be gained. -// -// Particularly here no additional privileges are gained as monitor does -// not need them for wait4 and kill syscalls. However, most likely this -// still will be executed as root since `apptainer oci` command set requires -// privileged execution. -func (e *EngineOperations) MonitorContainer(pid int, signals chan os.Signal) (syscall.WaitStatus, error) { - var status syscall.WaitStatus - - for { - s := <-signals - switch s { - case syscall.SIGCHLD: - if wpid, err := syscall.Wait4(pid, &status, syscall.WNOHANG, nil); err != nil { - return status, fmt.Errorf("error while waiting child: %s", err) - } else if wpid != pid { - continue - } - return status, nil - case syscall.SIGURG: - // Ignore SIGURG, which is used for non-cooperative goroutine - // preemption starting with Go 1.14. For more information, see - // https://github.com/golang/go/issues/24543. - break - default: - if err := syscall.Kill(pid, s.(syscall.Signal)); err != nil { - return status, fmt.Errorf("interrupted by signal %s", s.String()) - } - } - } -} diff --git a/internal/pkg/runtime/engine/oci/prepare_linux.go b/internal/pkg/runtime/engine/oci/prepare_linux.go deleted file mode 100644 index dd3df77bdc..0000000000 --- a/internal/pkg/runtime/engine/oci/prepare_linux.go +++ /dev/null @@ -1,272 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package oci - -import ( - "fmt" - "os" - - "github.com/apptainer/apptainer/internal/pkg/buildcfg" - "github.com/apptainer/apptainer/internal/pkg/cgroups" - "github.com/apptainer/apptainer/internal/pkg/runtime/engine/config/starter" - "github.com/apptainer/apptainer/internal/pkg/util/fs" - "github.com/apptainer/apptainer/pkg/ociruntime" - "github.com/apptainer/apptainer/pkg/sylog" - "github.com/apptainer/apptainer/pkg/util/apptainerconf" - "github.com/apptainer/apptainer/pkg/util/capabilities" - "github.com/creack/pty" - specs "github.com/opencontainers/runtime-spec/specs-go" -) - -// make master/slave as global variable to avoid GC close file descriptor -var ( - master *os.File - slave *os.File -) - -// PrepareConfig is called during stage1 to validate and prepare -// container configuration. It is responsible for reading capabilities, -// checking what namespaces are required, opening streams for attach and -// exec, etc. -// -// No additional privileges can be gained as any of them are already -// dropped by the time PrepareConfig is called. However, most likely this -// still will be executed as root since `apptainer oci` command set -// requires privileged execution. -// -//nolint:maintidx -func (e *EngineOperations) PrepareConfig(starterConfig *starter.Config) error { - if e.CommonConfig.EngineName != Name { - return fmt.Errorf("incorrect engine") - } - - if e.EngineConfig.OciConfig.Generator.Config != &e.EngineConfig.OciConfig.Spec { - return fmt.Errorf("bad engine configuration provided") - } - - if starterConfig.GetIsSUID() { - return fmt.Errorf("suid workflow disabled by administrator") - } - - if e.EngineConfig.OciConfig.Process == nil { - return fmt.Errorf("empty OCI process configuration") - } - - if e.EngineConfig.OciConfig.Linux == nil { - return fmt.Errorf("empty OCI linux configuration") - } - - // TODO - investigate whether this is the highest place to pull this value from apptainer.conf - if !fs.IsOwner(buildcfg.APPTAINER_CONF_FILE, 0) { - return fmt.Errorf("%s must be owned by root", buildcfg.APPTAINER_CONF_FILE) - } - sConf, err := apptainerconf.Parse(buildcfg.APPTAINER_CONF_FILE) - if err != nil { - return fmt.Errorf("unable to parse apptainer.conf file: %s", err) - } - e.EngineConfig.SystemdCgroups = sConf.SystemdCgroups - - // reset state config that could be passed to engine - e.EngineConfig.State = ociruntime.State{} - - user := &e.EngineConfig.OciConfig.Process.User - gids := make([]int, 0, len(user.AdditionalGids)+1) - - uid := int(user.UID) - gid := user.GID - - gids = append(gids, int(gid)) - for _, g := range user.AdditionalGids { - gids = append(gids, int(g)) - } - - starterConfig.SetTargetUID(uid) - starterConfig.SetTargetGID(gids) - - if !e.EngineConfig.Exec { - starterConfig.SetInstance(true) - } - - userNS := false - for _, ns := range e.EngineConfig.OciConfig.Linux.Namespaces { - if ns.Type == specs.UserNamespace { - userNS = true - break - } - } - - starterConfig.SetNsFlagsFromSpec(e.EngineConfig.OciConfig.Linux.Namespaces) - if err := starterConfig.SetNsPathFromSpec(e.EngineConfig.OciConfig.Linux.Namespaces); err != nil { - return err - } - - if userNS { - if len(e.EngineConfig.OciConfig.Linux.UIDMappings) == 0 { - return fmt.Errorf("user namespace invoked without uid mapping") - } - if len(e.EngineConfig.OciConfig.Linux.GIDMappings) == 0 { - return fmt.Errorf("user namespace invoked without gid mapping") - } - if err := starterConfig.AddUIDMappings(e.EngineConfig.OciConfig.Linux.UIDMappings); err != nil { - return err - } - if err := starterConfig.AddGIDMappings(e.EngineConfig.OciConfig.Linux.GIDMappings); err != nil { - return err - } - } - - if e.EngineConfig.OciConfig.Linux.RootfsPropagation != "" { - starterConfig.SetMountPropagation(e.EngineConfig.OciConfig.Linux.RootfsPropagation) - } else { - starterConfig.SetMountPropagation("private") - } - - starterConfig.SetNoNewPrivs(e.EngineConfig.OciConfig.Process.NoNewPrivileges) - - if e.EngineConfig.OciConfig.Process.Capabilities != nil { - if err := e.checkCapabilities(); err != nil { - return err - } - - // force cap_sys_admin for seccomp and no_new_priv flag - caps := append(e.EngineConfig.OciConfig.Process.Capabilities.Effective, "CAP_SYS_ADMIN") - starterConfig.SetCapabilities(capabilities.Effective, caps) - - caps = append(e.EngineConfig.OciConfig.Process.Capabilities.Permitted, "CAP_SYS_ADMIN") - starterConfig.SetCapabilities(capabilities.Permitted, caps) - - starterConfig.SetCapabilities(capabilities.Inheritable, e.EngineConfig.OciConfig.Process.Capabilities.Inheritable) - starterConfig.SetCapabilities(capabilities.Bounding, e.EngineConfig.OciConfig.Process.Capabilities.Bounding) - starterConfig.SetCapabilities(capabilities.Ambient, e.EngineConfig.OciConfig.Process.Capabilities.Ambient) - } - - e.EngineConfig.MasterPts = -1 - e.EngineConfig.SlavePts = -1 - e.EngineConfig.OutputStreams = [2]int{-1, -1} - e.EngineConfig.ErrorStreams = [2]int{-1, -1} - e.EngineConfig.InputStreams = [2]int{-1, -1} - - if e.EngineConfig.GetLogFormat() == "" { - sylog.Debugf("No log format specified, setting kubernetes log format by default") - e.EngineConfig.SetLogFormat("kubernetes") - } - - if !e.EngineConfig.Exec { - if e.EngineConfig.OciConfig.Process.Terminal { - var err error - - master, slave, err = pty.Open() - if err != nil { - return err - } - consoleSize := e.EngineConfig.OciConfig.Process.ConsoleSize - if consoleSize != nil { - var size pty.Winsize - - size.Cols = uint16(consoleSize.Width) - size.Rows = uint16(consoleSize.Height) - if err := pty.Setsize(slave, &size); err != nil { - return err - } - } - e.EngineConfig.MasterPts = int(master.Fd()) - if err := starterConfig.KeepFileDescriptor(e.EngineConfig.MasterPts); err != nil { - return err - } - e.EngineConfig.SlavePts = int(slave.Fd()) - if err := starterConfig.KeepFileDescriptor(e.EngineConfig.SlavePts); err != nil { - return err - } - } else { - r, w, err := os.Pipe() - if err != nil { - return err - } - e.EngineConfig.OutputStreams = [2]int{int(r.Fd()), int(w.Fd())} - if err := starterConfig.KeepFileDescriptor(e.EngineConfig.OutputStreams[0]); err != nil { - return err - } - if err := starterConfig.KeepFileDescriptor(e.EngineConfig.OutputStreams[1]); err != nil { - return err - } - - r, w, err = os.Pipe() - if err != nil { - return err - } - e.EngineConfig.ErrorStreams = [2]int{int(r.Fd()), int(w.Fd())} - if err := starterConfig.KeepFileDescriptor(e.EngineConfig.ErrorStreams[0]); err != nil { - return err - } - if err := starterConfig.KeepFileDescriptor(e.EngineConfig.ErrorStreams[1]); err != nil { - return err - } - - r, w, err = os.Pipe() - if err != nil { - return err - } - e.EngineConfig.InputStreams = [2]int{int(w.Fd()), int(r.Fd())} - if err := starterConfig.KeepFileDescriptor(e.EngineConfig.InputStreams[0]); err != nil { - return err - } - if err := starterConfig.KeepFileDescriptor(e.EngineConfig.InputStreams[1]); err != nil { - return err - } - } - } else { - starterConfig.SetNamespaceJoinOnly(true) - cPath := e.EngineConfig.OciConfig.Linux.CgroupsPath - if cPath == "" { - return nil - } - ppid := os.Getppid() - - sylog.Debugf("Adding process %d to instance cgroup %q", ppid, cPath) - manager, err := cgroups.GetManagerForGroup(cPath) - if err != nil { - return fmt.Errorf("couldn't create cgroup manager: %v", err) - } - if err := manager.AddProc(ppid); err != nil { - return fmt.Errorf("couldn't add process to instance cgroup: %v", err) - } - } - - return nil -} - -func (e *EngineOperations) checkCapabilities() error { - for _, capability := range e.EngineConfig.OciConfig.Process.Capabilities.Permitted { - if _, ok := capabilities.Map[capability]; !ok { - return fmt.Errorf("unrecognized capabilities %s", capability) - } - } - for _, capability := range e.EngineConfig.OciConfig.Process.Capabilities.Effective { - if _, ok := capabilities.Map[capability]; !ok { - return fmt.Errorf("unrecognized capabilities %s", capability) - } - } - for _, capability := range e.EngineConfig.OciConfig.Process.Capabilities.Inheritable { - if _, ok := capabilities.Map[capability]; !ok { - return fmt.Errorf("unrecognized capabilities %s", capability) - } - } - for _, capability := range e.EngineConfig.OciConfig.Process.Capabilities.Bounding { - if _, ok := capabilities.Map[capability]; !ok { - return fmt.Errorf("unrecognized capabilities %s", capability) - } - } - for _, capability := range e.EngineConfig.OciConfig.Process.Capabilities.Ambient { - if _, ok := capabilities.Map[capability]; !ok { - return fmt.Errorf("unrecognized capabilities %s", capability) - } - } - return nil -} diff --git a/internal/pkg/runtime/engine/oci/process_linux.go b/internal/pkg/runtime/engine/oci/process_linux.go deleted file mode 100644 index 09820a2d4f..0000000000 --- a/internal/pkg/runtime/engine/oci/process_linux.go +++ /dev/null @@ -1,497 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2020, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package oci - -import ( - "context" - "encoding/json" - "fmt" - "io" - "net" - "os" - osexec "os/exec" - "os/signal" - "path/filepath" - "strconv" - "strings" - "syscall" - - "github.com/apptainer/apptainer/internal/pkg/instance" - "github.com/apptainer/apptainer/internal/pkg/security" - "github.com/apptainer/apptainer/internal/pkg/util/exec" - "github.com/apptainer/apptainer/pkg/ociruntime" - "github.com/apptainer/apptainer/pkg/sylog" - "github.com/apptainer/apptainer/pkg/util/copy" - "github.com/apptainer/apptainer/pkg/util/rlimit" - "github.com/apptainer/apptainer/pkg/util/unix" - "github.com/creack/pty" - specs "github.com/opencontainers/runtime-spec/specs-go" -) - -// StartProcess is called during stage2 after RPC server finished -// environment preparation. This is the container process itself. -// -// No additional privileges can be gained during this call (unless container -// is executed as root intentionally) as starter will set uid/euid/suid -// to the targetUID (PrepareConfig will set it by calling starter.Config.SetTargetUID). -func (e *EngineOperations) StartProcess(masterConnFd int) error { - cwd := e.EngineConfig.OciConfig.Process.Cwd - - if cwd == "" { - cwd = "/" - } - - if !filepath.IsAbs(cwd) { - return fmt.Errorf("cwd property must be an absolute path") - } - - if err := os.Chdir(cwd); err != nil { - return fmt.Errorf("can't enter in current working directory: %s", err) - } - - if err := setRlimit(e.EngineConfig.OciConfig.Process.Rlimits); err != nil { - return err - } - - comm := os.NewFile(uintptr(masterConnFd), "master-socket") - masterConn, err := net.FileConn(comm) - comm.Close() - if err != nil { - return fmt.Errorf("failed to copy master unix socket descriptor: %s", err) - } - - if e.EngineConfig.EmptyProcess { - return e.emptyProcess(masterConn) - } - - args := e.EngineConfig.OciConfig.Process.Args - env := e.EngineConfig.OciConfig.Process.Env - - for _, e := range e.EngineConfig.OciConfig.Process.Env { - if strings.HasPrefix(e, "PATH=") { - os.Setenv("PATH", e[5:]) - } - } - - bpath, err := osexec.LookPath(args[0]) - if err != nil { - return fmt.Errorf("%s", err) - } - args[0] = bpath - - if e.EngineConfig.MasterPts != -1 { - slaveFd := e.EngineConfig.SlavePts - if err := syscall.Dup3(slaveFd, int(os.Stdin.Fd()), 0); err != nil { - return err - } - if err := syscall.Dup3(slaveFd, int(os.Stdout.Fd()), 0); err != nil { - return err - } - if err := syscall.Dup3(slaveFd, int(os.Stderr.Fd()), 0); err != nil { - return err - } - if err := syscall.Close(e.EngineConfig.MasterPts); err != nil { - return err - } - if err := syscall.Close(slaveFd); err != nil { - return err - } - if _, err := syscall.Setsid(); err != nil { - return err - } - if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, os.Stdin.Fd(), uintptr(syscall.TIOCSCTTY), 1); err != 0 { - return fmt.Errorf("failed to set crontrolling terminal: %s", err.Error()) - } - } else if e.EngineConfig.OutputStreams[1] != -1 { - if err := syscall.Dup3(e.EngineConfig.OutputStreams[1], int(os.Stdout.Fd()), 0); err != nil { - return err - } - if err := syscall.Close(e.EngineConfig.OutputStreams[1]); err != nil { - return err - } - if err := syscall.Close(e.EngineConfig.OutputStreams[0]); err != nil { - return err - } - - if err := syscall.Dup3(e.EngineConfig.ErrorStreams[1], int(os.Stderr.Fd()), 0); err != nil { - return err - } - if err := syscall.Close(e.EngineConfig.ErrorStreams[1]); err != nil { - return err - } - if err := syscall.Close(e.EngineConfig.ErrorStreams[0]); err != nil { - return err - } - - if err := syscall.Dup3(e.EngineConfig.InputStreams[1], int(os.Stdin.Fd()), 0); err != nil { - return err - } - if err := syscall.Close(e.EngineConfig.InputStreams[1]); err != nil { - return err - } - if err := syscall.Close(e.EngineConfig.InputStreams[0]); err != nil { - return err - } - } - - // trigger pre-start process - if _, err := masterConn.Write([]byte("t")); err != nil { - return fmt.Errorf("failed to pause process: %s", err) - } - if !e.EngineConfig.Exec { - // block on read start given - data := make([]byte, 1) - if _, err := masterConn.Read(data); err != nil { - return fmt.Errorf("failed to receive start signal: %s", err) - } - } - - if err := security.Configure(&e.EngineConfig.OciConfig.Spec); err != nil { - return fmt.Errorf("failed to apply security configuration: %s", err) - } - - err = syscall.Exec(args[0], args, env) - return fmt.Errorf("exec %s failed: %s", args[0], err) -} - -// PreStartProcess is called from master after before container startup. -// -// Additional privileges may be gained when running -// in suid flow. However, when a user namespace is requested and it is not -// a hybrid workflow (e.g. fakeroot), then there is no privileged saved uid -// and thus no additional privileges can be gained. -func (e *EngineOperations) PreStartProcess(ctx context.Context, pid int, masterConn net.Conn, fatalChan chan error) error { - if e.EngineConfig.Exec { - return nil - } - - file, err := instance.Get(e.CommonConfig.ContainerID, instance.OciSubDir) - if err != nil { - return err - } - e.EngineConfig.State.AttachSocket = filepath.Join(filepath.Dir(file.Path), "attach.sock") - - attach, err := unix.CreateSocket(e.EngineConfig.State.AttachSocket) - if err != nil { - return err - } - - e.EngineConfig.State.ControlSocket = filepath.Join(filepath.Dir(file.Path), "control.sock") - - control, err := unix.CreateSocket(e.EngineConfig.State.ControlSocket) - if err != nil { - return err - } - - logPath := e.EngineConfig.GetLogPath() - if logPath == "" { - containerID := e.CommonConfig.ContainerID - dir, err := instance.GetDir(containerID, instance.OciSubDir) - if err != nil { - return err - } - logPath = filepath.Join(dir, containerID+".log") - } - - format := e.EngineConfig.GetLogFormat() - formatter, ok := instance.LogFormats[format] - if !ok { - return fmt.Errorf("log format %s is not supported", format) - } - - logger, err := instance.NewLogger(logPath, formatter) - if err != nil { - return err - } - - pidFile := e.EngineConfig.GetPidFile() - if pidFile != "" { - if err := os.WriteFile(pidFile, []byte(strconv.Itoa(pid)), 0o644); err != nil { - return err - } - } - - if err := e.updateState(ociruntime.Created); err != nil { - return err - } - - start := make(chan bool, 1) - - go e.handleControl(masterConn, attach, control, logger, start, fatalChan) - - hooks := e.EngineConfig.OciConfig.Hooks - if hooks != nil { - for _, h := range hooks.Prestart { - if err := exec.Hook(ctx, &h, &e.EngineConfig.State.State); err != nil { - return err - } - } - } - - // detach process - syscall.Kill(os.Getppid(), syscall.SIGUSR1) - - // block until start event received - <-start - close(start) - - return nil -} - -// PostStartProcess is called from master after successful -// execution of the container process. It will execute OCI -// post start hooks (if any). -// -// Additional privileges may be gained when running -// in suid flow. However, when a user namespace is requested and it is not -// a hybrid workflow (e.g. fakeroot), then there is no privileged saved uid -// and thus no additional privileges can be gained. -// -// Here, however, oci engine does not escalate privileges, which means -// OCI hooks will be executed on behalf of a user who spawned a container -// (but not the one who runs it as targetUID may be arbitrary). -// -// Most likely this still will be executed as root since `apptainer oci` -// command set requires privileged execution. -func (e *EngineOperations) PostStartProcess(ctx context.Context, pid int) error { - if err := e.updateState(ociruntime.Running); err != nil { - return err - } - hooks := e.EngineConfig.OciConfig.Hooks - if hooks != nil { - for _, h := range hooks.Poststart { - if err := exec.Hook(ctx, &h, &e.EngineConfig.State.State); err != nil { - sylog.Warningf("%s", err) - } - } - } - return nil -} - -func setRlimit(rlimits []specs.POSIXRlimit) error { - resources := make(map[string]struct{}) - - for _, rl := range rlimits { - if err := rlimit.Set(rl.Type, rl.Soft, rl.Hard); err != nil { - return err - } - if _, found := resources[rl.Type]; found { - return fmt.Errorf("%s was already set", rl.Type) - } - resources[rl.Type] = struct{}{} - } - - return nil -} - -func (e *EngineOperations) emptyProcess(masterConn net.Conn) error { - // pause process on next read - if _, err := masterConn.Write([]byte("t")); err != nil { - return fmt.Errorf("failed to pause process: %s", err) - } - - // block on read start given - data := make([]byte, 1) - if _, err := masterConn.Read(data); err != nil { - return fmt.Errorf("failed to receive ack from master: %s", err) - } - - var status syscall.WaitStatus - signals := make(chan os.Signal, 1) - signal.Notify(signals, syscall.SIGCHLD, syscall.SIGINT, syscall.SIGTERM) - - if err := security.Configure(&e.EngineConfig.OciConfig.Spec); err != nil { - return fmt.Errorf("failed to apply security configuration: %s", err) - } - - masterConn.Close() - - for { - s := <-signals - switch s { - case syscall.SIGCHLD: - for { - if pid, _ := syscall.Wait4(-1, &status, syscall.WNOHANG, nil); pid <= 0 { - break - } - } - case syscall.SIGINT, syscall.SIGTERM: - os.Exit(0) - } - } -} - -func (e *EngineOperations) handleStream(l net.Listener, logger *instance.Logger, fatalChan chan error) { - var stdout io.ReadWriteCloser - var stderr io.ReadCloser - var stdin io.WriteCloser - var outputWriters *copy.MultiWriter - var errorWriters *copy.MultiWriter - var inputWriters *copy.MultiWriter - var tbuf *copy.TerminalBuffer - - hasTerminal := e.EngineConfig.OciConfig.Process.Terminal - - inputWriters = ©.MultiWriter{} - outputWriters = ©.MultiWriter{} - outWriter, _ := logger.NewWriter("stdout", true) - outputWriters.Add(outWriter) - - if hasTerminal { - stdout = os.NewFile(uintptr(e.EngineConfig.MasterPts), "stream-master-pts") - tbuf = copy.NewTerminalBuffer() - outputWriters.Add(tbuf) - inputWriters.Add(stdout) - } else { - outputStream := os.NewFile(uintptr(e.EngineConfig.OutputStreams[0]), "stdout-stream") - errorStream := os.NewFile(uintptr(e.EngineConfig.ErrorStreams[0]), "error-stream") - inputStream := os.NewFile(uintptr(e.EngineConfig.InputStreams[0]), "input-stream") - stdout = outputStream - stderr = errorStream - stdin = inputStream - outputWriters.Add(os.Stdout) - inputWriters.Add(stdin) - } - - if stderr != nil { - errorWriters = ©.MultiWriter{} - errWriter, _ := logger.NewWriter("stderr", true) - errorWriters.Add(errWriter) - errorWriters.Add(os.Stderr) - } - - go func() { - for { - c, err := l.Accept() - if err != nil { - fatalChan <- err - return - } - - go func() { - outputWriters.Add(c) - if stderr != nil { - errorWriters.Add(c) - } - - if tbuf != nil { - c.Write(tbuf.Line()) - } - - io.Copy(inputWriters, c) - - outputWriters.Del(c) - if stderr != nil { - errorWriters.Del(c) - } - c.Close() - }() - } - }() - - go func() { - io.Copy(outputWriters, stdout) - stdout.Close() - }() - - if stderr != nil { - go func() { - io.Copy(errorWriters, stderr) - stderr.Close() - }() - } - if stdin != nil { - go func() { - io.Copy(inputWriters, os.Stdin) - stdin.Close() - }() - } -} - -func (e *EngineOperations) handleControl(masterConn net.Conn, attach, control net.Listener, logger *instance.Logger, start chan bool, fatalChan chan error) { - var master *os.File - started := false - - if e.EngineConfig.OciConfig.Process.Terminal { - master = os.NewFile(uintptr(e.EngineConfig.MasterPts), "control-master-pts") - } - - for { - c, err := control.Accept() - if err != nil { - fatalChan <- err - return - } - dec := json.NewDecoder(c) - ctrl := &ociruntime.Control{} - if err := dec.Decode(ctrl); err != nil { - fatalChan <- err - return - } - - if ctrl.StartContainer && !started { - started = true - - e.handleStream(attach, logger, fatalChan) - - // since container process block on read, send it an - // ACK so when it will receive data, the container - // process will be executed - if _, err := masterConn.Write([]byte("s")); err != nil { - fatalChan <- fmt.Errorf("failed to send ACK to start process: %s", err) - return - } - - // send start event - start <- true - - // wait status update - e.waitStatusUpdate() - } - if ctrl.ConsoleSize != nil && master != nil { - size := &pty.Winsize{ - Cols: uint16(ctrl.ConsoleSize.Width), - Rows: uint16(ctrl.ConsoleSize.Height), - } - if err := pty.Setsize(master, size); err != nil { - fatalChan <- err - return - } - } - if ctrl.ReopenLog { - if err := logger.ReOpenFile(); err != nil { - fatalChan <- err - return - } - } - if ctrl.Pause { - if err := e.EngineConfig.Cgroups.Freeze(); err != nil { - fatalChan <- err - return - } - if err := e.updateState(ociruntime.Paused); err != nil { - fatalChan <- err - return - } - } - if ctrl.Resume { - if err := e.updateState(ociruntime.Running); err != nil { - fatalChan <- err - return - } - if err := e.EngineConfig.Cgroups.Thaw(); err != nil { - fatalChan <- err - return - } - } - - c.Close() - } -} diff --git a/internal/pkg/runtime/engine/oci/rpc/args.go b/internal/pkg/runtime/engine/oci/rpc/args.go deleted file mode 100644 index 6b0aca349c..0000000000 --- a/internal/pkg/runtime/engine/oci/rpc/args.go +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package rpc - -// TouchArgs defines the arguments to touch. -type TouchArgs struct { - Path string -} diff --git a/internal/pkg/runtime/engine/oci/rpc/client/client.go b/internal/pkg/runtime/engine/oci/rpc/client/client.go deleted file mode 100644 index 9ee0c05936..0000000000 --- a/internal/pkg/runtime/engine/oci/rpc/client/client.go +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package client - -import ( - "os" - - args "github.com/apptainer/apptainer/internal/pkg/runtime/engine/apptainer/rpc" - client "github.com/apptainer/apptainer/internal/pkg/runtime/engine/apptainer/rpc/client" - ociargs "github.com/apptainer/apptainer/internal/pkg/runtime/engine/oci/rpc" -) - -// RPC holds the state necessary for remote procedure calls. -type RPC struct { - client.RPC -} - -// MkdirAll calls the mkdir RPC using the supplied arguments. -func (t *RPC) MkdirAll(path string, perm os.FileMode) (int, error) { - arguments := &args.MkdirArgs{ - Path: path, - Perm: perm, - } - var reply int - err := t.Client.Call(t.Name+".MkdirAll", arguments, &reply) - return reply, err -} - -// Touch calls the touch RPC using the supplied arguments. -func (t *RPC) Touch(path string) (int, error) { - arguments := &ociargs.TouchArgs{ - Path: path, - } - var reply int - err := t.Client.Call(t.Name+".Touch", arguments, &reply) - return reply, err -} diff --git a/internal/pkg/runtime/engine/oci/rpc/server/server_linux.go b/internal/pkg/runtime/engine/oci/rpc/server/server_linux.go deleted file mode 100644 index fa2bfdfead..0000000000 --- a/internal/pkg/runtime/engine/oci/rpc/server/server_linux.go +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package server - -import ( - "os" - "syscall" - - "github.com/apptainer/apptainer/internal/pkg/util/fs" - - args "github.com/apptainer/apptainer/internal/pkg/runtime/engine/apptainer/rpc" - server "github.com/apptainer/apptainer/internal/pkg/runtime/engine/apptainer/rpc/server" - ociargs "github.com/apptainer/apptainer/internal/pkg/runtime/engine/oci/rpc" - "github.com/apptainer/apptainer/internal/pkg/util/mainthread" -) - -// Methods is a receiver type. -type Methods struct { - *server.Methods -} - -// MkdirAll performs a mkdir with the specified arguments. -func (t *Methods) MkdirAll(arguments *args.MkdirArgs, reply *int) (err error) { - mainthread.Execute(func() { - oldmask := syscall.Umask(0) - err = os.MkdirAll(arguments.Path, arguments.Perm) - syscall.Umask(oldmask) - }) - return err -} - -// Touch performs a touch with the specified arguments. -func (t *Methods) Touch(arguments *ociargs.TouchArgs, reply *int) (err error) { - return fs.Touch(arguments.Path) -} diff --git a/mlocal/frags/go_common_opts.mk b/mlocal/frags/go_common_opts.mk index f8ccf9cbab..dd797a133b 100644 --- a/mlocal/frags/go_common_opts.mk +++ b/mlocal/frags/go_common_opts.mk @@ -1,6 +1,6 @@ # go tool default build options GO111MODULE := on -GO_TAGS := containers_image_openpgp sylog oci_engine apptainer_engine fakeroot_engine +GO_TAGS := containers_image_openpgp sylog apptainer_engine fakeroot_engine GO_TAGS_SUID := containers_image_openpgp sylog apptainer_engine fakeroot_engine GO_LDFLAGS := # Need to use non-pie build on ppc64le From e604c9aba5a07f7c7de945ea2b880e6bc8f69337 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Fri, 25 Feb 2022 15:38:55 -0600 Subject: [PATCH 007/114] oci: add conmon for detached flow Signed-off-by: Edita Kizinevic --- .github/workflows/ci.yml | 6 +- e2e/oci/oci.go | 2 +- go.mod | 6 +- go.sum | 1138 +++++++++++++++++++- internal/app/apptainer/oci_attach_linux.go | 210 +++- internal/app/apptainer/oci_create_linux.go | 205 +++- internal/app/apptainer/oci_delete_linux.go | 34 +- internal/app/apptainer/oci_exec_linux.go | 15 +- internal/app/apptainer/oci_kill_linux.go | 16 +- internal/app/apptainer/oci_linux.go | 149 ++- internal/app/apptainer/oci_pause_linux.go | 28 +- internal/app/apptainer/oci_run_linux.go | 17 +- internal/app/apptainer/oci_start_linux.go | 16 +- internal/app/apptainer/oci_state_linux.go | 16 +- internal/app/apptainer/oci_update_linux.go | 16 +- 15 files changed, 1780 insertions(+), 94 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2c2894a98b..aed2e655a8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -238,7 +238,7 @@ jobs: go-version: 1.20.5 - name: Fetch deps - run: sudo apt-get -q update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential squashfs-tools squashfuse fuse-overlayfs fakeroot fuse2fs libseccomp-dev cryptsetup dbus-user-session + run: sudo apt-get -q update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential squashfs-tools squashfuse fuse-overlayfs fakeroot fuse2fs libseccomp-dev cryptsetup dbus-user-session conmon runc - name: Build and install Apptainer run: | @@ -268,7 +268,7 @@ jobs: go-version: 1.20.5 - name: Fetch deps - run: sudo apt-get -q update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential squashfs-tools libseccomp-dev cryptsetup dbus-user-session + run: sudo apt-get -q update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential squashfs-tools libseccomp-dev cryptsetup dbus-user-session conmon runc - name: Build and install Apptainer run: | @@ -314,7 +314,7 @@ jobs: - name: Fetch deps if: env.run_tests - run: sudo apt-get -q update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential uidmap squashfs-tools squashfuse fuse-overlayfs fakeroot fuse2fs libseccomp-dev cryptsetup dbus-user-session + run: sudo apt-get -q update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential uidmap squashfs-tools squashfuse fuse-overlayfs fakeroot fuse2fs libseccomp-dev cryptsetup dbus-user-session conmon runc - name: Fetch gocryptfs run: wget -O gocryptfs.tar.gz https://github.com/rfjakob/gocryptfs/releases/download/v2.3/gocryptfs_v2.3_linux-static_amd64.tar.gz && sudo tar xzvf gocryptfs.tar.gz -C /usr/local/bin gocryptfs diff --git a/e2e/oci/oci.go b/e2e/oci/oci.go index a89f913c74..bb4fdfe8ba 100644 --- a/e2e/oci/oci.go +++ b/e2e/oci/oci.go @@ -308,7 +308,7 @@ func (c ctx) testOciBasic(t *testing.T) { t, e2e.WithProfile(e2e.RootProfile), e2e.WithCommand("oci kill"), - e2e.WithArgs("-t", "2", containerID, "KILL"), + e2e.WithArgs(containerID, "KILL"), e2e.PostRun(func(t *testing.T) { if !t.Failed() { c.checkOciState(t, containerID, ociruntime.Stopped) diff --git a/go.mod b/go.mod index d465273efd..88d2a35eb0 100644 --- a/go.mod +++ b/go.mod @@ -17,8 +17,9 @@ require ( github.com/containerd/containerd v1.7.2 github.com/containernetworking/cni v1.1.2 github.com/containernetworking/plugins v1.3.0 + github.com/containers/common v0.47.5 github.com/containers/image/v5 v5.26.1 - github.com/creack/pty v1.1.18 + github.com/creack/pty v1.1.18 // indirect github.com/cyphar/filepath-securejoin v0.2.3 github.com/docker/docker v24.0.2+incompatible github.com/docker/go-units v0.5.0 @@ -26,6 +27,7 @@ require ( github.com/go-log/log v0.2.0 github.com/google/uuid v1.3.0 github.com/gosimple/slug v1.13.1 + github.com/moby/term v0.5.0 github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.1.0-rc4 github.com/opencontainers/runc v1.1.7 @@ -136,7 +138,6 @@ require ( github.com/moby/patternmatcher v0.5.0 // indirect github.com/moby/sys/mountinfo v0.6.2 // indirect github.com/moby/sys/sequential v0.5.0 // indirect - github.com/moby/term v0.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/morikuni/aec v1.0.0 // indirect @@ -152,7 +153,6 @@ require ( github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/safchain/ethtool v0.3.0 // indirect github.com/secure-systems-lab/go-securesystemslib v0.6.0 // indirect - github.com/sergi/go-diff v1.2.0 // indirect github.com/sigstore/fulcio v1.3.1 // indirect github.com/sigstore/rekor v1.2.2-0.20230601122533-4c81ff246d12 // indirect github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980 // indirect diff --git a/go.sum b/go.sum index 0ac66e539e..7c1f775e78 100644 --- a/go.sum +++ b/go.sum @@ -1,36 +1,138 @@ +bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= +cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= +cloud.google.com/go v0.98.0/go.mod h1:ua6Ush4NALrHk5QXDWnjvZHN93OuF0HfuEPq9I1X0cM= +cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/firestore v1.6.1/go.mod h1:asNXNOzBdyVQmEU+ggO8UPodTkEVFW5Qx+rwHnAz+EY= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/14rcole/gopopulate v0.0.0-20180821133914-b175b219e774/go.mod h1:6/0dYRLLXyJjbkIPeeGyoJ/eKOSI0eU6eTlCBYibgd0= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 h1:EKPd1INOIyr5hWOWhvpmQpY6tKjeG0hT1s3AMC/9fic= github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1/go.mod h1:VzwV+t+dZ9j/H867F1M2ziD+yLHtB46oM35FxxMJ4d0= github.com/AdamKorcz/go-fuzz-headers v0.0.0-20210312213058-32f4d319f0d2/go.mod h1:VPevheIvXETHZT/ddjwarP3POR5p/cnH9Hy5yoFnQjc= github.com/AdamKorcz/go-fuzz-headers v0.0.0-20210319161527-f761c2329661 h1:LxxqfxscKXL1kv7QNh4nggNf4Ais8B0ME8zWMCAsttY= github.com/AdamKorcz/go-fuzz-headers v0.0.0-20210319161527-f761c2329661/go.mod h1:VPevheIvXETHZT/ddjwarP3POR5p/cnH9Hy5yoFnQjc= +github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= +github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= +github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= +github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= +github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= +github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= +github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= +github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= +github.com/Microsoft/go-winio v0.5.1/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= +github.com/Microsoft/hcsshim v0.8.7-0.20190325164909-8abdbb8205e4/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= +github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= +github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8= +github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg= +github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00= +github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600= +github.com/Microsoft/hcsshim v0.8.20/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= +github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= +github.com/Microsoft/hcsshim v0.8.23/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg= +github.com/Microsoft/hcsshim v0.9.2/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= github.com/Microsoft/hcsshim v0.10.0-rc.8 h1:YSZVvlIIDD1UxQpJp0h+dnpLUw+TrY0cx8obKsp3bek= +github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= +github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63nhn5WAunQHLTznkw5W8b1Xc0dNjp83s= github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= +github.com/ProtonMail/go-crypto v0.0.0-20210920160938-87db9fbc61c7/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= +github.com/ProtonMail/go-crypto v0.0.0-20211112122917-428f8eabeeb3/go.mod h1:z4/9nQmJSSwwds7ejkxaJwO37dru3geImFUdJlaLzQo= github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 h1:wPbRQzjjwFc0ih8puEVAOFGELsn1zoIIYdxvML7mDxA= github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8/go.mod h1:I0gYDMZ6Z5GRU7l58bNFSkPTFN6Yl12dsUlAZ8xy98g= +github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d h1:UrqY+r/OJnIp5u0s1SbQ8dVfLCZJsnvazdBP5hS4iRs= +github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= +github.com/acomagu/bufpipe v1.0.3/go.mod h1:mxdxdup/WdsKVreO5GpW4+M/1CE2sMG4jeGJ2sYmHc4= github.com/adigunhammedolalekan/registry-auth v0.0.0-20200730122110-8cde180a3a60 h1:1IG6ye8dellBRE2uqvG0EzQScRqjsH/n5xOw+n0OGec= github.com/adigunhammedolalekan/registry-auth v0.0.0-20200730122110-8cde180a3a60/go.mod h1:DcXj4IQOoib2b4G2b8JU3VGV3ljXYbIq+PH4CcoAQTI= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= +github.com/alexflint/go-filemutex v1.1.0/go.mod h1:7P4iRhttt/nUvUOrYIhcpMzv2G6CY9UnI16Z+UJqRyk= github.com/alexflint/go-filemutex v1.2.0 h1:1v0TJPDtlhgpW4nJ+GvxCLSlUDC3+gW0CQQvlmfDR/s= github.com/alexflint/go-filemutex v1.2.0/go.mod h1:mYyQSWvw9Tx2/H2n9qXPb52tTYfE0pZAWcBq5mK025c= +github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apex/log v1.4.0/go.mod h1:UMNC4vQNC7hb5gyr47r18ylK1n34rV7GO+gb0wpXvcE= github.com/apex/log v1.9.0 h1:FHtw/xuaM8AgmvDDTI9fiwoAL25Sq2cxojnZICUU8l0= github.com/apex/log v1.9.0/go.mod h1:m82fZlWIuiWzWP04XCTXmnX0xRkYYbCdYn8jbJeLBEA= @@ -44,69 +146,229 @@ github.com/apptainer/container-library-client v1.4.5 h1:zESpm9aLqELb0zr3isDspYU+ github.com/apptainer/container-library-client v1.4.5/go.mod h1:EA5bDsL/dxzAxFh3zBszSdVpB25wIwv/66M1qLOzrNU= github.com/apptainer/sif/v2 v2.11.5 h1:EhSvg+eTDwlp5FNGdF7WBtr6LlHEIYmunewaTKimmrQ= github.com/apptainer/sif/v2 v2.11.5/go.mod h1:xSD5/qc/M+tS0K20RMexmTbqM4EOQMjN+hcPW6vJYg4= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= +github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= +github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= github.com/bshuster-repo/logrus-logstash-hook v1.0.0 h1:e+C0SB5R1pu//O4MQ3f9cFuPGoOVeF2fE4Og9otCc70= github.com/buger/goterm v1.0.4 h1:Z9YvGmOih81P0FbVtEYTFF6YsSgxSUKEhf/f9bTMXbY= github.com/buger/goterm v1.0.4/go.mod h1:HiFWV3xnkolgrBV3mY8m0X0Pumt4zg4QhbdOzQtB8tE= +github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= github.com/bugsnag/bugsnag-go v1.5.1 h1:NnfkWPiRGJlUg6s5mRlsbudWcW/B/eGFSad98JxitaU= github.com/bugsnag/bugsnag-go v1.5.1/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= +github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= +github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= github.com/bugsnag/panicwrap v1.2.0 h1:OzrKrRvXis8qEvOkfcxNcYbOd2O7xXS2nnKMEMABFQA= github.com/bugsnag/panicwrap v1.2.0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= +github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= +github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg= +github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLIdUjrmSXlK9pkrsDlLHbO8jiB8X8JnOc= +github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= +github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= +github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= +github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= github.com/cilium/ebpf v0.9.1 h1:64sn2K3UKw8NbP/blsixRpF3nXuyhz/VjRlRzvlBRu4= github.com/cilium/ebpf v0.9.1/go.mod h1:+OhNOIXx/Fnu1IE8bJz2dzOA+VSfyTfdNUVdlQnxUFY= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= +github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb h1:EDmT6Q9Zs+SbUoc7Ik9EfrFqcylYqgPZ9ANSbTAntnE= +github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= +github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= +github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= +github.com/containerd/aufs v1.0.0/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= +github.com/containerd/btrfs v0.0.0-20201111183144-404b9149801e/go.mod h1:jg2QkJcsabfHugurUvvPhS3E08Oxiuh5W/g1ybB4e0E= +github.com/containerd/btrfs v0.0.0-20210316141732-918d888fb676/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= +github.com/containerd/btrfs v1.0.0/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= +github.com/containerd/cgroups v0.0.0-20190717030353-c4b9ac5c7601/go.mod h1:X9rLEHIqSf/wfK8NsPqxJmeZgW4pcfzdXITDrUSJ6uI= +github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= +github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM= +github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= +github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= +github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= +github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= +github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= +github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= +github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= +github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw= +github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= +github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U= +github.com/containerd/containerd v1.2.10/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.1-0.20191213020239-082f7e3aed57/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.0-beta.2.0.20200729163537-40b22ef07410/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.9/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7V960Tmcumvqn8Mc+pCYQ= +github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU= +github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI= +github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s= +github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g= +github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c= +github.com/containerd/containerd v1.5.9/go.mod h1:fvQqCfadDGga5HZyn3j4+dx56qj2I9YwBrlSdalvJYQ= github.com/containerd/containerd v1.7.2 h1:UF2gdONnxO8I6byZXDi5sXWiWvlW3D/sci7dTQimEJo= github.com/containerd/containerd v1.7.2/go.mod h1:afcz74+K10M/+cjGHIVQrCt3RAQhUSCAjJ9iMYhhkuI= +github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cECdGN1O8G9bgKTlLhuPJimka6Xb/Gg7vYzCTNVxhvo= +github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y= +github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= +github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM= github.com/containerd/continuity v0.4.1 h1:wQnVrjIyQ8vhU2sgOiL5T07jo+ouqc2bnKsv5/EqGhU= +github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= +github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= +github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= +github.com/containerd/fifo v0.0.0-20201026212402-0724c46b320c/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= +github.com/containerd/fifo v0.0.0-20210316144830-115abcc95a1d/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= +github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= +github.com/containerd/go-cni v1.0.1/go.mod h1:+vUpYxKvAF72G9i1WoDOiPGRtQpqsNW/ZHtSlv++smU= +github.com/containerd/go-cni v1.0.2/go.mod h1:nrNABBHzu0ZwCug9Ije8hL2xBCYh/pjfMb1aZGrrohk= +github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= +github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= +github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328/go.mod h1:PpyHrqVs8FTi9vpyHwPwiNEGaACDxT/N/pLcvMSRA9g= +github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= +github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= +github.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak9TYCG3juvb0= +github.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6TNsg0ctmizkrOgXRNQjAPFWpMYRWuiB6dSF4Pfa5SA= +github.com/containerd/imgcrypt v1.1.1-0.20210312161619-7ed62a527887/go.mod h1:5AZJNI6sLHJljKuI9IHnw1pWqo/F0nGDOuR9zgTs7ow= +github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJrXQb0Dpc4ms= +github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c= +github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= +github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= +github.com/containerd/stargz-snapshotter/estargz v0.4.1/go.mod h1:x7Q9dg9QYb4+ELgxmo4gBUeJB0tl5dqH1Sdz0nJU1QM= +github.com/containerd/stargz-snapshotter/estargz v0.11.0/go.mod h1:/KsZXsJRllMbTKFfG0miFQWViQKdI9+9aSXs+HN0+ac= +github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= +github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= +github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8= +github.com/containerd/ttrpc v1.0.1/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= +github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= +github.com/containerd/ttrpc v1.1.0/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ= +github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= +github.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd/go.mod h1:GeKYzf2pQcqv7tJ0AoCuuhtnqhva5LNU3U+OyKxxJpk= +github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg= +github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s= +github.com/containerd/zfs v0.0.0-20200918131355-0a33824f23a2/go.mod h1:8IgZOBdv8fAgXddBT4dBXJPtxyRsejFIpXoklgxgEjw= +github.com/containerd/zfs v0.0.0-20210301145711-11e8f1707f62/go.mod h1:A9zfAbMlQwE+/is6hi0Xw8ktpL+6glmqZYtevJgaB8Y= +github.com/containerd/zfs v0.0.0-20210315114300-dde8f0fda960/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= +github.com/containerd/zfs v0.0.0-20210324211415-d5c4544f0433/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= +github.com/containerd/zfs v1.0.0/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= +github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= +github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= +github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= +github.com/containernetworking/cni v1.0.1/go.mod h1:AKuhXbN5EzmD4yTNtfSsX3tPcmtrBI6QcRV0NiNt15Y= github.com/containernetworking/cni v1.1.2 h1:wtRGZVv7olUHMOqouPpn3cXJWpJgM6+EUl31EQbXALQ= github.com/containernetworking/cni v1.1.2/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw= +github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM= +github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8= +github.com/containernetworking/plugins v1.0.1/go.mod h1:QHCfGpaTwYTbbH+nZXKVTxNBDZcxSOplJT5ico8/FLE= github.com/containernetworking/plugins v1.3.0 h1:QVNXMT6XloyMUoO2wUOqWTC1hWFV62Q6mVDp5H1HnjM= github.com/containernetworking/plugins v1.3.0/go.mod h1:Pc2wcedTQQCVuROOOaLBPPxrEXqqXBFt3cZ+/yVg6l0= +github.com/containers/common v0.47.5 h1:Qm9o+wVPO9sbggTKubN3xYMtPRaPv7dmcrJQgongHHw= +github.com/containers/common v0.47.5/go.mod h1:HgX0mFXyB0Tbe2REEIp9x9CxET6iSzmHfwR6S/t2LZc= +github.com/containers/image/v5 v5.19.1/go.mod h1:ewoo3u+TpJvGmsz64XgzbyTHwHtM94q7mgK/pX+v2SE= github.com/containers/image/v5 v5.26.1 h1:8y3xq8GO/6y8FR+nAedHPsAFiAtOrab9qHTBpbqaX8g= github.com/containers/image/v5 v5.26.1/go.mod h1:IwlOGzTkGnmfirXxt0hZeJlzv1zVukE03WZQ203Z9GA= +github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY= github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 h1:Qzk5C6cYglewc+UyGf6lc8Mj2UaPTHy/iF2De0/77CA= github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY= +github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc= +github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4= +github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= +github.com/containers/ocicrypt v1.1.2/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= github.com/containers/ocicrypt v1.1.7 h1:thhNr4fu2ltyGz8aMx8u48Ae0Pnbip3ePP9/mzkZ/3U= github.com/containers/ocicrypt v1.1.7/go.mod h1:7CAhjcj2H8AYp5YvEie7oVSK2AhBY8NscCYRawuDNtw= +github.com/containers/storage v1.38.2/go.mod h1:INP0RPLHWBxx+pTsO5uiHlDUGHDFvWZPWprAbAlQWPQ= github.com/containers/storage v1.48.0 h1:wiPs8J2xiFoOEAhxHDRtP6A90Jzj57VqzLRXOqeizns= github.com/containers/storage v1.48.0/go.mod h1:pRp3lkRo2qodb/ltpnudoXggrviRmaCmU5a5GhTBae0= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= +github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= github.com/coreos/go-iptables v0.6.0 h1:is9qnZMPYjLd8LYqmm/qlE+wwEgJIkTYdhV3rfZo4jk= github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= +github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= +github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY= github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= @@ -120,55 +382,114 @@ github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1S github.com/d2g/dhcp4client v1.0.0 h1:suYBsYZIkSlUMEz4TAYCczKf62IA2UWC+O8+KtdOhCo= github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s= github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5 h1:+CpLbZIeUn94m02LdEKPcgErLJ347NUwxPKs5u8ieiY= +github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8= +github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I= +github.com/danieljoos/wincred v1.1.0/go.mod h1:XYlo+eRTsVA9aHGp7NGjFkPla4m+DCL7hqDjlFjiygg= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= +github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/disiqueira/gotree/v3 v3.0.2/go.mod h1:ZuyjE4+mUQZlbpkI24AmruZKhg3VHEgPLDY8Qk+uUu8= github.com/distribution/distribution/v3 v3.0.0-20221208165359-362910506bc2 h1:aBfCb7iqHmDEIp6fBvC/hQUddQfg+3qdYjwzaiP9Hnc= +github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= +github.com/docker/cli v0.0.0-20191017083524-a8ff7f821017/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/cli v24.0.2+incompatible h1:QdqR7znue1mtkXIJ+ruQMGQhpw2JzMJLRXp6zpzF6tM= github.com/docker/cli v24.0.2+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= +github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/distribution v2.8.0+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v20.10.12+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v24.0.2+incompatible h1:eATx+oLz9WdNVkQrr0qjQ8HvRJ4bOOxfzEo8R+dA3cg= github.com/docker/docker v24.0.2+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= +github.com/docker/docker-credential-helpers v0.6.4/go.mod h1:ofX3UI0Gz1TteYBjtgs07O36Pyasyp66D2uKT7H8W1c= github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A= github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8= +github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= +github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI= github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQV8= github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4= github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/emirpasic/gods v1.18.1 h1:FXtiHYKDGKCW2KzwZKx0iC0PQmdlorYgdFG9jPXJ1Bc= github.com/emirpasic/gods v1.18.1/go.mod h1:8tpGGwCnJ5H4r6BWwaV6OrWmMoPhUl5jm/FMNAnJvWQ= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.1/go.mod h1:AY7fTTXNdv/aJ2O5jwpxAPOWUZ7hQAEvzN5Pf27BkQQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v0.6.2/go.mod h1:2t7qjJNvHPx8IjnBOzl9E9/baC+qXE/TeeyBRzgJDws= +github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a h1:yDWHCSQ40h88yih2JAcL6Ls/kVkSE8GFACTGVnMPruw= github.com/facebookgo/limitgroup v0.0.0-20150612190941-6abd8d71ec01 h1:IeaD1VDVBPlx3viJT9Md8if8IxxJnO+x0JCGb054heg= github.com/facebookgo/muster v0.0.0-20150708232844-fd3d7953fd52 h1:a4DFiKFJiDRGFD1qIcqGLX/WlUMD9dyLSLDt+9QZgt8= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7 h1:LofdAjjjqCSXMwLGgOgnE+rdPuvX9DxCqaHwKy7i/ko= github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= +github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= +github.com/go-git/go-billy/v5 v5.2.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-billy/v5 v5.3.1/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-git-fixtures/v4 v4.2.1/go.mod h1:K8zd3kDUAykwTdDCr+I0per6Y6vMiRR/nnVTBtavnB0= +github.com/go-git/go-git/v5 v5.4.2/go.mod h1:gQ1kArt6d+n+BGd+/B/I74HwRTLhth2+zti4ihgckDc= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-log/log v0.2.0 h1:z8i91GBudxD5L3RmF0KVpetCbcGWAV7q1Tw1eRwQM9Q= github.com/go-log/log v0.2.0/go.mod h1:xzCnwajcues/6w7lne3yK2QU7DBPW7kqbgPGG5AF65U= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -182,9 +503,14 @@ github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpX github.com/go-openapi/errors v0.20.2/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= github.com/go-openapi/errors v0.20.3 h1:rz6kiC84sqNQoqrtulzaL/VERgkoCyB6WdEkc2ujzUc= github.com/go-openapi/errors v0.20.3/go.mod h1:Z3FlZ4I8jEGxjUK+bugx3on2mIAk4txuAOhlsB1FSgk= +github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= +github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= +github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= github.com/go-openapi/jsonreference v0.20.0 h1:MYlu0sBgChmCfJxxUKZ8g1cPWFOB37YSZqewK7OKeyA= github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= @@ -193,6 +519,8 @@ github.com/go-openapi/loads v0.21.2 h1:r2a/xFIYeZ4Qd2TnGpWDIQNcP80dIaZgf704za8en github.com/go-openapi/loads v0.21.2/go.mod h1:Jq58Os6SSGz0rzh62ptiu8Z31I+OTHqmULx5e/gJbNw= github.com/go-openapi/runtime v0.26.0 h1:HYOFtG00FM1UvqrcxbEJg/SwvDRvYLQKGhw2zaQjTcc= github.com/go-openapi/runtime v0.26.0/go.mod h1:QgRGeZwrUcSHdeh4Ka9Glvo0ug1LC5WyE+EV88plZrQ= +github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= +github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= github.com/go-openapi/spec v0.20.6/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA= github.com/go-openapi/spec v0.20.9 h1:xnlYNQAwKd2VQRRfwTEI0DcK+2cbuvI/0c7jx3gA8/8= @@ -202,6 +530,8 @@ github.com/go-openapi/strfmt v0.21.1/go.mod h1:I/XVKeLc5+MM5oPNN7P6urMOpuLXEcNrC github.com/go-openapi/strfmt v0.21.3/go.mod h1:k+RzNO0Da+k3FrrynSNN8F7n/peCmQQqbbXjtDfvmGg= github.com/go-openapi/strfmt v0.21.7 h1:rspiXgNWgeUzhjo1YU01do6qsahtJNByjLVbPLNHb8k= github.com/go-openapi/strfmt v0.21.7/go.mod h1:adeGTkxE44sPyLk0JV235VQAO/ZXUr8KAzYjclFs3ew= +github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= +github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= @@ -237,20 +567,48 @@ github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWe github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= +github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= +github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= +github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= +github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU= +github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -260,132 +618,287 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v1.8.2 h1:H5XSIre1MB5NbPYFp+i1NBbb5qN1W8Y8YAQoAYbkm8k= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-containerregistry v0.5.1/go.mod h1:Ct15B4yir3PLOP5jsy0GNeYVaIZs/MK/Jz5any1wFW0= github.com/google/go-containerregistry v0.15.2 h1:MMkSh+tjSdnmJZO7ljvEqV1DjfekB6VUEAZgy3a+TQE= github.com/google/go-containerregistry v0.15.2/go.mod h1:wWK+LnOv4jXMM23IT/F1wdYftGWGr47Is8CG+pmHK1Q= +github.com/google/go-intervals v0.0.2/go.mod h1:MkaR3LNRfeKLPmqgJYs4E66z5InYjmCjbbr4TQlcT6Y= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20230323073829-e72429f035bd h1:r8yyd+DJDmsUhGrRBxH5Pj7KeFK5l+Y3FsgT8keqKtk= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= +github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= +github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gosimple/slug v1.13.1 h1:bQ+kpX9Qa6tHRaK+fZR0A0M2Kd7Pa5eHPPsb1JpHD+Q= github.com/gosimple/slug v1.13.1/go.mod h1:UiRaFH+GEilHstLUmcBgWcI42viBN7mAb818JrYOeFQ= github.com/gosimple/unidecode v1.0.1 h1:hZzFTMMqSswvf0LBJZCZgThIZrpDHFXux9KeGmn6T/o= github.com/gosimple/unidecode v1.0.1/go.mod h1:CP0Cr1Y1kogOtx0bJblKzsVWrqYaqfNOnHzpgWw4Awc= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= +github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= +github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-hclog v1.0.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY= +github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= +github.com/hashicorp/memberlist v0.2.2/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= +github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE= +github.com/hashicorp/serf v0.9.5/go.mod h1:UWDWwZeL5cuWDJdl0C6wrvrUwEqtQ4ZKBKKENpqIUyk= +github.com/hashicorp/serf v0.9.6/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4= github.com/honeycombio/beeline-go v1.10.0 h1:cUDe555oqvw8oD76BQJ8alk7FP0JZ/M/zXpNvOEDLDc= github.com/honeycombio/libhoney-go v1.16.0 h1:kPpqoz6vbOzgp7jC6SR7SkNj7rua7rgxvznI6M3KdHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= +github.com/j-keck/arping v1.0.2/go.mod h1:aJbELhR92bSk7tp79AWM/ftfc90EfEi2bQJrbBFOsPw= +github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= +github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= +github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg= +github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmhodges/clock v0.0.0-20160418191101-880ee4c33548 h1:dYTbLf4m0a5u0KLmPfB6mgxbcV7588bOCx79hxa5Sr4= +github.com/joefitzgerald/rainbow-reporter v0.1.0/go.mod h1:481CNgqmVHQZzdIbN52CupLJyoVwB10FQ/IQlF1pdL8= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 h1:iQTw/8FWTuc7uiaSepXwyf3o52HaUYcV+Tu66S3F5GA= github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= +github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kevinburke/ssh_config v1.1.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF4nAY/ojJ6r6mM= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.14.2/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.16.6 h1:91SKEy4K37vkp255cJ8QesJhjyRO0hn9i9G0GoUwLsk= github.com/klauspost/compress v1.16.6/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/pgzip v1.2.4/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/letsencrypt/boulder v0.0.0-20230213213521-fdfea0d469b6 h1:unJdfS94Y3k85TKy+mvKzjW5R9rIC+Lv4KGbE7uNu0I= github.com/letsencrypt/boulder v0.0.0-20230213213521-fdfea0d469b6/go.mod h1:PUgW5vI9ANEaV6qv9a6EKu8gAySgwf0xrzG9xIB/CK0= +github.com/linuxkit/virtsock v0.0.0-20201010232012-f8cee7dfc7a3/go.mod h1:3r6x7q95whyfWQpmGZTu3gk3v2YkMi05HEzl7Tf7YEo= +github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= +github.com/magefile/mage v1.11.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/manifoldco/promptui v0.9.0/go.mod h1:ka04sppxSGFAtxX0qhlYQjISsg9mR4GWtQEhdbn6Pgg= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= +github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= +github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= +github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk= github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= +github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/miekg/pkcs11 v1.1.1 h1:Ugu9pdy6vAYku5DEpVWVFPYnzV+bxB+iRdbuFSu7TvU= github.com/miekg/pkcs11 v1.1.1/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= +github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= +github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= github.com/mndrix/tap-go v0.0.0-20171203230836-629fa407e90b/go.mod h1:pzzDgJWZ34fGzaAZGFW22KVZDfyrYW+QABMrWnJBnSs= github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= github.com/moby/patternmatcher v0.5.0 h1:YCZgJOeULcxLw1Q+sVR636pmS7sPEn1Qo2iAN6M7DBo= github.com/moby/patternmatcher v0.5.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= +github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= +github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78= github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= +github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= +github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= +github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6/go.mod h1:E2VnQOmVuvZB6UYnnDB0qG5Nq/1tD9acaOpo6xmt0Kw= github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0= github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -401,103 +914,195 @@ github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJ github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/networkplumbing/go-nft v0.3.0 h1:IIc6yHjN85KyJx21p3ZEsO0iBMYHNXux22rc9Q8TfFw= github.com/networkplumbing/go-nft v0.3.0/go.mod h1:HnnM+tYvlGAsMU7yoYwXEVLLiDW9gdMmb5HoGcwpuQs= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= +github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= +github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.0.0/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= github.com/onsi/ginkgo/v2 v2.9.2 h1:BA2GMJOtfGAfagzYtrAlufIP0lq6QERkFmHLMLPwFSU= +github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= +github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.18.1/go.mod h1:0q+aL8jAiMXy9hbwj2mr5GziHiwhAIQpFmmtT5hitRs= github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= +github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= +github.com/opencontainers/image-spec v1.0.3-0.20211202193544-a5463b7f9c84/go.mod h1:Qnt1q4cjDNQI9bT832ziho5Iw2BhK8o1KwLOwW56VP4= github.com/opencontainers/image-spec v1.1.0-rc4 h1:oOxKUJWnFC4YGHCCMNql1x4YaDfYBTS5Y4x/Cgeo1E0= github.com/opencontainers/image-spec v1.1.0-rc4/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8= +github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc90/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0= +github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= +github.com/opencontainers/runc v1.1.0/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc= github.com/opencontainers/runc v1.1.7 h1:y2EZDS8sNng4Ksf0GUYNhKbTShZJPJg1FiXJNH/uoCk= github.com/opencontainers/runc v1.1.7/go.mod h1:CbUumNnWCuTGFukNXahoo/RFBZvDAgRh/smNYNOhA50= +github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.3-0.20200710190001-3e4195d92445/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.1.0-rc.3 h1:l04uafi6kxByhbxev7OWiuUv0LZxEsYUfDWZ6bztAuU= github.com/opencontainers/runtime-spec v1.1.0-rc.3/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= github.com/opencontainers/runtime-tools v0.9.0/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 h1:DmNGcqH3WDbV5k8OJ+esPWbqUOX5rMLR2PMvziDMJi0= github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626/go.mod h1:BRHJJd0E+cx42OybVYSgUvZmU0B8P9gZuRXlZUP7TKI= github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= +github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= +github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= github.com/opencontainers/selinux v1.9.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= +github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= github.com/opencontainers/selinux v1.11.0 h1:+5Zbo97w3Lbmb3PeqQtpmTkMwsW5nRI3YaLpt7tQ7oU= github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M586T4DlDRYpFkyec= github.com/opencontainers/umoci v0.4.7 h1:mbIbtMpZ3v9oMpKaLopnWoLykgmnixeLzq51EzAX5nQ= github.com/opencontainers/umoci v0.4.7/go.mod h1:lgJ4bnwJezsN1o/5d7t/xdRPvmf8TvBko5kKYJsYvgo= +github.com/ostreedev/ostree-go v0.0.0-20190702140239-759a8c1ac913/go.mod h1:J6OG6YJVEWopen4avK3VNQSnALmmjvniMmni/YFYAwc= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= +github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= +github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.0.8 h1:0ctb6s9mE31h0/lhu+J6OPmVeDxJn+kYnJc2jZR9tGQ= github.com/pelletier/go-toml/v2 v2.0.8/go.mod h1:vuYfssBdrU2XDZ9bYydBu6t+6a6PYNcZljzZR9VXg+4= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 h1:Ii+DKncOVM8Cu1Hc+ETb5K+23HdAMvESYE3ZJ5b5cMI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.10.1/go.mod h1:lYOWFsE0bwd1+KfKJaKeuokY15vzFx25BLbzYYoAxZI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= +github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= +github.com/proglottis/gpgme v0.1.1/go.mod h1:fPbW/EZ0LvwQtH8Hy7eixhp1eF3G39dtx7GUN+0Gmy0= github.com/proglottis/gpgme v0.1.3 h1:Crxx0oz4LKB3QXc5Ea0J19K/3ICfy3ftr5exgUK1AU0= github.com/proglottis/gpgme v0.1.3/go.mod h1:fPbW/EZ0LvwQtH8Hy7eixhp1eF3G39dtx7GUN+0Gmy0= +github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= +github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= +github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= +github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= +github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= +github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rootless-containers/proto v0.1.0 h1:gS1JOMEtk1YDYHCzBAf/url+olMJbac7MTrgSeP6zh4= github.com/rootless-containers/proto v0.1.0/go.mod h1:vgkUFZbQd0gcE/K/ZwtE4MYjZPu0UNHLXIQxhyqAFh8= +github.com/russross/blackfriday v1.6.0/go.mod h1:ti0ldHuxg49ri4ksnFxlkCfN+hvslNlmVHqNRXXJNAY= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= +github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= github.com/safchain/ethtool v0.3.0 h1:gimQJpsI6sc1yIqP/y8GYgiXn/NjgvpM0RNoWLVVmP0= github.com/safchain/ethtool v0.3.0/go.mod h1:SA9BwrgyAqNo7M+uaL6IYbxpm5wk3L7Mm6ocLW+CJUs= +github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= +github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= +github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sebdah/goldie/v2 v2.5.3 h1:9ES/mNN+HNUbNWpVAlrzuZ7jE+Nrczbj8uFRjM7624Y= +github.com/sebdah/goldie/v2 v2.5.3/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= github.com/seccomp/containers-golang v0.6.0 h1:VWPMMIDr8pAtNjCX0WvLEEK9EQi5lAm4HtJbDtAtFvQ= github.com/seccomp/containers-golang v0.6.0/go.mod h1:Dd9mONHvW4YdbSzdm23yf2CFw0iqvqLhO0mEFvPIvm4= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= +github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/seccomp/libseccomp-golang v0.10.0 h1:aA4bp+/Zzi0BnWZ2F1wgNBs5gTpm+na2rWM6M9YjLpY= github.com/seccomp/libseccomp-golang v0.10.0/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/secure-systems-lab/go-securesystemslib v0.6.0 h1:T65atpAVCJQK14UA57LMdZGpHi4QYSH/9FZyNGqMYIA= github.com/secure-systems-lab/go-securesystemslib v0.6.0/go.mod h1:8Mtpo9JKks/qhPG4HGZ2LGMvrPbzuxwfz/f/zLfEWkk= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8= @@ -509,6 +1114,8 @@ github.com/sigstore/rekor v1.2.2-0.20230601122533-4c81ff246d12 h1:x/WnxasgR40qGY github.com/sigstore/rekor v1.2.2-0.20230601122533-4c81ff246d12/go.mod h1:8c+a8Yo7r8gKuYbIaz+c3oOdw9iMXx+tMdOg2+b+2jQ= github.com/sigstore/sigstore v1.7.1 h1:fCATemikcBK0cG4+NcM940MfoIgmioY1vC6E66hXxks= github.com/sigstore/sigstore v1.7.1/go.mod h1:0PmMzfJP2Y9+lugD0wer4e7TihR5tM7NcIs3bQNk5xg= +github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= +github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= @@ -520,21 +1127,44 @@ github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= +github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= +github.com/spf13/cobra v1.3.0/go.mod h1:BrRVncBjOJa/eUcVVm9CE+oC6as8k+VYr4NY7WCi9V4= github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= +github.com/spf13/viper v1.10.0/go.mod h1:SoyBPwAtKDzypXNDFKN5kzH7ppppbGZtls1UpIy5AsM= github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980 h1:lIOOHPEbXzO3vnmx2gok1Tfs31Q8GQqKLc8vVqyQq/I= github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= +github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -546,13 +1176,19 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/sylabs/json-resp v0.8.2 h1:k2dgtMXL+nztYtxrI24Zck/sfexyly4D1+X504eZTKQ= github.com/sylabs/json-resp v0.8.2/go.mod h1:Q9X4wRlZNPv3x76KaL8vTCBO4aC/DP2gh13xdtEqd1g= github.com/sylabs/oras-go v1.2.4-0.20230628133146-a64659fc0454 h1:mYW7NTm96PhI8MLJ9Sp0cE8evQf1FL4q64P4lVqNtvI= github.com/sylabs/oras-go v1.2.4-0.20230628133146-a64659fc0454/go.mod h1:H1q/Fxq/+StNafSx4Svb30ozrUEgeo5yBwKMXGnZV+w= +github.com/sylabs/release-tools v0.1.0/go.mod h1:pqP/z/11/rYMQ0OM/Nn7TxGijw7KfZwW9UolD/J1TUo= +github.com/sylabs/sif/v2 v2.3.1/go.mod h1:NnvveH62GiibimL00MrI6YYcZfb7DnZMcRo/40giY+0= +github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= +github.com/tchap/go-patricia v2.3.0+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= github.com/theupdateframework/go-tuf v0.5.2 h1:habfDzTmpbzBLIFGWa2ZpVhYvFBoK0C1onC3a4zuPRA= github.com/theupdateframework/go-tuf v0.5.2/go.mod h1:SyMV5kg5n4uEclsyxXJZI2UxPFJNDc4Y+r7wv+MlvTA= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= @@ -566,48 +1202,86 @@ github.com/tj/go-buffer v1.1.0/go.mod h1:iyiJpfFcR2B9sXu7KvjbT9fpM4mOelRSDTbntVj github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0= github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao= github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.11 h1:kpFauv27b6ynzBNT/Xy+1k+fK4WswhN/6PN5WhFAGw8= github.com/ulikunitz/xz v0.5.11/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.19.1/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.12 h1:igJgVw1JdKH+trcLWLeLwZjU9fEfPesQ+9/e4MQ44S8= github.com/urfave/cli v1.22.12/go.mod h1:sSBEIC79qR6OvcmsD4U3KABeOTxDqQtdDnaFuUN30b8= github.com/vbatts/go-mtree v0.5.0 h1:dM+5XZdqH0j9CSZeerhoN/tAySdwnmevaZHO1XGW2Vc= github.com/vbatts/go-mtree v0.5.0/go.mod h1:7JbaNHyBMng+RP8C3Q4E+4Ca8JnGQA2R/MB+jb4tSOk= +github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI= github.com/vbatts/tar-split v0.11.3 h1:hLFqsOLQ1SsppQNTMpkpPXClLDfC2A3Zgy9OUU+RVck= github.com/vbatts/tar-split v0.11.3/go.mod h1:9QlHN18E+fEH7RdG+QAJJcuya3rqT7eXSTY7wGrAokY= +github.com/vbauerster/mpb/v7 v7.3.2/go.mod h1:wfxIZcOJq/bG1/lAtfzMXcOiSvbqVi/5GX5WCSi+IsA= github.com/vbauerster/mpb/v8 v8.4.0 h1:Jq2iNA7T6SydpMVOwaT+2OBWlXS9Th8KEvBqeu5eeTo= github.com/vbauerster/mpb/v8 v8.4.0/go.mod h1:vjp3hSTuCtR+x98/+2vW3eZ8XzxvGoP8CPseHMhiPyc= +github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= +github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= +github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= +github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs= github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= +github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= +github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= +github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU= github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= +github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0= +github.com/xanzy/ssh-agent v0.3.1/go.mod h1:QIE4lCeL7nkC25x+yA3LBIYfwCc1TFziCtG7cBAac6w= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo= github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= github.com/yvasiyarov/go-metrics v0.0.0-20150112132944-c25f46c4b940 h1:p7OofyZ509h8DmPLh8Hn+EIIZm/xYhdZHJ9GnXHdr6U= github.com/yvasiyarov/go-metrics v0.0.0-20150112132944-c25f46c4b940/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50 h1:hlE8//ciYMztlGpl/VA+Zm1AcTPHYkHJPbHqE6WJUXE= +github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f h1:ERexzlUfuTvpE74urLSbIQW0Z/6hF9t8U4NsJLaioAY= +github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= +go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/bbolt v1.3.7 h1:j+zJOnnEjF/kyHlDDgGnVL/AIqIJPq8UoB2GSNfkUfQ= go.etcd.io/bbolt v1.3.7/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= +go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= +go.etcd.io/etcd/api/v3 v3.5.1/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= +go.etcd.io/etcd/client/pkg/v3 v3.5.1/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= +go.etcd.io/etcd/client/v2 v2.305.1/go.mod h1:pMEacxZW7o8pg4CrFE7pquyCJJzZvkvdD2RibOCCCGs= go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg= go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng= go.mongodb.org/mongo-driver v1.10.0/go.mod h1:wsihk0Kdgv8Kqu1Anit4sfK+22vSFbUrAVEYRhCXrA8= @@ -616,55 +1290,172 @@ go.mongodb.org/mongo-driver v1.11.3/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5queth go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352 h1:CCriYyAfq1Br1aIYettdHZTy8mBTIPo7We18TuO/bak= go.mozilla.org/pkcs7 v0.0.0-20210826202110-33d05740a352/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opentelemetry.io/otel v1.15.0 h1:NIl24d4eiLJPM0vKn4HjLYM+UZf6gSfi9Z+NmCxkWbk= go.opentelemetry.io/otel v1.15.0/go.mod h1:qfwLEbWhLPk5gyWrne4XnF0lC8wtywbuJbgfAE3zbek= go.opentelemetry.io/otel/trace v1.15.0 h1:5Fwje4O2ooOxkfyqI/kJwxWotggDLix4BSAvpE1wlpo= go.opentelemetry.io/otel/trace v1.15.0/go.mod h1:CUsmE2Ht1CRkvE8OsMESvraoZrrcgD1J2W8GV1ev0Y4= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= +golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.10.0 h1:LKqV2xt9+kDzSTfOhx4FrkEBcMrAgHSYgzywV9zcGmM= golang.org/x/crypto v0.10.0/go.mod h1:o4eNf7Ede1fv+hwOwZsTHl9EsPFO6q6ZvYR8vYfY45I= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc= golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210929193557-e81a3d93ecf6/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211005180243-6b3c2da341f1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -672,46 +1463,130 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200720211630-cb9d2d5c5666/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210331175145-43e1dd70ce54/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211001092434-39dca1131b70/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -720,51 +1595,246 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.9.0 h1:GRRCnKYhdQrD8kfRAdQ6Zcw1P0OcELxGLKJvtjVMZ28= golang.org/x/term v0.9.0/go.mod h1:M6DEAAIenWoTxdKrOltXcmDY3rSplQUkrvaDU5FcQyo= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190706070813-72ffa07ba3db/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20200916195026-c9a70fc28ce3/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.8.0 h1:vSDcovVPld282ceKgDimkRSC8kpaH1dgyc9UMzlt84Y= golang.org/x/tools v0.8.0/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU= +google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= +google.golang.org/api v0.62.0/go.mod h1:dKmwPCydfsad4qCH08MSdgWjfHOyfpd4VtDGgRFdavw= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190522204451-c2c4e71fbf69/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200117163144-32f20d992d24/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200527145253-8367513e4ece/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211008145708-270636b82663/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211028162531-8db9c33dc351/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211129164237-f09f9a12af12/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211203200212-54befc351ae9/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -777,26 +1847,43 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/alexcesaro/statsd.v2 v2.0.0 h1:FXkZSCZIH17vLCO5sO2UucTHsH9pc+17F6pl3JVCwMc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/go-jose/go-jose.v2 v2.6.1 h1:qEzJlIDmG9q5VO0M/o8tGS65QMHMS1w01TQJB1VPJ4U= gopkg.in/go-jose/go-jose.v2 v2.6.1/go.mod h1:zzZDPkNNw/c9IE7Z9jr11mBZQhKQTMzoEEIoEdZlFBI= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.66.2/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= @@ -808,9 +1895,58 @@ gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= gotest.tools/v3 v3.5.0 h1:Ljk6PdHdOhAb5aDMWXjDLMMhph+BpztA4v1QdqEW2eY= gotest.tools/v3 v3.5.0/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= +k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= +k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= +k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc= +k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= +k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM= +k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q= +k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= +k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k= +k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0= +k8s.io/code-generator v0.19.7/go.mod h1:lwEq3YnLYb/7uVXLorOJfxg+cUu2oihFhHZ0n9NIla0= +k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= +k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI= +k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM= +k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM= +k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= +k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= +k8s.io/cri-api v0.20.6/go.mod h1:ew44AjNXwyn1s0U4xCKGodU7J1HzBeZ1MpGrpa5r8Yc= +k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20200428234225-8167cfdcfc14/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= +k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= +k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= +k8s.io/utils v0.0.0-20201110183641-67b214c5f920/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= mvdan.cc/sh/v3 v3.6.1-0.20221221181323-d3feb15bed3a h1:AK8d/j//vNToLhsO0B15DybhgHIQ27GbmvcTXv7S7VY= mvdan.cc/sh/v3 v3.6.1-0.20221221181323-d3feb15bed3a/go.mod h1:U4mhtBLZ32iWhif5/lD+ygy1zrgaQhUu+XFy7C8+TTA= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= +sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/internal/app/apptainer/oci_attach_linux.go b/internal/app/apptainer/oci_attach_linux.go index f5086842bc..47fead0bf2 100644 --- a/internal/app/apptainer/oci_attach_linux.go +++ b/internal/app/apptainer/oci_attach_linux.go @@ -2,19 +2,223 @@ // Apptainer a Series of LF Projects LLC. // For website terms of use, trademark policy, privacy policy and other // project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2021, Sylabs Inc. All rights reserved. +// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. +// Includes code from https://github.com/containers/podman +// Released under the Apache License Version 2.0 + package apptainer import ( + "bufio" "context" "fmt" + "io" + "net" + "os" + "path/filepath" + + "github.com/apptainer/apptainer/pkg/sylog" + "github.com/containers/common/pkg/config" + "github.com/moby/term" + "github.com/pkg/errors" + "golang.org/x/sys/unix" ) -// OciAttach attaches console to a running container +var ErrDetach = errors.New("detached from container") + +// OciAttach attaches the console to a running container func OciAttach(ctx context.Context, containerID string) error { - return fmt.Errorf("TODO - NOT IMPLEMENTED") + streams := AttachStreams{ + OutputStream: os.Stdout, + ErrorStream: os.Stderr, + InputStream: bufio.NewReader(os.Stdin), + AttachOutput: true, + AttachError: true, + AttachInput: true, + } + + sd, err := stateDir(containerID) + if err != nil { + return fmt.Errorf("while computing state directory: %w", err) + } + attachSock := filepath.Join(sd, bundleLink, attachSocket) + conn, err := openUnixSocket(attachSock) + if err != nil { + return errors.Wrapf(err, "while connecting to attach socket: %w", attachSock) + } + defer func() { + if err := conn.Close(); err != nil { + sylog.Errorf("while closing attach socket: %w", err) + } + }() + + detachKeys, err := processDetachKeys(config.DefaultDetachKeys) + if err != nil { + return fmt.Errorf("invalid detach key sequence: %w", err) + } + + receiveStdoutError, stdinDone := setupStdioChannels(streams, conn, detachKeys) + + return readStdio(conn, streams, receiveStdoutError, stdinDone) +} + +// The following utility functions are taken from https://github.com/containers/podman +// Released under the Apache License Version 2.0 + +func openUnixSocket(path string) (*net.UnixConn, error) { + fd, err := unix.Open(path, unix.O_PATH, 0) + if err != nil { + return nil, err + } + defer unix.Close(fd) + return net.DialUnix("unixpacket", nil, &net.UnixAddr{Name: fmt.Sprintf("/proc/self/fd/%d", fd), Net: "unixpacket"}) +} + +func setupStdioChannels(streams AttachStreams, conn *net.UnixConn, detachKeys []byte) (chan error, chan error) { + receiveStdoutError := make(chan error) + go func() { + receiveStdoutError <- redirectResponseToOutputStreams(streams.OutputStream, streams.ErrorStream, streams.AttachOutput, streams.AttachError, conn) + }() + + stdinDone := make(chan error) + go func() { + var err error + if streams.AttachInput { + _, err = CopyDetachable(conn, streams.InputStream, detachKeys) + } + stdinDone <- err + }() + + return receiveStdoutError, stdinDone +} + +func redirectResponseToOutputStreams(outputStream, errorStream io.Writer, writeOutput, writeError bool, conn io.Reader) error { + var err error + buf := make([]byte, 8192+1) /* Sync with conmon STDIO_BUF_SIZE */ + for { + nr, er := conn.Read(buf) + if nr > 0 { + var dst io.Writer + var doWrite bool + switch buf[0] { + case AttachPipeStdout: + dst = outputStream + doWrite = writeOutput + case AttachPipeStderr: + dst = errorStream + doWrite = writeError + default: + sylog.Infof("Received unexpected attach type %+d", buf[0]) + } + if dst == nil { + return errors.New("output destination cannot be nil") + } + + if doWrite { + nw, ew := dst.Write(buf[1:nr]) + if ew != nil { + err = ew + break + } + if nr != nw+1 { + err = io.ErrShortWrite + break + } + } + } + if er == io.EOF { + break + } + if er != nil { + err = er + break + } + } + return err +} + +func readStdio(conn *net.UnixConn, streams AttachStreams, receiveStdoutError, stdinDone chan error) error { + var err error + select { + case err = <-receiveStdoutError: + conn.CloseWrite() + return err + case err = <-stdinDone: + if err == ErrDetach { + conn.CloseWrite() + return err + } + if err == nil { + // copy stdin is done, close it + if connErr := conn.CloseWrite(); connErr != nil { + sylog.Errorf("Unable to close conn: %v", connErr) + } + } + if streams.AttachOutput || streams.AttachError { + return <-receiveStdoutError + } + } + return nil +} + +func CopyDetachable(dst io.Writer, src io.Reader, keys []byte) (written int64, err error) { + buf := make([]byte, 32*1024) + for { + nr, er := src.Read(buf) + if nr > 0 { + preservBuf := []byte{} + for i, key := range keys { + preservBuf = append(preservBuf, buf[0:nr]...) + if nr != 1 || buf[0] != key { + break + } + if i == len(keys)-1 { + return 0, ErrDetach + } + nr, er = src.Read(buf) + } + var nw int + var ew error + if len(preservBuf) > 0 { + nw, ew = dst.Write(preservBuf) + nr = len(preservBuf) + } else { + nw, ew = dst.Write(buf[0:nr]) + } + if nw > 0 { + written += int64(nw) + } + if ew != nil { + err = ew + break + } + if nr != nw { + err = io.ErrShortWrite + break + } + } + if er != nil { + if er != io.EOF { + err = er + } + break + } + } + return written, err +} + +func processDetachKeys(keys string) ([]byte, error) { + // Check the validity of the provided keys first + if len(keys) == 0 { + return []byte{}, nil + } + detachKeys, err := term.ToBytes(keys) + if err != nil { + return nil, fmt.Errorf("invalid detach keys: %w", err) + } + return detachKeys, nil } diff --git a/internal/app/apptainer/oci_create_linux.go b/internal/app/apptainer/oci_create_linux.go index c204e51f83..42f6684502 100644 --- a/internal/app/apptainer/oci_create_linux.go +++ b/internal/app/apptainer/oci_create_linux.go @@ -7,42 +7,223 @@ // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. +// Includes code from https://github.com/containers/podman +// Released under the Apache License Version 2.0 + package apptainer import ( + "bufio" + "context" + "encoding/json" "fmt" "os" + "os/exec" + "path" "path/filepath" "syscall" + "time" + "github.com/apptainer/apptainer/internal/pkg/buildcfg" "github.com/apptainer/apptainer/pkg/sylog" + "github.com/google/uuid" + "golang.org/x/sys/unix" ) // OciCreate creates a container from an OCI bundle func OciCreate(containerID string, args *OciArgs) error { + // chdir to bundle and lock it, so another oci create cannot use the same bundle absBundle, err := filepath.Abs(args.BundlePath) if err != nil { - return fmt.Errorf("failed to determine bundle absolute path: %s", err) + return fmt.Errorf("failed to determine bundle absolute path: %w", err) } - if err := os.Chdir(absBundle); err != nil { - return fmt.Errorf("failed to change directory to %s: %s", absBundle, err) + return fmt.Errorf("failed to change directory to %s: %w", absBundle, err) + } + if err := lockBundle(absBundle); err != nil { + return fmt.Errorf("while locking bundle: %w", err) + } + + // Create our own state location for conmon and apptainer related files + sd, err := stateDir(containerID) + if err != nil { + return fmt.Errorf("while computing state directory: %w", err) + } + err = os.MkdirAll(sd, 0o700) + if err != nil { + return fmt.Errorf("while creating state directory: %w", err) + } + containerUUID, err := uuid.NewRandom() + if err != nil { + return err } + // Pipes for sync and start communication with conmon + syncFds, err := unix.Socketpair(unix.AF_LOCAL, unix.SOCK_SEQPACKET|unix.SOCK_CLOEXEC, 0) + if err != nil { + return fmt.Errorf("could not create sync socket pair: %w", err) + } + syncChild := os.NewFile(uintptr(syncFds[0]), "sync_child") + syncParent := os.NewFile(uintptr(syncFds[1]), "sync_parent") + defer syncParent.Close() + + startFds, err := unix.Socketpair(unix.AF_LOCAL, unix.SOCK_SEQPACKET|unix.SOCK_CLOEXEC, 0) + if err != nil { + return fmt.Errorf("could not create sync socket pair: %w", err) + } + startChild := os.NewFile(uintptr(startFds[0]), "start_child") + startParent := os.NewFile(uintptr(startFds[1]), "start_parent") + defer startParent.Close() + + apptainerBin := filepath.Join(buildcfg.BINDIR, "apptainer") cmdArgs := []string{ - "--root=" + OciStateDir, - "create", - "-b", absBundle, + "--api-version", "1", + "--cid", containerID, + "--name", containerID, + "--cuuid", containerUUID.String(), + "--runtime", runc, + "--conmon-pidfile", path.Join(sd, conmonPidFile), + "--container-pidfile", path.Join(sd, containerPidFile), + "--log-path", path.Join(sd, containerLogFile), + "--runtime-arg", "--root", + "--runtime-arg", RuncStateDir, + "--runtime-arg", "--log", + "--runtime-arg", path.Join(sd, runcLogFile), + "--full-attach", + "--terminal", + "--bundle", absBundle, + "--exit-command", apptainerBin, + "--exit-command-arg", "--debug", + "--exit-command-arg", "oci", + "--exit-command-arg", "cleanup", + "--exit-command-arg", containerID, } - if args.PidFile != "" { - cmdArgs = append(cmdArgs, "--pid-file="+args.PidFile) + + cmd := exec.Command(conmon, cmdArgs...) + cmd.Dir = absBundle + cmd.Env = append(cmd.Env, fmt.Sprintf("_OCI_SYNCPIPE=%d", 3), fmt.Sprintf("_OCI_STARTPIPE=%d", 4)) + cmd.Stdin = os.Stdin + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.SysProcAttr = &syscall.SysProcAttr{ + Setpgid: true, } - cmdArgs = append(cmdArgs, containerID) + cmd.ExtraFiles = append(cmd.ExtraFiles, syncChild, startChild) - sylog.Debugf("Calling runc with args %v", cmdArgs) - if err := syscall.Exec(runc, cmdArgs, []string{}); err != nil { - return fmt.Errorf("while calling runc: %w", err) + // Run conmon and close it's end of the pipes in our parent process + sylog.Debugf("Starting conmon with args %v", cmdArgs) + if err := cmd.Start(); err != nil { + if err2 := releaseBundle(absBundle); err2 != nil { + sylog.Errorf("while releasing bundle: %w", containerID) + } + return fmt.Errorf("while starting conmon: %w", err) } + syncChild.Close() + startChild.Close() + // No other setup at present... just signal conmon to start work + writeConmonPipeData(startParent) + // After conmon receives from start pipe it should start container and exit + // without error. + err = cmd.Wait() + if err != nil { + if err2 := releaseBundle(absBundle); err2 != nil { + sylog.Errorf("while releasing bundle: %w", containerID) + } + return fmt.Errorf("while starting conmon: %w", err) + } + + // We check for errors from runc (which conmon invokes) via the sync pipe + pid, err := readConmonPipeData(syncParent, path.Join(sd, runcLogFile)) + if err != nil { + if err2 := OciDelete(context.TODO(), containerID); err2 != nil { + sylog.Errorf("Removing container %s from runtime after creation failed", containerID) + } + return err + } + + // Create a symlink from the state dir to the bundle, so it's easy to find later on. + bundleLink := path.Join(sd, "bundle") + if err := os.Symlink(absBundle, bundleLink); err != nil { + return fmt.Errorf("could not link attach socket: %w", err) + } + + sylog.Infof("Container %s created with PID %d", containerID, pid) return nil } + +// The following utility functions are taken from https://github.com/containers/podman +// Released under the Apache License Version 2.0 + +func readConmonPipeData(pipe *os.File, ociLog string) (int, error) { + // syncInfo is used to return data from monitor process to daemon + type syncInfo struct { + Data int `json:"data"` + Message string `json:"message,omitempty"` + } + + // Wait to get container pid from conmon + type syncStruct struct { + si *syncInfo + err error + } + ch := make(chan syncStruct) + go func() { + var si *syncInfo + rdr := bufio.NewReader(pipe) + b, err := rdr.ReadBytes('\n') + if err != nil { + ch <- syncStruct{err: err} + } + if err := json.Unmarshal(b, &si); err != nil { + ch <- syncStruct{err: err} + return + } + ch <- syncStruct{si: si} + }() + + data := -1 + select { + case ss := <-ch: + if ss.err != nil { + if ociLog != "" { + ociLogData, err := os.ReadFile(ociLog) + if err == nil { + var ociErr ociError + if err := json.Unmarshal(ociLogData, &ociErr); err == nil { + return -1, fmt.Errorf("runc error: %s", ociErr.Msg) + } + } + } + return -1, fmt.Errorf("container create failed (no logs from conmon): %w", ss.err) + } + sylog.Debugf("Received: %d", ss.si.Data) + if ss.si.Data < 0 { + if ociLog != "" { + ociLogData, err := os.ReadFile(ociLog) + if err == nil { + var ociErr ociError + if err := json.Unmarshal(ociLogData, &ociErr); err == nil { + return ss.si.Data, fmt.Errorf("runc error: %s", ociErr.Msg) + } + } + } + // If we failed to parse the JSON errors, then print the output as it is + if ss.si.Message != "" { + return ss.si.Data, fmt.Errorf("runc error: %s", ss.si.Message) + } + return ss.si.Data, fmt.Errorf("container creation failed") + } + data = ss.si.Data + case <-time.After(createTimeout): + return -1, fmt.Errorf("container creation timeout") + } + return data, nil +} + +// writeConmonPipeData writes nonce data to a pipe +func writeConmonPipeData(pipe *os.File) error { + someData := []byte{0} + _, err := pipe.Write(someData) + return err +} diff --git a/internal/app/apptainer/oci_delete_linux.go b/internal/app/apptainer/oci_delete_linux.go index 47f8ff8b92..59e04d3c12 100644 --- a/internal/app/apptainer/oci_delete_linux.go +++ b/internal/app/apptainer/oci_delete_linux.go @@ -12,7 +12,9 @@ package apptainer import ( "context" "fmt" - "syscall" + "os" + "os/exec" + "path/filepath" "github.com/apptainer/apptainer/pkg/sylog" ) @@ -20,15 +22,37 @@ import ( // OciDelete deletes container resources func OciDelete(ctx context.Context, containerID string) error { runcArgs := []string{ - "--root=" + OciStateDir, + "--root", RuncStateDir, "delete", containerID, } + cmd := exec.Command(runc, runcArgs...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = os.Stdout sylog.Debugf("Calling runc with args %v", runcArgs) - if err := syscall.Exec(runc, runcArgs, []string{}); err != nil { - return fmt.Errorf("while calling runc: %w", err) + err := cmd.Run() + if err != nil { + return fmt.Errorf("while calling runc delete: %w", err) } - return nil + sd, err := stateDir(containerID) + if err != nil { + return fmt.Errorf("while computing state directory: %w", err) + } + + bLink := filepath.Join(sd, bundleLink) + bundle, err := filepath.EvalSymlinks(bLink) + if err != nil { + return fmt.Errorf("while finding bundle directory: %w", err) + } + + sylog.Debugf("Removing bundle symlink") + if err := os.Remove(bLink); err != nil { + return fmt.Errorf("while removing bundle symlink: %w", err) + } + + sylog.Debugf("Releasing bundle lock") + return releaseBundle(bundle) } diff --git a/internal/app/apptainer/oci_exec_linux.go b/internal/app/apptainer/oci_exec_linux.go index 8886a9bb91..16cb6fd401 100644 --- a/internal/app/apptainer/oci_exec_linux.go +++ b/internal/app/apptainer/oci_exec_linux.go @@ -10,8 +10,8 @@ package apptainer import ( - "fmt" - "syscall" + "os" + "os/exec" "github.com/apptainer/apptainer/pkg/sylog" ) @@ -19,16 +19,15 @@ import ( // OciExec executes a command in a container func OciExec(containerID string, cmdArgs []string) error { runcArgs := []string{ - "--root=" + OciStateDir, + "--root", RuncStateDir, "exec", containerID, } runcArgs = append(runcArgs, cmdArgs...) - + cmd := exec.Command(runc, runcArgs...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = os.Stdout sylog.Debugf("Calling runc with args %v", runcArgs) - if err := syscall.Exec(runc, runcArgs, []string{}); err != nil { - return fmt.Errorf("while calling runc: %w", err) - } - return nil } diff --git a/internal/app/apptainer/oci_kill_linux.go b/internal/app/apptainer/oci_kill_linux.go index daafb9b419..b0a63a97f0 100644 --- a/internal/app/apptainer/oci_kill_linux.go +++ b/internal/app/apptainer/oci_kill_linux.go @@ -10,8 +10,8 @@ package apptainer import ( - "fmt" - "syscall" + "os" + "os/exec" "github.com/apptainer/apptainer/pkg/sylog" ) @@ -19,16 +19,16 @@ import ( // OciKill kills container process func OciKill(containerID string, killSignal string) error { runcArgs := []string{ - "--root=" + OciStateDir, + "--root", RuncStateDir, "kill", containerID, killSignal, } + cmd := exec.Command(runc, runcArgs...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = os.Stdout sylog.Debugf("Calling runc with args %v", runcArgs) - if err := syscall.Exec(runc, runcArgs, []string{}); err != nil { - return fmt.Errorf("while calling runc: %w", err) - } - - return nil + return cmd.Run() } diff --git a/internal/app/apptainer/oci_linux.go b/internal/app/apptainer/oci_linux.go index 3a83ca10a6..562c404580 100644 --- a/internal/app/apptainer/oci_linux.go +++ b/internal/app/apptainer/oci_linux.go @@ -2,16 +2,52 @@ // Apptainer a Series of LF Projects LLC. // For website terms of use, trademark policy, privacy policy and other // project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2020, Sylabs Inc. All rights reserved. +// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. +// Includes code from https://github.com/containers/podman +// Released under the Apache License Version 2.0 + package apptainer +import ( + "bufio" + "fmt" + "io" + "os" + "path" + "path/filepath" + "time" + + "github.com/apptainer/apptainer/internal/pkg/util/fs" + "github.com/apptainer/apptainer/internal/pkg/util/user" + "github.com/apptainer/apptainer/pkg/syfs" + "github.com/apptainer/apptainer/pkg/util/fs/lock" + securejoin "github.com/cyphar/filepath-securejoin" +) + const ( - OciStateDir = "/run/apptainer-oci" - runc = "/usr/bin/runc" + // Absolute path for runc + runc = "/usr/bin/runc" + // Absolute path for conmon + conmon = "/usr/bin/conmon" + // Absolute path for the runc state + RuncStateDir = "/run/apptainer-oci" + // Relative path inside ~/.apptainer for conmon and apptainer state + ociPath = "oci" + // State directory files + containerPidFile = "container.pid" + containerLogFile = "container.log" + runcLogFile = "runc.log" + conmonPidFile = "conmon.pid" + bundleLink = "bundle" + // Files in the OCI bundle root + bundleLock = ".apptainer-oci.lock" + attachSocket = "attach" + // Timeouts + createTimeout = 30 * time.Second ) // OciArgs contains CLI arguments @@ -26,3 +62,110 @@ type OciArgs struct { EmptyProcess bool ForceKill bool } + +// AttachStreams contains streams that will be attached to the container +type AttachStreams struct { + // OutputStream will be attached to container's STDOUT + OutputStream io.WriteCloser + // ErrorStream will be attached to container's STDERR + ErrorStream io.WriteCloser + // InputStream will be attached to container's STDIN + InputStream *bufio.Reader + // AttachOutput is whether to attach to STDOUT + // If false, stdout will not be attached + AttachOutput bool + // AttachError is whether to attach to STDERR + // If false, stdout will not be attached + AttachError bool + // AttachInput is whether to attach to STDIN + // If false, stdout will not be attached + AttachInput bool +} + +/* Sync with stdpipe_t in conmon.c */ +const ( + AttachPipeStdin = 1 + AttachPipeStdout = 2 + AttachPipeStderr = 3 +) + +type ociError struct { + Level string `json:"level,omitempty"` + Time string `json:"time,omitempty"` + Msg string `json:"msg,omitempty"` +} + +// stateDir returns the path to container state handled by conmon/apptainer +// (as opposed to runc's state in RuncStateDir) +func stateDir(containerID string) (string, error) { + hostname, err := os.Hostname() + if err != nil { + return "", err + } + + u, err := user.CurrentOriginal() + if err != nil { + return "", err + } + + configDir, err := syfs.ConfigDirForUsername(u.Name) + if err != nil { + return "", err + } + + rootPath := filepath.Join(configDir, ociPath) + containerPath := filepath.Join(hostname, containerID) + path, err := securejoin.SecureJoin(rootPath, containerPath) + if err != nil { + return "", err + } + return path, err +} + +// lockBundle creates a lock file in a bundle directory +func lockBundle(bundlePath string) error { + bl := path.Join(bundlePath, bundleLock) + _, err := os.Stat(bl) + if err == nil { + return fmt.Errorf("bundle is locked by another process") + } + if !os.IsNotExist(err) { + return fmt.Errorf("while stat-ing lock file: %w", err) + } + + fd, err := lock.Exclusive(bundlePath) + if err != nil { + return fmt.Errorf("while acquiring directory lock: %w", err) + } + defer lock.Release(fd) + + err = fs.EnsureFileWithPermission(bl, 0o600) + if err != nil { + return fmt.Errorf("while creating lock file: %w", err) + } + return nil +} + +// releaseBundle removes a lock file in a bundle directory +func releaseBundle(bundlePath string) error { + bl := path.Join(bundlePath, bundleLock) + _, err := os.Stat(bl) + if os.IsNotExist(err) { + return fmt.Errorf("bundle is not locked") + } + if err != nil { + return fmt.Errorf("while stat-ing lock file: %w", err) + } + + fd, err := lock.Exclusive(bundlePath) + if err != nil { + return fmt.Errorf("while acquiring directory lock: %w", err) + } + defer lock.Release(fd) + + err = os.Remove(bl) + if err != nil { + return fmt.Errorf("while removing lock file: %w", err) + } + return nil +} diff --git a/internal/app/apptainer/oci_pause_linux.go b/internal/app/apptainer/oci_pause_linux.go index 3358b2f255..d00f509823 100644 --- a/internal/app/apptainer/oci_pause_linux.go +++ b/internal/app/apptainer/oci_pause_linux.go @@ -10,8 +10,8 @@ package apptainer import ( - "fmt" - "syscall" + "os" + "os/exec" "github.com/apptainer/apptainer/pkg/sylog" ) @@ -19,31 +19,31 @@ import ( // OciPause pauses processes in a container func OciPause(containerID string) error { runcArgs := []string{ - "--root=" + OciStateDir, + "--root", RuncStateDir, "pause", containerID, } + cmd := exec.Command(runc, runcArgs...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = os.Stdout sylog.Debugf("Calling runc with args %v", runcArgs) - if err := syscall.Exec(runc, runcArgs, []string{}); err != nil { - return fmt.Errorf("while calling runc: %w", err) - } - - return nil + return cmd.Run() } // OciResume pauses processes in a container func OciResume(containerID string) error { runcArgs := []string{ - "--root=" + OciStateDir, + "--root", RuncStateDir, "resume", containerID, } + cmd := exec.Command(runc, runcArgs...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = os.Stdout sylog.Debugf("Calling runc with args %v", runcArgs) - if err := syscall.Exec(runc, runcArgs, []string{}); err != nil { - return fmt.Errorf("while calling runc: %w", err) - } - - return nil + return cmd.Run() } diff --git a/internal/app/apptainer/oci_run_linux.go b/internal/app/apptainer/oci_run_linux.go index 593b61b839..2fd43ca902 100644 --- a/internal/app/apptainer/oci_run_linux.go +++ b/internal/app/apptainer/oci_run_linux.go @@ -13,8 +13,8 @@ import ( "context" "fmt" "os" + "os/exec" "path/filepath" - "syscall" "github.com/apptainer/apptainer/pkg/sylog" ) @@ -31,19 +31,18 @@ func OciRun(ctx context.Context, containerID string, args *OciArgs) error { } runcArgs := []string{ - "--root=" + OciStateDir, - "create", + "--root", RuncStateDir, + "run", "-b", absBundle, } if args.PidFile != "" { runcArgs = append(runcArgs, "--pid-file="+args.PidFile) } runcArgs = append(runcArgs, containerID) - + cmd := exec.Command(runc, runcArgs...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = os.Stdout sylog.Debugf("Calling runc with args %v", runcArgs) - if err := syscall.Exec(runc, runcArgs, []string{}); err != nil { - return fmt.Errorf("while calling runc: %w", err) - } - - return nil + return cmd.Run() } diff --git a/internal/app/apptainer/oci_start_linux.go b/internal/app/apptainer/oci_start_linux.go index 1d8b9d62d4..c39e0872f7 100644 --- a/internal/app/apptainer/oci_start_linux.go +++ b/internal/app/apptainer/oci_start_linux.go @@ -10,8 +10,8 @@ package apptainer import ( - "fmt" - "syscall" + "os" + "os/exec" "github.com/apptainer/apptainer/pkg/sylog" ) @@ -19,15 +19,15 @@ import ( // OciStart starts a previously create container func OciStart(containerID string) error { runcArgs := []string{ - "--root=" + OciStateDir, + "--root", RuncStateDir, "start", containerID, } + cmd := exec.Command(runc, runcArgs...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = os.Stdout sylog.Debugf("Calling runc with args %v", runcArgs) - if err := syscall.Exec(runc, runcArgs, []string{}); err != nil { - return fmt.Errorf("while calling runc: %w", err) - } - - return nil + return cmd.Run() } diff --git a/internal/app/apptainer/oci_state_linux.go b/internal/app/apptainer/oci_state_linux.go index f2483a398d..49e4da9306 100644 --- a/internal/app/apptainer/oci_state_linux.go +++ b/internal/app/apptainer/oci_state_linux.go @@ -10,8 +10,8 @@ package apptainer import ( - "fmt" - "syscall" + "os" + "os/exec" "github.com/apptainer/apptainer/pkg/sylog" ) @@ -19,15 +19,15 @@ import ( // OciState query container state func OciState(containerID string, args *OciArgs) error { runcArgs := []string{ - "--root=" + OciStateDir, + "--root", RuncStateDir, "state", containerID, } + cmd := exec.Command(runc, runcArgs...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = os.Stdout sylog.Debugf("Calling runc with args %v", runcArgs) - if err := syscall.Exec(runc, runcArgs, []string{}); err != nil { - return fmt.Errorf("while calling runc: %w", err) - } - - return nil + return cmd.Run() } diff --git a/internal/app/apptainer/oci_update_linux.go b/internal/app/apptainer/oci_update_linux.go index b175d0bd66..1fb95f9f92 100644 --- a/internal/app/apptainer/oci_update_linux.go +++ b/internal/app/apptainer/oci_update_linux.go @@ -10,8 +10,8 @@ package apptainer import ( - "fmt" - "syscall" + "os" + "os/exec" "github.com/apptainer/apptainer/pkg/sylog" ) @@ -19,16 +19,16 @@ import ( // OciUpdate updates container cgroups resources func OciUpdate(containerID string, args *OciArgs) error { runcArgs := []string{ - "--root=" + OciStateDir, + "--root", RuncStateDir, "update", "-r", args.FromFile, containerID, } + cmd := exec.Command(runc, runcArgs...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = os.Stdout sylog.Debugf("Calling runc with args %v", runcArgs) - if err := syscall.Exec(runc, runcArgs, []string{}); err != nil { - return fmt.Errorf("while calling runc: %w", err) - } - - return nil + return cmd.Run() } From fed0ed9ef73aff2ca8bb052d4d31aef189737865 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Mon, 28 Feb 2022 14:00:44 -0600 Subject: [PATCH 008/114] oci: use FindBin for conmon, runc Signed-off-by: Edita Kizinevic --- internal/app/apptainer/oci_attach_linux.go | 4 ++-- internal/app/apptainer/oci_create_linux.go | 13 +++++++++++-- internal/app/apptainer/oci_delete_linux.go | 7 ++++++- internal/app/apptainer/oci_exec_linux.go | 5 +++++ internal/app/apptainer/oci_kill_linux.go | 5 +++++ internal/app/apptainer/oci_linux.go | 4 ---- internal/app/apptainer/oci_pause_linux.go | 9 +++++++++ internal/app/apptainer/oci_run_linux.go | 5 +++++ internal/app/apptainer/oci_start_linux.go | 5 +++++ internal/app/apptainer/oci_state_linux.go | 5 +++++ internal/app/apptainer/oci_update_linux.go | 5 +++++ internal/pkg/util/bin/bin.go | 4 +++- 12 files changed, 61 insertions(+), 10 deletions(-) diff --git a/internal/app/apptainer/oci_attach_linux.go b/internal/app/apptainer/oci_attach_linux.go index 47fead0bf2..e2d43b658c 100644 --- a/internal/app/apptainer/oci_attach_linux.go +++ b/internal/app/apptainer/oci_attach_linux.go @@ -48,11 +48,11 @@ func OciAttach(ctx context.Context, containerID string) error { attachSock := filepath.Join(sd, bundleLink, attachSocket) conn, err := openUnixSocket(attachSock) if err != nil { - return errors.Wrapf(err, "while connecting to attach socket: %w", attachSock) + return fmt.Errorf("while connecting to attach socket: %w", err) } defer func() { if err := conn.Close(); err != nil { - sylog.Errorf("while closing attach socket: %w", err) + sylog.Errorf("while closing attach socket: %v", err) } }() diff --git a/internal/app/apptainer/oci_create_linux.go b/internal/app/apptainer/oci_create_linux.go index 42f6684502..3c781c21ab 100644 --- a/internal/app/apptainer/oci_create_linux.go +++ b/internal/app/apptainer/oci_create_linux.go @@ -25,6 +25,7 @@ import ( "time" "github.com/apptainer/apptainer/internal/pkg/buildcfg" + "github.com/apptainer/apptainer/internal/pkg/util/bin" "github.com/apptainer/apptainer/pkg/sylog" "github.com/google/uuid" "golang.org/x/sys/unix" @@ -32,6 +33,14 @@ import ( // OciCreate creates a container from an OCI bundle func OciCreate(containerID string, args *OciArgs) error { + conmon, err := bin.FindBin("conmon") + if err != nil { + return err + } + runc, err := bin.FindBin("runc") + if err != nil { + return err + } // chdir to bundle and lock it, so another oci create cannot use the same bundle absBundle, err := filepath.Abs(args.BundlePath) if err != nil { @@ -114,7 +123,7 @@ func OciCreate(containerID string, args *OciArgs) error { sylog.Debugf("Starting conmon with args %v", cmdArgs) if err := cmd.Start(); err != nil { if err2 := releaseBundle(absBundle); err2 != nil { - sylog.Errorf("while releasing bundle: %w", containerID) + sylog.Errorf("while releasing bundle: %v", err) } return fmt.Errorf("while starting conmon: %w", err) } @@ -128,7 +137,7 @@ func OciCreate(containerID string, args *OciArgs) error { err = cmd.Wait() if err != nil { if err2 := releaseBundle(absBundle); err2 != nil { - sylog.Errorf("while releasing bundle: %w", containerID) + sylog.Errorf("while releasing bundle: %v", err) } return fmt.Errorf("while starting conmon: %w", err) } diff --git a/internal/app/apptainer/oci_delete_linux.go b/internal/app/apptainer/oci_delete_linux.go index 59e04d3c12..f5033635e8 100644 --- a/internal/app/apptainer/oci_delete_linux.go +++ b/internal/app/apptainer/oci_delete_linux.go @@ -16,11 +16,16 @@ import ( "os/exec" "path/filepath" + "github.com/apptainer/apptainer/internal/pkg/util/bin" "github.com/apptainer/apptainer/pkg/sylog" ) // OciDelete deletes container resources func OciDelete(ctx context.Context, containerID string) error { + runc, err := bin.FindBin("runc") + if err != nil { + return err + } runcArgs := []string{ "--root", RuncStateDir, "delete", @@ -32,7 +37,7 @@ func OciDelete(ctx context.Context, containerID string) error { cmd.Stderr = os.Stderr cmd.Stdin = os.Stdout sylog.Debugf("Calling runc with args %v", runcArgs) - err := cmd.Run() + err = cmd.Run() if err != nil { return fmt.Errorf("while calling runc delete: %w", err) } diff --git a/internal/app/apptainer/oci_exec_linux.go b/internal/app/apptainer/oci_exec_linux.go index 16cb6fd401..d1c8ab4fd2 100644 --- a/internal/app/apptainer/oci_exec_linux.go +++ b/internal/app/apptainer/oci_exec_linux.go @@ -13,11 +13,16 @@ import ( "os" "os/exec" + "github.com/apptainer/apptainer/internal/pkg/util/bin" "github.com/apptainer/apptainer/pkg/sylog" ) // OciExec executes a command in a container func OciExec(containerID string, cmdArgs []string) error { + runc, err := bin.FindBin("runc") + if err != nil { + return err + } runcArgs := []string{ "--root", RuncStateDir, "exec", diff --git a/internal/app/apptainer/oci_kill_linux.go b/internal/app/apptainer/oci_kill_linux.go index b0a63a97f0..0981e3b8d1 100644 --- a/internal/app/apptainer/oci_kill_linux.go +++ b/internal/app/apptainer/oci_kill_linux.go @@ -13,11 +13,16 @@ import ( "os" "os/exec" + "github.com/apptainer/apptainer/internal/pkg/util/bin" "github.com/apptainer/apptainer/pkg/sylog" ) // OciKill kills container process func OciKill(containerID string, killSignal string) error { + runc, err := bin.FindBin("runc") + if err != nil { + return err + } runcArgs := []string{ "--root", RuncStateDir, "kill", diff --git a/internal/app/apptainer/oci_linux.go b/internal/app/apptainer/oci_linux.go index 562c404580..0159344905 100644 --- a/internal/app/apptainer/oci_linux.go +++ b/internal/app/apptainer/oci_linux.go @@ -29,10 +29,6 @@ import ( ) const ( - // Absolute path for runc - runc = "/usr/bin/runc" - // Absolute path for conmon - conmon = "/usr/bin/conmon" // Absolute path for the runc state RuncStateDir = "/run/apptainer-oci" // Relative path inside ~/.apptainer for conmon and apptainer state diff --git a/internal/app/apptainer/oci_pause_linux.go b/internal/app/apptainer/oci_pause_linux.go index d00f509823..65a816fbc5 100644 --- a/internal/app/apptainer/oci_pause_linux.go +++ b/internal/app/apptainer/oci_pause_linux.go @@ -13,11 +13,16 @@ import ( "os" "os/exec" + "github.com/apptainer/apptainer/internal/pkg/util/bin" "github.com/apptainer/apptainer/pkg/sylog" ) // OciPause pauses processes in a container func OciPause(containerID string) error { + runc, err := bin.FindBin("runc") + if err != nil { + return err + } runcArgs := []string{ "--root", RuncStateDir, "pause", @@ -34,6 +39,10 @@ func OciPause(containerID string) error { // OciResume pauses processes in a container func OciResume(containerID string) error { + runc, err := bin.FindBin("runc") + if err != nil { + return err + } runcArgs := []string{ "--root", RuncStateDir, "resume", diff --git a/internal/app/apptainer/oci_run_linux.go b/internal/app/apptainer/oci_run_linux.go index 2fd43ca902..2337228a5f 100644 --- a/internal/app/apptainer/oci_run_linux.go +++ b/internal/app/apptainer/oci_run_linux.go @@ -16,11 +16,16 @@ import ( "os/exec" "path/filepath" + "github.com/apptainer/apptainer/internal/pkg/util/bin" "github.com/apptainer/apptainer/pkg/sylog" ) // OciRun runs a container (equivalent to create/start/delete) func OciRun(ctx context.Context, containerID string, args *OciArgs) error { + runc, err := bin.FindBin("runc") + if err != nil { + return err + } absBundle, err := filepath.Abs(args.BundlePath) if err != nil { return fmt.Errorf("failed to determine bundle absolute path: %s", err) diff --git a/internal/app/apptainer/oci_start_linux.go b/internal/app/apptainer/oci_start_linux.go index c39e0872f7..275a2e65d9 100644 --- a/internal/app/apptainer/oci_start_linux.go +++ b/internal/app/apptainer/oci_start_linux.go @@ -13,11 +13,16 @@ import ( "os" "os/exec" + "github.com/apptainer/apptainer/internal/pkg/util/bin" "github.com/apptainer/apptainer/pkg/sylog" ) // OciStart starts a previously create container func OciStart(containerID string) error { + runc, err := bin.FindBin("runc") + if err != nil { + return err + } runcArgs := []string{ "--root", RuncStateDir, "start", diff --git a/internal/app/apptainer/oci_state_linux.go b/internal/app/apptainer/oci_state_linux.go index 49e4da9306..b229a66029 100644 --- a/internal/app/apptainer/oci_state_linux.go +++ b/internal/app/apptainer/oci_state_linux.go @@ -13,11 +13,16 @@ import ( "os" "os/exec" + "github.com/apptainer/apptainer/internal/pkg/util/bin" "github.com/apptainer/apptainer/pkg/sylog" ) // OciState query container state func OciState(containerID string, args *OciArgs) error { + runc, err := bin.FindBin("runc") + if err != nil { + return err + } runcArgs := []string{ "--root", RuncStateDir, "state", diff --git a/internal/app/apptainer/oci_update_linux.go b/internal/app/apptainer/oci_update_linux.go index 1fb95f9f92..811d43cd23 100644 --- a/internal/app/apptainer/oci_update_linux.go +++ b/internal/app/apptainer/oci_update_linux.go @@ -13,11 +13,16 @@ import ( "os" "os/exec" + "github.com/apptainer/apptainer/internal/pkg/util/bin" "github.com/apptainer/apptainer/pkg/sylog" ) // OciUpdate updates container cgroups resources func OciUpdate(containerID string, args *OciArgs) error { + runc, err := bin.FindBin("runc") + if err != nil { + return err + } runcArgs := []string{ "--root", RuncStateDir, "update", diff --git a/internal/pkg/util/bin/bin.go b/internal/pkg/util/bin/bin.go index d11283c963..35554163b2 100644 --- a/internal/pkg/util/bin/bin.go +++ b/internal/pkg/util/bin/bin.go @@ -47,7 +47,8 @@ func FindBin(name string) (path string, err error) { return findOnPath(name, true) // All other executables // We will always search the user's PATH first for these - case "curl", + case "conmon", + "curl", "debootstrap", "dnf", "fakeroot", @@ -63,6 +64,7 @@ func FindBin(name string) (path string, err error) { "pacstrap", "rpm", "rpmkeys", + "runc", "squashfuse", "squashfuse_ll", "SUSEConnect", From 0cd88c27ab9c52cd3d45f47ca46c185a95f90a7d Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Mon, 28 Feb 2022 14:10:05 -0600 Subject: [PATCH 009/114] oci: Update LICENSE docs and copyright lines Signed-off-by: Edita Kizinevic --- LICENSE_DEPENDENCIES.md | 16 +- LICENSE_THIRD_PARTY.md | 214 +++++++++++++++++++++ internal/app/apptainer/oci_delete_linux.go | 2 +- internal/app/apptainer/oci_exec_linux.go | 2 +- internal/app/apptainer/oci_kill_linux.go | 2 +- 5 files changed, 231 insertions(+), 5 deletions(-) diff --git a/LICENSE_DEPENDENCIES.md b/LICENSE_DEPENDENCIES.md index e698e11f14..5e6581f94b 100644 --- a/LICENSE_DEPENDENCIES.md +++ b/LICENSE_DEPENDENCIES.md @@ -35,6 +35,12 @@ The dependencies and their licenses are as follows: **License URL:** +## github.com/containers/common + +**License:** Apache-2.0 + +**License URL:** + ## github.com/containers/image/v5 **License:** Apache-2.0 @@ -53,11 +59,11 @@ The dependencies and their licenses are as follows: **License URL:** -## github.com/containers/storage/pkg +## github.com/containers/storage **License:** Apache-2.0 -**License URL:** +**License URL:** ## github.com/coreos/go-iptables/iptables @@ -287,6 +293,12 @@ The dependencies and their licenses are as follows: **License URL:** +## github.com/opencontainers/selinux + +**License:** Apache-2.0 + +**License URL:** + ## github.com/opencontainers/umoci **License:** Apache-2.0 diff --git a/LICENSE_THIRD_PARTY.md b/LICENSE_THIRD_PARTY.md index a2374b0469..052d722b85 100644 --- a/LICENSE_THIRD_PARTY.md +++ b/LICENSE_THIRD_PARTY.md @@ -288,3 +288,217 @@ The source files: * `internal/app/apptainer/instance_linux.go` Contain code from the docker cli project, under the Apache License, Version 2.0. + +## github.com/containers/podman + +The source files: + +* `internal/app/apptainer/oci_linux.go` +* `internal/app/apptainer/oci_attach_linux.go` +* `internal/app/apptainer/oci_create_linux.go` + +Contain code from the podman project, under the Apache License, Version 2.0. + +```text + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +``` diff --git a/internal/app/apptainer/oci_delete_linux.go b/internal/app/apptainer/oci_delete_linux.go index f5033635e8..c2648449c4 100644 --- a/internal/app/apptainer/oci_delete_linux.go +++ b/internal/app/apptainer/oci_delete_linux.go @@ -2,7 +2,7 @@ // Apptainer a Series of LF Projects LLC. // For website terms of use, trademark policy, privacy policy and other // project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2020, Sylabs Inc. All rights reserved. +// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. diff --git a/internal/app/apptainer/oci_exec_linux.go b/internal/app/apptainer/oci_exec_linux.go index d1c8ab4fd2..a6dd62ba7b 100644 --- a/internal/app/apptainer/oci_exec_linux.go +++ b/internal/app/apptainer/oci_exec_linux.go @@ -2,7 +2,7 @@ // Apptainer a Series of LF Projects LLC. // For website terms of use, trademark policy, privacy policy and other // project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2019, Sylabs Inc. All rights reserved. +// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. diff --git a/internal/app/apptainer/oci_kill_linux.go b/internal/app/apptainer/oci_kill_linux.go index 0981e3b8d1..2d12aee6f2 100644 --- a/internal/app/apptainer/oci_kill_linux.go +++ b/internal/app/apptainer/oci_kill_linux.go @@ -2,7 +2,7 @@ // Apptainer a Series of LF Projects LLC. // For website terms of use, trademark policy, privacy policy and other // project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2019, Sylabs Inc. All rights reserved. +// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. From 3df5cf66bd20f3e07ded294d3bf5678e1af3180f Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Mon, 28 Feb 2022 14:19:12 -0600 Subject: [PATCH 010/114] e2e: adapt oci help text Signed-off-by: Edita Kizinevic --- e2e/testdata/help/help-oci-create.txt | 18 +++++++----------- e2e/testdata/help/help-oci-kill.txt | 7 +++---- e2e/testdata/help/help-oci-run.txt | 16 +++++++--------- e2e/testdata/help/help-oci-state.txt | 4 +--- 4 files changed, 18 insertions(+), 27 deletions(-) diff --git a/e2e/testdata/help/help-oci-create.txt b/e2e/testdata/help/help-oci-create.txt index f71d62c5c7..682843d962 100644 --- a/e2e/testdata/help/help-oci-create.txt +++ b/e2e/testdata/help/help-oci-create.txt @@ -8,17 +8,13 @@ Description: bundle directory Options: - -b, --bundle string specify the OCI bundle path (required) - --empty-process run container without executing container - process (eg: for POD container) - -h, --help help for create - --log-format string specify the log file format. Available - formats are basic, kubernetes and json - (default "kubernetes") - -l, --log-path string specify the log file path - --pid-file string specify the pid file - -s, --sync-socket string specify the path to unix socket for state - synchronization + -b, --bundle string specify the OCI bundle path (required) + -h, --help help for create + --log-format string specify the log file format. Available formats + are basic, kubernetes and json (default + "kubernetes") + -l, --log-path string specify the log file path + --pid-file string specify the pid file Examples: diff --git a/e2e/testdata/help/help-oci-kill.txt b/e2e/testdata/help/help-oci-kill.txt index 1cab71a963..9eab494f27 100644 --- a/e2e/testdata/help/help-oci-kill.txt +++ b/e2e/testdata/help/help-oci-kill.txt @@ -8,10 +8,9 @@ Description: identified by container ID. Options: - -f, --force kill container process with SIGKILL - -h, --help help for kill - -s, --signal string signal sent to the container (default "SIGTERM") - -t, --timeout uint32 timeout in second before killing container + -f, --force kill container process with SIGKILL + -h, --help help for kill + -s, --signal string signal sent to the container (default "SIGTERM") Examples: diff --git a/e2e/testdata/help/help-oci-run.txt b/e2e/testdata/help/help-oci-run.txt index 6a095ea8d9..062e5c75e5 100644 --- a/e2e/testdata/help/help-oci-run.txt +++ b/e2e/testdata/help/help-oci-run.txt @@ -7,15 +7,13 @@ Description: Run will invoke equivalent of create/start/attach/delete commands in a row. Options: - -b, --bundle string specify the OCI bundle path (required) - -h, --help help for run - --log-format string specify the log file format. Available - formats are basic, kubernetes and json - (default "kubernetes") - -l, --log-path string specify the log file path - --pid-file string specify the pid file - -s, --sync-socket string specify the path to unix socket for state - synchronization + -b, --bundle string specify the OCI bundle path (required) + -h, --help help for run + --log-format string specify the log file format. Available formats + are basic, kubernetes and json (default + "kubernetes") + -l, --log-path string specify the log file path + --pid-file string specify the pid file Examples: diff --git a/e2e/testdata/help/help-oci-state.txt b/e2e/testdata/help/help-oci-state.txt index 7f7807d137..19ca7590c6 100644 --- a/e2e/testdata/help/help-oci-state.txt +++ b/e2e/testdata/help/help-oci-state.txt @@ -8,9 +8,7 @@ Description: container identified by container ID. Options: - -h, --help help for state - -s, --sync-socket string specify the path to unix socket for state - synchronization + -h, --help help for state Examples: From 5eb1db000885ce17d4fd2637967e433ddfca868e Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Wed, 9 Mar 2022 09:20:15 -0600 Subject: [PATCH 011/114] oci: fix: ensure `oci exec` calls runc and e2e test checks this As noted in review, the `runc` call for `oci exec` was not actually being made. E2E test for `oci exec` was only checking return code, not for output so it passed. Fix these issues. In `e2e/suite.go` also move registry creation so that it is only performed for the man tests in PID+mount NS. It is not needed for the OCI/CGROUPS e2e tests and wastes CI time. Signed-off-by: Edita Kizinevic --- e2e/oci/oci.go | 5 +++-- e2e/suite.go | 1 - internal/app/apptainer/oci_exec_linux.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/e2e/oci/oci.go b/e2e/oci/oci.go index bb4fdfe8ba..bed8aa84de 100644 --- a/e2e/oci/oci.go +++ b/e2e/oci/oci.go @@ -300,8 +300,9 @@ func (c ctx) testOciBasic(t *testing.T) { t, e2e.WithProfile(e2e.RootProfile), e2e.WithCommand("oci exec"), - e2e.WithArgs(containerID, "id"), - e2e.ExpectExit(0), + e2e.WithArgs(containerID, "hostname"), + e2e.ExpectExit(0, + e2e.ExpectOutput(e2e.ContainMatch, "apptainer")), ) c.env.RunApptainer( diff --git a/e2e/suite.go b/e2e/suite.go index 0f14e17541..a4b657c07c 100644 --- a/e2e/suite.go +++ b/e2e/suite.go @@ -196,7 +196,6 @@ func Run(t *testing.T) { }) suite := testhelper.NewSuite(t, testenv) - suite.AddGroup("ACTIONS", actions.E2ETests) suite.AddGroup("BUILDCFG", e2ebuildcfg.E2ETests) suite.AddGroup("BUILD", imgbuild.E2ETests) diff --git a/internal/app/apptainer/oci_exec_linux.go b/internal/app/apptainer/oci_exec_linux.go index a6dd62ba7b..67c8ab5f88 100644 --- a/internal/app/apptainer/oci_exec_linux.go +++ b/internal/app/apptainer/oci_exec_linux.go @@ -34,5 +34,5 @@ func OciExec(containerID string, cmdArgs []string) error { cmd.Stderr = os.Stderr cmd.Stdin = os.Stdout sylog.Debugf("Calling runc with args %v", runcArgs) - return nil + return cmd.Run() } From 7952c83943a82af21f8d81690d72a7e8a56a9395 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Wed, 9 Mar 2022 09:23:36 -0600 Subject: [PATCH 012/114] oci: simplify AttachStreams type As suggested in review the streams can be straight `io.Writer` / `io.Reader`. Signed-off-by: Edita Kizinevic --- internal/app/apptainer/oci_linux.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/internal/app/apptainer/oci_linux.go b/internal/app/apptainer/oci_linux.go index 0159344905..04d981964d 100644 --- a/internal/app/apptainer/oci_linux.go +++ b/internal/app/apptainer/oci_linux.go @@ -13,7 +13,6 @@ package apptainer import ( - "bufio" "fmt" "io" "os" @@ -62,11 +61,11 @@ type OciArgs struct { // AttachStreams contains streams that will be attached to the container type AttachStreams struct { // OutputStream will be attached to container's STDOUT - OutputStream io.WriteCloser + OutputStream io.Writer // ErrorStream will be attached to container's STDERR - ErrorStream io.WriteCloser + ErrorStream io.Writer // InputStream will be attached to container's STDIN - InputStream *bufio.Reader + InputStream io.Reader // AttachOutput is whether to attach to STDOUT // If false, stdout will not be attached AttachOutput bool From 3d24eb12dbfd044fc5c551886ae75d60e1cd51cb Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Wed, 9 Mar 2022 09:49:18 -0600 Subject: [PATCH 013/114] oci: chore: update missed copyright headers Signed-off-by: Edita Kizinevic --- cmd/internal/cli/oci_linux.go | 2 +- internal/app/apptainer/oci_pause_linux.go | 2 +- internal/app/apptainer/oci_start_linux.go | 2 +- internal/app/apptainer/oci_state_linux.go | 2 +- internal/app/apptainer/oci_update_linux.go | 2 +- internal/pkg/instance/instance_linux.go | 2 +- internal/pkg/util/bin/bin.go | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/internal/cli/oci_linux.go b/cmd/internal/cli/oci_linux.go index d38d139c24..01f3c2805c 100644 --- a/cmd/internal/cli/oci_linux.go +++ b/cmd/internal/cli/oci_linux.go @@ -2,7 +2,7 @@ // Apptainer a Series of LF Projects LLC. // For website terms of use, trademark policy, privacy policy and other // project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2020, Sylabs Inc. All rights reserved. +// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. diff --git a/internal/app/apptainer/oci_pause_linux.go b/internal/app/apptainer/oci_pause_linux.go index 65a816fbc5..8d74234522 100644 --- a/internal/app/apptainer/oci_pause_linux.go +++ b/internal/app/apptainer/oci_pause_linux.go @@ -2,7 +2,7 @@ // Apptainer a Series of LF Projects LLC. // For website terms of use, trademark policy, privacy policy and other // project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2019, Sylabs Inc. All rights reserved. +// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. diff --git a/internal/app/apptainer/oci_start_linux.go b/internal/app/apptainer/oci_start_linux.go index 275a2e65d9..324b758feb 100644 --- a/internal/app/apptainer/oci_start_linux.go +++ b/internal/app/apptainer/oci_start_linux.go @@ -2,7 +2,7 @@ // Apptainer a Series of LF Projects LLC. // For website terms of use, trademark policy, privacy policy and other // project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2019, Sylabs Inc. All rights reserved. +// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. diff --git a/internal/app/apptainer/oci_state_linux.go b/internal/app/apptainer/oci_state_linux.go index b229a66029..1ab04b2906 100644 --- a/internal/app/apptainer/oci_state_linux.go +++ b/internal/app/apptainer/oci_state_linux.go @@ -2,7 +2,7 @@ // Apptainer a Series of LF Projects LLC. // For website terms of use, trademark policy, privacy policy and other // project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2019, Sylabs Inc. All rights reserved. +// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. diff --git a/internal/app/apptainer/oci_update_linux.go b/internal/app/apptainer/oci_update_linux.go index 811d43cd23..983b3ea9bd 100644 --- a/internal/app/apptainer/oci_update_linux.go +++ b/internal/app/apptainer/oci_update_linux.go @@ -2,7 +2,7 @@ // Apptainer a Series of LF Projects LLC. // For website terms of use, trademark policy, privacy policy and other // project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2021, Sylabs Inc. All rights reserved. +// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. diff --git a/internal/pkg/instance/instance_linux.go b/internal/pkg/instance/instance_linux.go index 59c560b78b..f0c343c1f8 100644 --- a/internal/pkg/instance/instance_linux.go +++ b/internal/pkg/instance/instance_linux.go @@ -2,7 +2,7 @@ // Apptainer a Series of LF Projects LLC. // For website terms of use, trademark policy, privacy policy and other // project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2021, Sylabs Inc. All rights reserved. +// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. diff --git a/internal/pkg/util/bin/bin.go b/internal/pkg/util/bin/bin.go index 35554163b2..f91a0c2548 100644 --- a/internal/pkg/util/bin/bin.go +++ b/internal/pkg/util/bin/bin.go @@ -2,7 +2,7 @@ // Apptainer a Series of LF Projects LLC. // For website terms of use, trademark policy, privacy policy and other // project policies see https://lfprojects.org/policies -// Copyright (c) 2019-2021, Sylabs Inc. All rights reserved. +// Copyright (c) 2019-2022, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. From 588249ece5a61dbdc9260b70a0ec0d1107bc2158 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Tue, 25 Oct 2022 12:08:35 +0100 Subject: [PATCH 014/114] refactor: Move runc/conmon functionality out of app/singularity In preparation for further OCI runtime work, move the code that calls out to runc/conmon from internal/app/singularity into internal/pkg/runtime/launcher/oci The oci.Launcher will make use of the basic OCI operations to run containers from its Exec function, so this is a good location while that work proceeds. The functions will be likely be modified considerably, and potentially moved again in future, as the design/implementation of the OCI runtime interaction is developed. The internal/app/singularity OCI* functions are left as a minimal shim layer, between the CLI layer and the launcher, at this time. Signed-off-by: Edita Kizinevic --- cmd/internal/cli/oci_linux.go | 3 +- internal/app/apptainer/oci_delete_linux.go | 63 ----- internal/app/apptainer/oci_exec_linux.go | 38 --- internal/app/apptainer/oci_kill_linux.go | 39 --- internal/app/apptainer/oci_linux.go | 165 ++++-------- internal/app/apptainer/oci_mount_linux.go | 32 --- internal/app/apptainer/oci_pause_linux.go | 58 ----- internal/app/apptainer/oci_run_linux.go | 53 ---- internal/app/apptainer/oci_start_linux.go | 38 --- internal/app/apptainer/oci_state_linux.go | 38 --- internal/app/apptainer/oci_update_linux.go | 39 --- .../runtime/launcher/oci/launcher_linux.go | 3 +- .../runtime/launcher/oci}/oci_attach_linux.go | 42 ++- .../runtime/launcher/oci/oci_conmon_linux.go} | 18 +- .../pkg/runtime/launcher/oci/oci_linux.go | 120 +++++++++ .../runtime/launcher/oci/oci_runc_linux.go | 240 ++++++++++++++++++ 16 files changed, 460 insertions(+), 529 deletions(-) delete mode 100644 internal/app/apptainer/oci_delete_linux.go delete mode 100644 internal/app/apptainer/oci_exec_linux.go delete mode 100644 internal/app/apptainer/oci_kill_linux.go delete mode 100644 internal/app/apptainer/oci_mount_linux.go delete mode 100644 internal/app/apptainer/oci_pause_linux.go delete mode 100644 internal/app/apptainer/oci_run_linux.go delete mode 100644 internal/app/apptainer/oci_start_linux.go delete mode 100644 internal/app/apptainer/oci_state_linux.go delete mode 100644 internal/app/apptainer/oci_update_linux.go rename internal/{app/apptainer => pkg/runtime/launcher/oci}/oci_attach_linux.go (81%) rename internal/{app/apptainer/oci_create_linux.go => pkg/runtime/launcher/oci/oci_conmon_linux.go} (94%) create mode 100644 internal/pkg/runtime/launcher/oci/oci_linux.go create mode 100644 internal/pkg/runtime/launcher/oci/oci_runc_linux.go diff --git a/cmd/internal/cli/oci_linux.go b/cmd/internal/cli/oci_linux.go index 01f3c2805c..545a13e907 100644 --- a/cmd/internal/cli/oci_linux.go +++ b/cmd/internal/cli/oci_linux.go @@ -12,6 +12,7 @@ package cli import ( "github.com/apptainer/apptainer/docs" "github.com/apptainer/apptainer/internal/app/apptainer" + "github.com/apptainer/apptainer/internal/pkg/runtime/launcher/oci" "github.com/apptainer/apptainer/pkg/cmdline" "github.com/apptainer/apptainer/pkg/sylog" "github.com/spf13/cobra" @@ -241,7 +242,7 @@ var OciAttachCmd = &cobra.Command{ DisableFlagsInUseLine: true, PreRun: CheckRoot, Run: func(cmd *cobra.Command, args []string) { - if err := apptainer.OciAttach(cmd.Context(), args[0]); err != nil { + if err := oci.Attach(cmd.Context(), args[0]); err != nil { sylog.Fatalf("%s", err) } }, diff --git a/internal/app/apptainer/oci_delete_linux.go b/internal/app/apptainer/oci_delete_linux.go deleted file mode 100644 index c2648449c4..0000000000 --- a/internal/app/apptainer/oci_delete_linux.go +++ /dev/null @@ -1,63 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package apptainer - -import ( - "context" - "fmt" - "os" - "os/exec" - "path/filepath" - - "github.com/apptainer/apptainer/internal/pkg/util/bin" - "github.com/apptainer/apptainer/pkg/sylog" -) - -// OciDelete deletes container resources -func OciDelete(ctx context.Context, containerID string) error { - runc, err := bin.FindBin("runc") - if err != nil { - return err - } - runcArgs := []string{ - "--root", RuncStateDir, - "delete", - containerID, - } - - cmd := exec.Command(runc, runcArgs...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.Stdin = os.Stdout - sylog.Debugf("Calling runc with args %v", runcArgs) - err = cmd.Run() - if err != nil { - return fmt.Errorf("while calling runc delete: %w", err) - } - - sd, err := stateDir(containerID) - if err != nil { - return fmt.Errorf("while computing state directory: %w", err) - } - - bLink := filepath.Join(sd, bundleLink) - bundle, err := filepath.EvalSymlinks(bLink) - if err != nil { - return fmt.Errorf("while finding bundle directory: %w", err) - } - - sylog.Debugf("Removing bundle symlink") - if err := os.Remove(bLink); err != nil { - return fmt.Errorf("while removing bundle symlink: %w", err) - } - - sylog.Debugf("Releasing bundle lock") - return releaseBundle(bundle) -} diff --git a/internal/app/apptainer/oci_exec_linux.go b/internal/app/apptainer/oci_exec_linux.go deleted file mode 100644 index 67c8ab5f88..0000000000 --- a/internal/app/apptainer/oci_exec_linux.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package apptainer - -import ( - "os" - "os/exec" - - "github.com/apptainer/apptainer/internal/pkg/util/bin" - "github.com/apptainer/apptainer/pkg/sylog" -) - -// OciExec executes a command in a container -func OciExec(containerID string, cmdArgs []string) error { - runc, err := bin.FindBin("runc") - if err != nil { - return err - } - runcArgs := []string{ - "--root", RuncStateDir, - "exec", - containerID, - } - runcArgs = append(runcArgs, cmdArgs...) - cmd := exec.Command(runc, runcArgs...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.Stdin = os.Stdout - sylog.Debugf("Calling runc with args %v", runcArgs) - return cmd.Run() -} diff --git a/internal/app/apptainer/oci_kill_linux.go b/internal/app/apptainer/oci_kill_linux.go deleted file mode 100644 index 2d12aee6f2..0000000000 --- a/internal/app/apptainer/oci_kill_linux.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package apptainer - -import ( - "os" - "os/exec" - - "github.com/apptainer/apptainer/internal/pkg/util/bin" - "github.com/apptainer/apptainer/pkg/sylog" -) - -// OciKill kills container process -func OciKill(containerID string, killSignal string) error { - runc, err := bin.FindBin("runc") - if err != nil { - return err - } - runcArgs := []string{ - "--root", RuncStateDir, - "kill", - containerID, - killSignal, - } - - cmd := exec.Command(runc, runcArgs...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.Stdin = os.Stdout - sylog.Debugf("Calling runc with args %v", runcArgs) - return cmd.Run() -} diff --git a/internal/app/apptainer/oci_linux.go b/internal/app/apptainer/oci_linux.go index 04d981964d..9e5f16d7d7 100644 --- a/internal/app/apptainer/oci_linux.go +++ b/internal/app/apptainer/oci_linux.go @@ -13,36 +13,10 @@ package apptainer import ( - "fmt" - "io" - "os" - "path" - "path/filepath" - "time" - - "github.com/apptainer/apptainer/internal/pkg/util/fs" - "github.com/apptainer/apptainer/internal/pkg/util/user" - "github.com/apptainer/apptainer/pkg/syfs" - "github.com/apptainer/apptainer/pkg/util/fs/lock" - securejoin "github.com/cyphar/filepath-securejoin" -) + "context" -const ( - // Absolute path for the runc state - RuncStateDir = "/run/apptainer-oci" - // Relative path inside ~/.apptainer for conmon and apptainer state - ociPath = "oci" - // State directory files - containerPidFile = "container.pid" - containerLogFile = "container.log" - runcLogFile = "runc.log" - conmonPidFile = "conmon.pid" - bundleLink = "bundle" - // Files in the OCI bundle root - bundleLock = ".apptainer-oci.lock" - attachSocket = "attach" - // Timeouts - createTimeout = 30 * time.Second + "github.com/apptainer/apptainer/internal/pkg/runtime/launcher/oci" + ocibundle "github.com/apptainer/apptainer/pkg/ocibundle/sif" ) // OciArgs contains CLI arguments @@ -58,109 +32,70 @@ type OciArgs struct { ForceKill bool } -// AttachStreams contains streams that will be attached to the container -type AttachStreams struct { - // OutputStream will be attached to container's STDOUT - OutputStream io.Writer - // ErrorStream will be attached to container's STDERR - ErrorStream io.Writer - // InputStream will be attached to container's STDIN - InputStream io.Reader - // AttachOutput is whether to attach to STDOUT - // If false, stdout will not be attached - AttachOutput bool - // AttachError is whether to attach to STDERR - // If false, stdout will not be attached - AttachError bool - // AttachInput is whether to attach to STDIN - // If false, stdout will not be attached - AttachInput bool +// OciRun runs a container (equivalent to create/start/delete) +func OciRun(ctx context.Context, containerID string, args *OciArgs) error { + return oci.Run(ctx, containerID, args.BundlePath, args.PidFile) } -/* Sync with stdpipe_t in conmon.c */ -const ( - AttachPipeStdin = 1 - AttachPipeStdout = 2 - AttachPipeStderr = 3 -) - -type ociError struct { - Level string `json:"level,omitempty"` - Time string `json:"time,omitempty"` - Msg string `json:"msg,omitempty"` +// OciCreate creates a container from an OCI bundle +func OciCreate(containerID string, args *OciArgs) error { + return oci.Create(containerID, args.BundlePath) } -// stateDir returns the path to container state handled by conmon/apptainer -// (as opposed to runc's state in RuncStateDir) -func stateDir(containerID string) (string, error) { - hostname, err := os.Hostname() - if err != nil { - return "", err - } +// OciStart starts a previously create container +func OciStart(containerID string) error { + return oci.Start(containerID) +} - u, err := user.CurrentOriginal() - if err != nil { - return "", err - } +// OciDelete deletes container resources +func OciDelete(ctx context.Context, containerID string) error { + return oci.Delete(ctx, containerID) +} - configDir, err := syfs.ConfigDirForUsername(u.Name) - if err != nil { - return "", err - } +// OciExec executes a command in a container +func OciExec(containerID string, cmdArgs []string) error { + return oci.Exec(containerID, cmdArgs) +} - rootPath := filepath.Join(configDir, ociPath) - containerPath := filepath.Join(hostname, containerID) - path, err := securejoin.SecureJoin(rootPath, containerPath) - if err != nil { - return "", err - } - return path, err +// OciKill kills container process +func OciKill(containerID string, killSignal string) error { + return oci.Kill(containerID, killSignal) } -// lockBundle creates a lock file in a bundle directory -func lockBundle(bundlePath string) error { - bl := path.Join(bundlePath, bundleLock) - _, err := os.Stat(bl) - if err == nil { - return fmt.Errorf("bundle is locked by another process") - } - if !os.IsNotExist(err) { - return fmt.Errorf("while stat-ing lock file: %w", err) - } +// OciPause pauses processes in a container +func OciPause(containerID string) error { + return oci.Pause(containerID) +} - fd, err := lock.Exclusive(bundlePath) - if err != nil { - return fmt.Errorf("while acquiring directory lock: %w", err) - } - defer lock.Release(fd) +// OciResume pauses processes in a container +func OciResume(containerID string) error { + return oci.Resume(containerID) +} - err = fs.EnsureFileWithPermission(bl, 0o600) - if err != nil { - return fmt.Errorf("while creating lock file: %w", err) - } - return nil +// OciState queries container state +func OciState(containerID string, args *OciArgs) error { + return oci.State(containerID) } -// releaseBundle removes a lock file in a bundle directory -func releaseBundle(bundlePath string) error { - bl := path.Join(bundlePath, bundleLock) - _, err := os.Stat(bl) - if os.IsNotExist(err) { - return fmt.Errorf("bundle is not locked") - } - if err != nil { - return fmt.Errorf("while stat-ing lock file: %w", err) - } +// OciUpdate updates container cgroups resources +func OciUpdate(containerID string, args *OciArgs) error { + return oci.Update(containerID, args.FromFile) +} - fd, err := lock.Exclusive(bundlePath) +// OciMount mount a SIF image to create an OCI bundle +func OciMount(image string, bundle string) error { + d, err := ocibundle.FromSif(image, bundle, true) if err != nil { - return fmt.Errorf("while acquiring directory lock: %w", err) + return err } - defer lock.Release(fd) + return d.Create(nil) +} - err = os.Remove(bl) +// OciUmount umount SIF and delete OCI bundle +func OciUmount(bundle string) error { + d, err := ocibundle.FromSif("", bundle, true) if err != nil { - return fmt.Errorf("while removing lock file: %w", err) + return err } - return nil + return d.Delete() } diff --git a/internal/app/apptainer/oci_mount_linux.go b/internal/app/apptainer/oci_mount_linux.go deleted file mode 100644 index d7f0e758ac..0000000000 --- a/internal/app/apptainer/oci_mount_linux.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2019, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package apptainer - -import ( - ocibundle "github.com/apptainer/apptainer/pkg/ocibundle/sif" -) - -// OciMount mount a SIF image to create an OCI bundle -func OciMount(image string, bundle string) error { - d, err := ocibundle.FromSif(image, bundle, true) - if err != nil { - return err - } - return d.Create(nil) -} - -// OciUmount umount SIF and delete OCI bundle -func OciUmount(bundle string) error { - d, err := ocibundle.FromSif("", bundle, true) - if err != nil { - return err - } - return d.Delete() -} diff --git a/internal/app/apptainer/oci_pause_linux.go b/internal/app/apptainer/oci_pause_linux.go deleted file mode 100644 index 8d74234522..0000000000 --- a/internal/app/apptainer/oci_pause_linux.go +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package apptainer - -import ( - "os" - "os/exec" - - "github.com/apptainer/apptainer/internal/pkg/util/bin" - "github.com/apptainer/apptainer/pkg/sylog" -) - -// OciPause pauses processes in a container -func OciPause(containerID string) error { - runc, err := bin.FindBin("runc") - if err != nil { - return err - } - runcArgs := []string{ - "--root", RuncStateDir, - "pause", - containerID, - } - - cmd := exec.Command(runc, runcArgs...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.Stdin = os.Stdout - sylog.Debugf("Calling runc with args %v", runcArgs) - return cmd.Run() -} - -// OciResume pauses processes in a container -func OciResume(containerID string) error { - runc, err := bin.FindBin("runc") - if err != nil { - return err - } - runcArgs := []string{ - "--root", RuncStateDir, - "resume", - containerID, - } - - cmd := exec.Command(runc, runcArgs...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.Stdin = os.Stdout - sylog.Debugf("Calling runc with args %v", runcArgs) - return cmd.Run() -} diff --git a/internal/app/apptainer/oci_run_linux.go b/internal/app/apptainer/oci_run_linux.go deleted file mode 100644 index 2337228a5f..0000000000 --- a/internal/app/apptainer/oci_run_linux.go +++ /dev/null @@ -1,53 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package apptainer - -import ( - "context" - "fmt" - "os" - "os/exec" - "path/filepath" - - "github.com/apptainer/apptainer/internal/pkg/util/bin" - "github.com/apptainer/apptainer/pkg/sylog" -) - -// OciRun runs a container (equivalent to create/start/delete) -func OciRun(ctx context.Context, containerID string, args *OciArgs) error { - runc, err := bin.FindBin("runc") - if err != nil { - return err - } - absBundle, err := filepath.Abs(args.BundlePath) - if err != nil { - return fmt.Errorf("failed to determine bundle absolute path: %s", err) - } - - if err := os.Chdir(absBundle); err != nil { - return fmt.Errorf("failed to change directory to %s: %s", absBundle, err) - } - - runcArgs := []string{ - "--root", RuncStateDir, - "run", - "-b", absBundle, - } - if args.PidFile != "" { - runcArgs = append(runcArgs, "--pid-file="+args.PidFile) - } - runcArgs = append(runcArgs, containerID) - cmd := exec.Command(runc, runcArgs...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.Stdin = os.Stdout - sylog.Debugf("Calling runc with args %v", runcArgs) - return cmd.Run() -} diff --git a/internal/app/apptainer/oci_start_linux.go b/internal/app/apptainer/oci_start_linux.go deleted file mode 100644 index 324b758feb..0000000000 --- a/internal/app/apptainer/oci_start_linux.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package apptainer - -import ( - "os" - "os/exec" - - "github.com/apptainer/apptainer/internal/pkg/util/bin" - "github.com/apptainer/apptainer/pkg/sylog" -) - -// OciStart starts a previously create container -func OciStart(containerID string) error { - runc, err := bin.FindBin("runc") - if err != nil { - return err - } - runcArgs := []string{ - "--root", RuncStateDir, - "start", - containerID, - } - - cmd := exec.Command(runc, runcArgs...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.Stdin = os.Stdout - sylog.Debugf("Calling runc with args %v", runcArgs) - return cmd.Run() -} diff --git a/internal/app/apptainer/oci_state_linux.go b/internal/app/apptainer/oci_state_linux.go deleted file mode 100644 index 1ab04b2906..0000000000 --- a/internal/app/apptainer/oci_state_linux.go +++ /dev/null @@ -1,38 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package apptainer - -import ( - "os" - "os/exec" - - "github.com/apptainer/apptainer/internal/pkg/util/bin" - "github.com/apptainer/apptainer/pkg/sylog" -) - -// OciState query container state -func OciState(containerID string, args *OciArgs) error { - runc, err := bin.FindBin("runc") - if err != nil { - return err - } - runcArgs := []string{ - "--root", RuncStateDir, - "state", - containerID, - } - - cmd := exec.Command(runc, runcArgs...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.Stdin = os.Stdout - sylog.Debugf("Calling runc with args %v", runcArgs) - return cmd.Run() -} diff --git a/internal/app/apptainer/oci_update_linux.go b/internal/app/apptainer/oci_update_linux.go deleted file mode 100644 index 983b3ea9bd..0000000000 --- a/internal/app/apptainer/oci_update_linux.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright (c) Contributors to the Apptainer project, established as -// Apptainer a Series of LF Projects LLC. -// For website terms of use, trademark policy, privacy policy and other -// project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. -// This software is licensed under a 3-clause BSD license. Please consult the -// LICENSE.md file distributed with the sources of this project regarding your -// rights to use or distribute this software. - -package apptainer - -import ( - "os" - "os/exec" - - "github.com/apptainer/apptainer/internal/pkg/util/bin" - "github.com/apptainer/apptainer/pkg/sylog" -) - -// OciUpdate updates container cgroups resources -func OciUpdate(containerID string, args *OciArgs) error { - runc, err := bin.FindBin("runc") - if err != nil { - return err - } - runcArgs := []string{ - "--root", RuncStateDir, - "update", - "-r", args.FromFile, - containerID, - } - - cmd := exec.Command(runc, runcArgs...) - cmd.Stdout = os.Stdout - cmd.Stderr = os.Stderr - cmd.Stdin = os.Stdout - sylog.Debugf("Calling runc with args %v", runcArgs) - return cmd.Run() -} diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index ca4151be86..8169dda206 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -8,7 +8,8 @@ // rights to use or distribute this software. // Package oci implements a Launcher that will configure and launch a container -// with an OCI runtime. +// with an OCI runtime. It also provides implementations of OCI state +// transitions that can be called directly, Create/Start/Kill etc. package oci import ( diff --git a/internal/app/apptainer/oci_attach_linux.go b/internal/pkg/runtime/launcher/oci/oci_attach_linux.go similarity index 81% rename from internal/app/apptainer/oci_attach_linux.go rename to internal/pkg/runtime/launcher/oci/oci_attach_linux.go index e2d43b658c..1b32deb7d4 100644 --- a/internal/app/apptainer/oci_attach_linux.go +++ b/internal/pkg/runtime/launcher/oci/oci_attach_linux.go @@ -10,7 +10,7 @@ // Includes code from https://github.com/containers/podman // Released under the Apache License Version 2.0 -package apptainer +package oci import ( "bufio" @@ -30,9 +30,35 @@ import ( var ErrDetach = errors.New("detached from container") -// OciAttach attaches the console to a running container -func OciAttach(ctx context.Context, containerID string) error { - streams := AttachStreams{ +// attachStreams contains streams that will be attached to the container +type attachStreams struct { + // OutputStream will be attached to container's STDOUT + OutputStream io.Writer + // ErrorStream will be attached to container's STDERR + ErrorStream io.Writer + // InputStream will be attached to container's STDIN + InputStream io.Reader + // AttachOutput is whether to attach to STDOUT + // If false, stdout will not be attached + AttachOutput bool + // AttachError is whether to attach to STDERR + // If false, stdout will not be attached + AttachError bool + // AttachInput is whether to attach to STDIN + // If false, stdout will not be attached + AttachInput bool +} + +/* Sync with stdpipe_t in conmon.c */ +const ( + AttachPipeStdin = 1 + AttachPipeStdout = 2 + AttachPipeStderr = 3 +) + +// Attach attaches the console to a running container +func Attach(ctx context.Context, containerID string) error { + streams := attachStreams{ OutputStream: os.Stdout, ErrorStream: os.Stderr, InputStream: bufio.NewReader(os.Stdin), @@ -78,7 +104,7 @@ func openUnixSocket(path string) (*net.UnixConn, error) { return net.DialUnix("unixpacket", nil, &net.UnixAddr{Name: fmt.Sprintf("/proc/self/fd/%d", fd), Net: "unixpacket"}) } -func setupStdioChannels(streams AttachStreams, conn *net.UnixConn, detachKeys []byte) (chan error, chan error) { +func setupStdioChannels(streams attachStreams, conn *net.UnixConn, detachKeys []byte) (chan error, chan error) { receiveStdoutError := make(chan error) go func() { receiveStdoutError <- redirectResponseToOutputStreams(streams.OutputStream, streams.ErrorStream, streams.AttachOutput, streams.AttachError, conn) @@ -88,7 +114,7 @@ func setupStdioChannels(streams AttachStreams, conn *net.UnixConn, detachKeys [] go func() { var err error if streams.AttachInput { - _, err = CopyDetachable(conn, streams.InputStream, detachKeys) + _, err = copyDetachable(conn, streams.InputStream, detachKeys) } stdinDone <- err }() @@ -141,7 +167,7 @@ func redirectResponseToOutputStreams(outputStream, errorStream io.Writer, writeO return err } -func readStdio(conn *net.UnixConn, streams AttachStreams, receiveStdoutError, stdinDone chan error) error { +func readStdio(conn *net.UnixConn, streams attachStreams, receiveStdoutError, stdinDone chan error) error { var err error select { case err = <-receiveStdoutError: @@ -165,7 +191,7 @@ func readStdio(conn *net.UnixConn, streams AttachStreams, receiveStdoutError, st return nil } -func CopyDetachable(dst io.Writer, src io.Reader, keys []byte) (written int64, err error) { +func copyDetachable(dst io.Writer, src io.Reader, keys []byte) (written int64, err error) { buf := make([]byte, 32*1024) for { nr, er := src.Read(buf) diff --git a/internal/app/apptainer/oci_create_linux.go b/internal/pkg/runtime/launcher/oci/oci_conmon_linux.go similarity index 94% rename from internal/app/apptainer/oci_create_linux.go rename to internal/pkg/runtime/launcher/oci/oci_conmon_linux.go index 3c781c21ab..dcf2cde0cd 100644 --- a/internal/app/apptainer/oci_create_linux.go +++ b/internal/pkg/runtime/launcher/oci/oci_conmon_linux.go @@ -10,7 +10,7 @@ // Includes code from https://github.com/containers/podman // Released under the Apache License Version 2.0 -package apptainer +package oci import ( "bufio" @@ -31,8 +31,14 @@ import ( "golang.org/x/sys/unix" ) -// OciCreate creates a container from an OCI bundle -func OciCreate(containerID string, args *OciArgs) error { +type ociError struct { + Level string `json:"level,omitempty"` + Time string `json:"time,omitempty"` + Msg string `json:"msg,omitempty"` +} + +// Create creates a container from an OCI bundle +func Create(containerID, bundlePath string) error { conmon, err := bin.FindBin("conmon") if err != nil { return err @@ -42,7 +48,7 @@ func OciCreate(containerID string, args *OciArgs) error { return err } // chdir to bundle and lock it, so another oci create cannot use the same bundle - absBundle, err := filepath.Abs(args.BundlePath) + absBundle, err := filepath.Abs(bundlePath) if err != nil { return fmt.Errorf("failed to determine bundle absolute path: %w", err) } @@ -95,7 +101,7 @@ func OciCreate(containerID string, args *OciArgs) error { "--container-pidfile", path.Join(sd, containerPidFile), "--log-path", path.Join(sd, containerLogFile), "--runtime-arg", "--root", - "--runtime-arg", RuncStateDir, + "--runtime-arg", runcStateDir, "--runtime-arg", "--log", "--runtime-arg", path.Join(sd, runcLogFile), "--full-attach", @@ -145,7 +151,7 @@ func OciCreate(containerID string, args *OciArgs) error { // We check for errors from runc (which conmon invokes) via the sync pipe pid, err := readConmonPipeData(syncParent, path.Join(sd, runcLogFile)) if err != nil { - if err2 := OciDelete(context.TODO(), containerID); err2 != nil { + if err2 := Delete(context.TODO(), containerID); err2 != nil { sylog.Errorf("Removing container %s from runtime after creation failed", containerID) } return err diff --git a/internal/pkg/runtime/launcher/oci/oci_linux.go b/internal/pkg/runtime/launcher/oci/oci_linux.go new file mode 100644 index 0000000000..5298bf2849 --- /dev/null +++ b/internal/pkg/runtime/launcher/oci/oci_linux.go @@ -0,0 +1,120 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +// Includes code from https://github.com/containers/podman +// Released under the Apache License Version 2.0 + +package oci + +import ( + "fmt" + "os" + "path" + "path/filepath" + "time" + + "github.com/apptainer/apptainer/internal/pkg/util/fs" + "github.com/apptainer/apptainer/internal/pkg/util/user" + "github.com/apptainer/apptainer/pkg/syfs" + "github.com/apptainer/apptainer/pkg/util/fs/lock" + securejoin "github.com/cyphar/filepath-securejoin" +) + +const ( + // Absolute path for the runc state + runcStateDir = "/run/apptainer-oci" + // Relative path inside ~/.apptainer for conmon and apptainer state + ociPath = "oci" + // State directory files + containerPidFile = "container.pid" + containerLogFile = "container.log" + runcLogFile = "runc.log" + conmonPidFile = "conmon.pid" + bundleLink = "bundle" + // Files in the OCI bundle root + bundleLock = ".apptainer-oci.lock" + attachSocket = "attach" + // Timeouts + createTimeout = 30 * time.Second +) + +// stateDir returns the path to container state handled by conmon/apptainer +// (as opposed to runc's state in RuncStateDir) +func stateDir(containerID string) (string, error) { + hostname, err := os.Hostname() + if err != nil { + return "", err + } + + u, err := user.CurrentOriginal() + if err != nil { + return "", err + } + + configDir, err := syfs.ConfigDirForUsername(u.Name) + if err != nil { + return "", err + } + + rootPath := filepath.Join(configDir, ociPath) + containerPath := filepath.Join(hostname, containerID) + path, err := securejoin.SecureJoin(rootPath, containerPath) + if err != nil { + return "", err + } + return path, err +} + +// lockBundle creates a lock file in a bundle directory +func lockBundle(bundlePath string) error { + bl := path.Join(bundlePath, bundleLock) + _, err := os.Stat(bl) + if err == nil { + return fmt.Errorf("bundle is locked by another process") + } + if !os.IsNotExist(err) { + return fmt.Errorf("while stat-ing lock file: %w", err) + } + + fd, err := lock.Exclusive(bundlePath) + if err != nil { + return fmt.Errorf("while acquiring directory lock: %w", err) + } + defer lock.Release(fd) + + err = fs.EnsureFileWithPermission(bl, 0o600) + if err != nil { + return fmt.Errorf("while creating lock file: %w", err) + } + return nil +} + +// releaseBundle removes a lock file in a bundle directory +func releaseBundle(bundlePath string) error { + bl := path.Join(bundlePath, bundleLock) + _, err := os.Stat(bl) + if os.IsNotExist(err) { + return fmt.Errorf("bundle is not locked") + } + if err != nil { + return fmt.Errorf("while stat-ing lock file: %w", err) + } + + fd, err := lock.Exclusive(bundlePath) + if err != nil { + return fmt.Errorf("while acquiring directory lock: %w", err) + } + defer lock.Release(fd) + + err = os.Remove(bl) + if err != nil { + return fmt.Errorf("while removing lock file: %w", err) + } + return nil +} diff --git a/internal/pkg/runtime/launcher/oci/oci_runc_linux.go b/internal/pkg/runtime/launcher/oci/oci_runc_linux.go new file mode 100644 index 0000000000..d96a1a702e --- /dev/null +++ b/internal/pkg/runtime/launcher/oci/oci_runc_linux.go @@ -0,0 +1,240 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +// Includes code from https://github.com/containers/podman +// Released under the Apache License Version 2.0 + +package oci + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + + "github.com/apptainer/apptainer/internal/pkg/util/bin" + "github.com/apptainer/apptainer/pkg/sylog" +) + +// Delete deletes container resources +func Delete(ctx context.Context, containerID string) error { + runc, err := bin.FindBin("runc") + if err != nil { + return err + } + runcArgs := []string{ + "--root", runcStateDir, + "delete", + containerID, + } + + cmd := exec.Command(runc, runcArgs...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = os.Stdout + sylog.Debugf("Calling runc with args %v", runcArgs) + err = cmd.Run() + if err != nil { + return fmt.Errorf("while calling runc delete: %w", err) + } + + sd, err := stateDir(containerID) + if err != nil { + return fmt.Errorf("while computing state directory: %w", err) + } + + bLink := filepath.Join(sd, bundleLink) + bundle, err := filepath.EvalSymlinks(bLink) + if err != nil { + return fmt.Errorf("while finding bundle directory: %w", err) + } + + sylog.Debugf("Removing bundle symlink") + if err := os.Remove(bLink); err != nil { + return fmt.Errorf("while removing bundle symlink: %w", err) + } + + sylog.Debugf("Releasing bundle lock") + return releaseBundle(bundle) +} + +// Exec executes a command in a container +func Exec(containerID string, cmdArgs []string) error { + runc, err := bin.FindBin("runc") + if err != nil { + return err + } + runcArgs := []string{ + "--root", runcStateDir, + "exec", + containerID, + } + runcArgs = append(runcArgs, cmdArgs...) + cmd := exec.Command(runc, runcArgs...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = os.Stdout + sylog.Debugf("Calling runc with args %v", runcArgs) + return cmd.Run() +} + +// Kill kills container process +func Kill(containerID string, killSignal string) error { + runc, err := bin.FindBin("runc") + if err != nil { + return err + } + runcArgs := []string{ + "--root", runcStateDir, + "kill", + containerID, + killSignal, + } + + cmd := exec.Command(runc, runcArgs...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = os.Stdout + sylog.Debugf("Calling runc with args %v", runcArgs) + return cmd.Run() +} + +// Pause pauses processes in a container +func Pause(containerID string) error { + runc, err := bin.FindBin("runc") + if err != nil { + return err + } + runcArgs := []string{ + "--root", runcStateDir, + "pause", + containerID, + } + + cmd := exec.Command(runc, runcArgs...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = os.Stdout + sylog.Debugf("Calling runc with args %v", runcArgs) + return cmd.Run() +} + +// Resume pauses processes in a container +func Resume(containerID string) error { + runc, err := bin.FindBin("runc") + if err != nil { + return err + } + runcArgs := []string{ + "--root", runcStateDir, + "resume", + containerID, + } + + cmd := exec.Command(runc, runcArgs...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = os.Stdout + sylog.Debugf("Calling runc with args %v", runcArgs) + return cmd.Run() +} + +// Run runs a container (equivalent to create/start/delete) +func Run(ctx context.Context, containerID, bundlePath, pidFile string) error { + runc, err := bin.FindBin("runc") + if err != nil { + return err + } + absBundle, err := filepath.Abs(bundlePath) + if err != nil { + return fmt.Errorf("failed to determine bundle absolute path: %s", err) + } + + if err := os.Chdir(absBundle); err != nil { + return fmt.Errorf("failed to change directory to %s: %s", absBundle, err) + } + + runcArgs := []string{ + "--root", runcStateDir, + "run", + "-b", absBundle, + } + if pidFile != "" { + runcArgs = append(runcArgs, "--pid-file="+pidFile) + } + runcArgs = append(runcArgs, containerID) + cmd := exec.Command(runc, runcArgs...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = os.Stdout + sylog.Debugf("Calling runc with args %v", runcArgs) + return cmd.Run() +} + +// Start starts a previously created container +func Start(containerID string) error { + runc, err := bin.FindBin("runc") + if err != nil { + return err + } + runcArgs := []string{ + "--root", runcStateDir, + "start", + containerID, + } + + cmd := exec.Command(runc, runcArgs...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = os.Stdout + sylog.Debugf("Calling runc with args %v", runcArgs) + return cmd.Run() +} + +// State queries container state +func State(containerID string) error { + runc, err := bin.FindBin("runc") + if err != nil { + return err + } + runcArgs := []string{ + "--root", runcStateDir, + "state", + containerID, + } + + cmd := exec.Command(runc, runcArgs...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = os.Stdout + sylog.Debugf("Calling runc with args %v", runcArgs) + return cmd.Run() +} + +// Update updates container cgroups resources +func Update(containerID, cgFile string) error { + runc, err := bin.FindBin("runc") + if err != nil { + return err + } + runcArgs := []string{ + "--root", runcStateDir, + "update", + "-r", cgFile, + containerID, + } + + cmd := exec.Command(runc, runcArgs...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = os.Stdout + sylog.Debugf("Calling runc with args %v", runcArgs) + return cmd.Run() +} From 955e9fb1107202e09d0bf936a8cdc7a9803b8a80 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Thu, 27 Oct 2022 12:57:20 +0100 Subject: [PATCH 015/114] feat: run action for OCI bundle As a first step toward run/shell/exec actions on native OCI images, implement a minimal `singularity run --oci mybundle` which: * Requires an on-disk bundle with appropriate `config.json`. * Runs this bundle using `crun` or `runc`. * Makes no attempt to handle any arguments or options. * Does not modify the `config.json` - i.e. it must match namespace / mapping requirements for rootless execution etc. At this stage, the functionality is essentially equivalent to `singularity oci run` and is not yet useful. The primary purpose of the PR is to refactor some of the code that passes args for launching a container. In addition, we now use `crun` in preference to `runc` if available. `crun` supports e.g. single uid->uid mapping in a usernamespace (without root mapping). Closes sylabs/singularity#598 Signed-off-by: Edita Kizinevic --- .github/workflows/ci.yml | 6 +- CHANGELOG.md | 3 +- cmd/internal/cli/actions.go | 55 ++++++++--- cmd/internal/cli/checkpoint.go | 5 +- cmd/internal/cli/instance_actions_linux.go | 7 +- cmd/internal/cli/startvm.go | 6 +- e2e/actions/actions.go | 23 +---- e2e/actions/oci.go | 82 ++++++++++++++++ internal/pkg/runtime/launcher/launcher.go | 2 +- .../runtime/launcher/native/launcher_linux.go | 5 +- .../runtime/launcher/oci/launcher_linux.go | 23 ++++- .../launcher/oci/launcher_linux_test.go | 12 --- .../runtime/launcher/oci/oci_conmon_linux.go | 6 +- .../pkg/runtime/launcher/oci/oci_linux.go | 24 ++++- .../runtime/launcher/oci/oci_runc_linux.go | 96 +++++++++---------- internal/pkg/util/bin/bin.go | 1 + 16 files changed, 240 insertions(+), 116 deletions(-) create mode 100644 e2e/actions/oci.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aed2e655a8..cd748c6677 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -238,7 +238,7 @@ jobs: go-version: 1.20.5 - name: Fetch deps - run: sudo apt-get -q update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential squashfs-tools squashfuse fuse-overlayfs fakeroot fuse2fs libseccomp-dev cryptsetup dbus-user-session conmon runc + run: sudo apt-get -q update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential squashfs-tools squashfuse fuse-overlayfs fakeroot fuse2fs libseccomp-dev cryptsetup dbus-user-session conmon crun - name: Build and install Apptainer run: | @@ -268,7 +268,7 @@ jobs: go-version: 1.20.5 - name: Fetch deps - run: sudo apt-get -q update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential squashfs-tools libseccomp-dev cryptsetup dbus-user-session conmon runc + run: sudo apt-get -q update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential squashfs-tools libseccomp-dev cryptsetup dbus-user-session conmon crun - name: Build and install Apptainer run: | @@ -314,7 +314,7 @@ jobs: - name: Fetch deps if: env.run_tests - run: sudo apt-get -q update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential uidmap squashfs-tools squashfuse fuse-overlayfs fakeroot fuse2fs libseccomp-dev cryptsetup dbus-user-session conmon runc + run: sudo apt-get -q update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential uidmap squashfs-tools squashfuse fuse-overlayfs fakeroot fuse2fs libseccomp-dev cryptsetup dbus-user-session conmon crun - name: Fetch gocryptfs run: wget -O gocryptfs.tar.gz https://github.com/rfjakob/gocryptfs/releases/download/v2.3/gocryptfs_v2.3_linux-static_amd64.tar.gz && sudo tar xzvf gocryptfs.tar.gz -C /usr/local/bin gocryptfs diff --git a/CHANGELOG.md b/CHANGELOG.md index 22601e6bd6..f8696354ae 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,8 @@ For older changes see the [archived Singularity change log](https://github.com/a working directory, though `--pwd` is still supported for compatibility. - When building RPM, we will now use `/var/lib/apptainer` (rather than `/var/apptainer`) to store local state files. -- The `apptainer oci` command group now uses `runc` to manage containers. +- The `apptainer oci` command group now uses `crun`, when available, or otherwise + `runc` to manage containers. - The `apptainer oci` flags `--sync-socket`, `--empty-process`, and `--timeout` have been removed. diff --git a/cmd/internal/cli/actions.go b/cmd/internal/cli/actions.go index 448b49c6b7..e9f410c527 100644 --- a/cmd/internal/cli/actions.go +++ b/cmd/internal/cli/actions.go @@ -195,13 +195,21 @@ var ExecCmd = &cobra.Command{ Args: cobra.MinimumNArgs(2), PreRun: actionPreRun, Run: func(cmd *cobra.Command, args []string) { - a := append([]string{"/.singularity.d/actions/exec"}, args[1:]...) + // apptainer exec [args...] + image := args[0] + containerCmd := "/.singularity.d/actions/exec" + containerArgs := args[1:] + // OCI runtime does not use an action script + if ociRuntime { + containerCmd = args[1] + containerArgs = args[2:] + } setVM(cmd) if vm { - execVM(cmd, args[0], a) + execVM(cmd, image, containerCmd, containerArgs) return } - if err := launchContainer(cmd, args[0], a, ""); err != nil { + if err := launchContainer(cmd, image, containerCmd, containerArgs, ""); err != nil { sylog.Fatalf("%s", err) } }, @@ -223,13 +231,21 @@ var ShellCmd = &cobra.Command{ sylog.Warningf("Parameters to shell command are ignored") } - a := []string{"/.singularity.d/actions/shell"} + // apptainer shell + image := args[0] + containerCmd := "/.singularity.d/actions/shell" + containerArgs := []string{} + // OCI runtime does not use an action script + if ociRuntime { + // TODO - needs to have bash -> sh fallback logic implemented somewhere. + containerCmd = "/bin/sh" + } setVM(cmd) if vm { - execVM(cmd, args[0], a) + execVM(cmd, image, containerCmd, containerArgs) return } - if err := launchContainer(cmd, args[0], a, ""); err != nil { + if err := launchContainer(cmd, image, containerCmd, containerArgs, ""); err != nil { sylog.Fatalf("%s", err) } }, @@ -247,13 +263,20 @@ var RunCmd = &cobra.Command{ Args: cobra.MinimumNArgs(1), PreRun: actionPreRun, Run: func(cmd *cobra.Command, args []string) { - a := append([]string{"/.singularity.d/actions/run"}, args[1:]...) + // apptainer run [args...] + image := args[0] + containerCmd := "/.singularity.d/actions/run" + containerArgs := args[1:] + // OCI runtime does not use an action script + if ociRuntime { + containerCmd = "" + } setVM(cmd) if vm { - execVM(cmd, args[0], a) + execVM(cmd, args[0], containerCmd, containerArgs) return } - if err := launchContainer(cmd, args[0], a, ""); err != nil { + if err := launchContainer(cmd, image, containerCmd, containerArgs, ""); err != nil { sylog.Fatalf("%s", err) } }, @@ -271,13 +294,15 @@ var TestCmd = &cobra.Command{ Args: cobra.MinimumNArgs(1), PreRun: actionPreRun, Run: func(cmd *cobra.Command, args []string) { - a := append([]string{"/.singularity.d/actions/test"}, args[1:]...) - setVM(cmd) + // apptainer test [args...] + image := args[0] + containerCmd := "/.singularity.d/actions/test" + containerArgs := args[1:] if vm { - execVM(cmd, args[0], a) + execVM(cmd, image, containerCmd, containerArgs) return } - if err := launchContainer(cmd, args[0], a, ""); err != nil { + if err := launchContainer(cmd, image, containerCmd, containerArgs, ""); err != nil { sylog.Fatalf("%s", err) } }, @@ -288,7 +313,7 @@ var TestCmd = &cobra.Command{ Example: docs.RunTestExample, } -func launchContainer(cmd *cobra.Command, image string, args []string, instanceName string) error { +func launchContainer(cmd *cobra.Command, image string, containerCmd string, containerArgs []string, instanceName string) error { ns := launcher.Namespaces{ User: userNamespace, UTS: utsNamespace, @@ -380,5 +405,5 @@ func launchContainer(cmd *cobra.Command, image string, args []string, instanceNa } } - return l.Exec(cmd.Context(), image, args, instanceName) + return l.Exec(cmd.Context(), image, containerCmd, containerArgs, instanceName) } diff --git a/cmd/internal/cli/checkpoint.go b/cmd/internal/cli/checkpoint.go index 04f17f0a25..f37a150805 100644 --- a/cmd/internal/cli/checkpoint.go +++ b/cmd/internal/cli/checkpoint.go @@ -166,8 +166,9 @@ var CheckpointInstanceCmd = &cobra.Command{ sylog.Infof("Using checkpoint %q", e.Name()) - a := append([]string{"/.singularity.d/actions/exec"}, dmtcp.CheckpointArgs(port)...) - if err := launchContainer(cmd, "instance://"+args[0], a, ""); err != nil { + containerCmd := "/.singularity.d/actions/exec" + containerArgs := dmtcp.CheckpointArgs(port) + if err := launchContainer(cmd, "instance://"+args[0], containerCmd, containerArgs, ""); err != nil { sylog.Fatalf("%s", err) } }, diff --git a/cmd/internal/cli/instance_actions_linux.go b/cmd/internal/cli/instance_actions_linux.go index da351533fe..e3a19eecba 100644 --- a/cmd/internal/cli/instance_actions_linux.go +++ b/cmd/internal/cli/instance_actions_linux.go @@ -49,13 +49,14 @@ func instanceAction(cmd *cobra.Command, args []string) { script = "run" killCont = "kill -CONT 1; " } - a := append([]string{killCont + "/.singularity.d/actions/" + script}, args[2:]...) + containerCmd := killCont + "/.singularity.d/actions/" + script + containerArgs := args[2:] setVM(cmd) if vm { - execVM(cmd, image, a) + execVM(cmd, image, containerCmd, containerArgs) return } - if err := launchContainer(cmd, image, a, name); err != nil { + if err := launchContainer(cmd, image, containerCmd, containerArgs, name); err != nil { sylog.Fatalf("%s", err) } diff --git a/cmd/internal/cli/startvm.go b/cmd/internal/cli/startvm.go index 53d050d99f..b1917bfbff 100644 --- a/cmd/internal/cli/startvm.go +++ b/cmd/internal/cli/startvm.go @@ -32,7 +32,7 @@ func getHypervisorArgs(sifImage, bzImage, initramfs, singAction, cliExtra string return args } -func execVM(cmd *cobra.Command, image string, args []string) { +func execVM(cmd *cobra.Command, image string, containerCmd string, containerArgs []string) { // SIF image we are running sifImage := image @@ -46,8 +46,8 @@ func execVM(cmd *cobra.Command, image string, args []string) { isInternal = true } else { // Get our "action" (run, exec, shell) based on the action script being called - singAction = filepath.Base(args[0]) - cliExtra = strings.Join(args[1:], " ") + singAction = filepath.Base(containerCmd) + cliExtra = strings.Join(containerArgs, " ") } if err := startVM(sifImage, singAction, cliExtra, isInternal); err != nil { diff --git a/e2e/actions/actions.go b/e2e/actions/actions.go index 866bc2a50e..8a7a508836 100644 --- a/e2e/actions/actions.go +++ b/e2e/actions/actions.go @@ -2861,24 +2861,6 @@ func (c actionTests) relWorkdirScratch(t *testing.T) { } } -func (c actionTests) ociRuntime(t *testing.T) { - e2e.EnsureImage(t, c.env) - - for _, p := range []e2e.Profile{e2e.OCIUserProfile, e2e.OCIRootProfile} { - c.env.RunApptainer( - t, - e2e.AsSubtest(p.String()), - e2e.WithProfile(p), - e2e.WithCommand("exec"), - e2e.WithArgs(c.env.ImagePath, "/bin/true"), - e2e.ExpectExit( - 255, - e2e.ExpectError(e2e.ContainMatch, "not implemented"), - ), - ) - } -} - // E2ETests is the main func to trigger the test suite func E2ETests(env e2e.TestEnv) testhelper.Tests { c := actionTests{ @@ -2924,9 +2906,12 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { "no-mount": c.actionNoMount, // test --no-mount "compat": np(c.actionCompat), // test --compat "umask": np(c.actionUmask), // test umask propagation - "ociRuntime": c.ociRuntime, // test --oci (unimplemented) "invalidRemote": np(c.invalidRemote), // GHSA-5mv9-q7fq-9394 "fakeroot home": c.actionFakerootHome, // test home dir in fakeroot "relWorkdirScratch": np(c.relWorkdirScratch), // test relative --workdir with --scratch + // + // OCI Runtime Mode + // + "ociRun": c.actionOciRun, // apptainer run --oci } } diff --git a/e2e/actions/oci.go b/e2e/actions/oci.go new file mode 100644 index 0000000000..c49c9d3f26 --- /dev/null +++ b/e2e/actions/oci.go @@ -0,0 +1,82 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2022, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package actions + +import ( + "os" + "testing" + + "github.com/apptainer/apptainer/e2e/internal/e2e" + "github.com/apptainer/apptainer/internal/pkg/test/tool/require" + "github.com/pkg/errors" +) + +func (c actionTests) ociBundle(t *testing.T) (string, func()) { + require.Seccomp(t) + require.Filesystem(t, "overlay") + + bundleDir, err := os.MkdirTemp(c.env.TestDir, "bundle-") + if err != nil { + err = errors.Wrapf(err, "creating temporary bundle directory at %q", c.env.TestDir) + t.Fatalf("failed to create bundle directory: %+v", err) + } + c.env.RunApptainer( + t, + e2e.WithProfile(e2e.RootProfile), + e2e.WithCommand("oci mount"), + e2e.WithArgs(c.env.ImagePath, bundleDir), + e2e.ExpectExit(0), + ) + + cleanup := func() { + c.env.RunApptainer( + t, + e2e.WithProfile(e2e.RootProfile), + e2e.WithCommand("oci umount"), + e2e.WithArgs(bundleDir), + e2e.ExpectExit(0), + ) + os.RemoveAll(bundleDir) + } + + return bundleDir, cleanup +} + +func (c actionTests) actionOciRun(t *testing.T) { + e2e.EnsureImage(t, c.env) + + bundle, cleanup := c.ociBundle(t) + defer cleanup() + + tests := []struct { + name string + argv []string + exit int + }{ + { + name: "NoCommand", + argv: []string{bundle}, + exit: 0, + }, + } + + for _, tt := range tests { + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(e2e.OCIRootProfile), + e2e.WithCommand("run"), + // While we don't support args we are entering a /bin/sh interactively, so we need to exit. + e2e.ConsoleRun(e2e.ConsoleSendLine("exit")), + e2e.WithArgs(tt.argv...), + e2e.ExpectExit(tt.exit), + ) + } +} diff --git a/internal/pkg/runtime/launcher/launcher.go b/internal/pkg/runtime/launcher/launcher.go index 1a79a55889..177dc373b2 100644 --- a/internal/pkg/runtime/launcher/launcher.go +++ b/internal/pkg/runtime/launcher/launcher.go @@ -29,5 +29,5 @@ type Launcher interface { // the container#s initial process. If instanceName is specified, the // container must be launched as a background instance, otherwist it must // run interactively, attached to the console. - Exec(ctx context.Context, image string, args []string, instanceName string) error + Exec(ctx context.Context, image string, cmd string, args []string, instanceName string) error } diff --git a/internal/pkg/runtime/launcher/native/launcher_linux.go b/internal/pkg/runtime/launcher/native/launcher_linux.go index 3d706ad6b3..255893f044 100644 --- a/internal/pkg/runtime/launcher/native/launcher_linux.go +++ b/internal/pkg/runtime/launcher/native/launcher_linux.go @@ -107,7 +107,7 @@ func NewLauncher(opts ...launcher.Option) (*Launcher, error) { // This includes interactive containers, instances, and joining an existing instance. // //nolint:maintidx -func (l *Launcher) Exec(ctx context.Context, image string, args []string, instanceName string) error { +func (l *Launcher) Exec(ctx context.Context, image string, cmd string, args []string, instanceName string) error { var err error var fakerootPath string @@ -182,6 +182,9 @@ func (l *Launcher) Exec(ctx context.Context, image string, args []string, instan } } + // Native runtime expects command to execute as arg[0] + args = append([]string{cmd}, args...) + // Set arguments to pass to contained process. l.generator.SetProcessArgs(args) diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index 8169dda206..88dbbf3fc3 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -20,6 +20,7 @@ import ( "github.com/apptainer/apptainer/internal/pkg/buildcfg" "github.com/apptainer/apptainer/internal/pkg/runtime/launcher" + "github.com/google/uuid" ) var ( @@ -233,7 +234,23 @@ func checkOpts(lo launcher.Options) error { return nil } -// Exec is not yet implemented. -func (l *Launcher) Exec(ctx context.Context, image string, args []string, instanceName string) error { - return ErrNotImplemented +// Exec will interactively execute a container via the runc low-level runtime. +func (l *Launcher) Exec(ctx context.Context, image string, cmd string, args []string, instanceName string) error { + if instanceName != "" { + return fmt.Errorf("%w: instanceName", ErrNotImplemented) + } + + if cmd != "" { + return fmt.Errorf("%w: cmd %v", ErrNotImplemented, cmd) + } + + if len(args) > 0 { + return fmt.Errorf("%w: args %v", ErrNotImplemented, args) + } + + id, err := uuid.NewRandom() + if err != nil { + return fmt.Errorf("while generating container id: %w", err) + } + return Run(ctx, id.String(), image, "") } diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux_test.go b/internal/pkg/runtime/launcher/oci/launcher_linux_test.go index e75452fb41..a64b87e1be 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux_test.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux_test.go @@ -10,7 +10,6 @@ package oci import ( - "context" "reflect" "testing" @@ -58,14 +57,3 @@ func TestNewLauncher(t *testing.T) { }) } } - -func TestExec(t *testing.T) { - l, err := NewLauncher([]launcher.Option{}...) - if err != nil { - t.Errorf("Couldn't initialize launcher: %s", err) - } - - if err := l.Exec(context.Background(), "", []string{}, ""); err != ErrNotImplemented { - t.Errorf("Expected %v, got %v", ErrNotImplemented, err) - } -} diff --git a/internal/pkg/runtime/launcher/oci/oci_conmon_linux.go b/internal/pkg/runtime/launcher/oci/oci_conmon_linux.go index dcf2cde0cd..0649c83172 100644 --- a/internal/pkg/runtime/launcher/oci/oci_conmon_linux.go +++ b/internal/pkg/runtime/launcher/oci/oci_conmon_linux.go @@ -43,7 +43,7 @@ func Create(containerID, bundlePath string) error { if err != nil { return err } - runc, err := bin.FindBin("runc") + runtimeBin, err := runtime() if err != nil { return err } @@ -96,12 +96,12 @@ func Create(containerID, bundlePath string) error { "--cid", containerID, "--name", containerID, "--cuuid", containerUUID.String(), - "--runtime", runc, + "--runtime", runtimeBin, "--conmon-pidfile", path.Join(sd, conmonPidFile), "--container-pidfile", path.Join(sd, containerPidFile), "--log-path", path.Join(sd, containerLogFile), "--runtime-arg", "--root", - "--runtime-arg", runcStateDir, + "--runtime-arg", runtimeStateDir(), "--runtime-arg", "--log", "--runtime-arg", path.Join(sd, runcLogFile), "--full-attach", diff --git a/internal/pkg/runtime/launcher/oci/oci_linux.go b/internal/pkg/runtime/launcher/oci/oci_linux.go index 5298bf2849..af219dcb4d 100644 --- a/internal/pkg/runtime/launcher/oci/oci_linux.go +++ b/internal/pkg/runtime/launcher/oci/oci_linux.go @@ -19,16 +19,16 @@ import ( "path/filepath" "time" + "github.com/apptainer/apptainer/internal/pkg/util/bin" "github.com/apptainer/apptainer/internal/pkg/util/fs" "github.com/apptainer/apptainer/internal/pkg/util/user" "github.com/apptainer/apptainer/pkg/syfs" + "github.com/apptainer/apptainer/pkg/sylog" "github.com/apptainer/apptainer/pkg/util/fs/lock" securejoin "github.com/cyphar/filepath-securejoin" ) const ( - // Absolute path for the runc state - runcStateDir = "/run/apptainer-oci" // Relative path inside ~/.apptainer for conmon and apptainer state ociPath = "oci" // State directory files @@ -44,6 +44,26 @@ const ( createTimeout = 30 * time.Second ) +// runtime returns path to the OCI runtime - crun (preferred), or runc. +func runtime() (path string, err error) { + path, err = bin.FindBin("crun") + if err == nil { + return + } + sylog.Debugf("While finding crun: %s", err) + sylog.Warningf("crun not found. Will attempt to use runc, but not all functionality is supported.") + return bin.FindBin("runc") +} + +// runtimeStateDir returns path to use for crun/runc's state handling. +func runtimeStateDir() string { + uid := os.Getuid() + if uid == 0 { + return "/run/apptainer-oci" + } + return fmt.Sprintf("/run/user/%d/apptainer-oci", uid) +} + // stateDir returns the path to container state handled by conmon/apptainer // (as opposed to runc's state in RuncStateDir) func stateDir(containerID string) (string, error) { diff --git a/internal/pkg/runtime/launcher/oci/oci_runc_linux.go b/internal/pkg/runtime/launcher/oci/oci_runc_linux.go index d96a1a702e..cb1fff97b9 100644 --- a/internal/pkg/runtime/launcher/oci/oci_runc_linux.go +++ b/internal/pkg/runtime/launcher/oci/oci_runc_linux.go @@ -25,21 +25,21 @@ import ( // Delete deletes container resources func Delete(ctx context.Context, containerID string) error { - runc, err := bin.FindBin("runc") + runtimeBin, err := runtime() if err != nil { return err } - runcArgs := []string{ - "--root", runcStateDir, + runtimeArgs := []string{ + "--root", runtimeStateDir(), "delete", containerID, } - cmd := exec.Command(runc, runcArgs...) + cmd := exec.Command(runtimeBin, runtimeArgs...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr cmd.Stdin = os.Stdout - sylog.Debugf("Calling runc with args %v", runcArgs) + sylog.Debugf("Calling %s with args %v", runtimeBin, runtimeArgs) err = cmd.Run() if err != nil { return fmt.Errorf("while calling runc delete: %w", err) @@ -67,88 +67,88 @@ func Delete(ctx context.Context, containerID string) error { // Exec executes a command in a container func Exec(containerID string, cmdArgs []string) error { - runc, err := bin.FindBin("runc") + runtimeBin, err := runtime() if err != nil { return err } - runcArgs := []string{ - "--root", runcStateDir, + runtimeArgs := []string{ + "--root", runtimeStateDir(), "exec", containerID, } - runcArgs = append(runcArgs, cmdArgs...) - cmd := exec.Command(runc, runcArgs...) + runtimeArgs = append(runtimeArgs, cmdArgs...) + cmd := exec.Command(runtimeBin, runtimeArgs...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr cmd.Stdin = os.Stdout - sylog.Debugf("Calling runc with args %v", runcArgs) + sylog.Debugf("Calling %s with args %v", runtimeBin, runtimeArgs) return cmd.Run() } // Kill kills container process func Kill(containerID string, killSignal string) error { - runc, err := bin.FindBin("runc") + runtimeBin, err := runtime() if err != nil { return err } - runcArgs := []string{ - "--root", runcStateDir, + runtimeArgs := []string{ + "--root", runtimeStateDir(), "kill", containerID, killSignal, } - cmd := exec.Command(runc, runcArgs...) + cmd := exec.Command(runtimeBin, runtimeArgs...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr cmd.Stdin = os.Stdout - sylog.Debugf("Calling runc with args %v", runcArgs) + sylog.Debugf("Calling %s with args %v", runtimeBin, runtimeArgs) return cmd.Run() } // Pause pauses processes in a container func Pause(containerID string) error { - runc, err := bin.FindBin("runc") + runtimeBin, err := runtime() if err != nil { return err } - runcArgs := []string{ - "--root", runcStateDir, + runtimeArgs := []string{ + "--root", runtimeStateDir(), "pause", containerID, } - cmd := exec.Command(runc, runcArgs...) + cmd := exec.Command(runtimeBin, runtimeArgs...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr cmd.Stdin = os.Stdout - sylog.Debugf("Calling runc with args %v", runcArgs) + sylog.Debugf("Calling %s with args %v", runtimeBin, runtimeArgs) return cmd.Run() } // Resume pauses processes in a container func Resume(containerID string) error { - runc, err := bin.FindBin("runc") + runtimeBin, err := runtime() if err != nil { return err } - runcArgs := []string{ - "--root", runcStateDir, + runtimeArgs := []string{ + "--root", runtimeStateDir(), "resume", containerID, } - cmd := exec.Command(runc, runcArgs...) + cmd := exec.Command(runtimeBin, runtimeArgs...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr cmd.Stdin = os.Stdout - sylog.Debugf("Calling runc with args %v", runcArgs) + sylog.Debugf("Calling %s with args %v", runtimeBin, runtimeArgs) return cmd.Run() } // Run runs a container (equivalent to create/start/delete) func Run(ctx context.Context, containerID, bundlePath, pidFile string) error { - runc, err := bin.FindBin("runc") + runtimeBin, err := runtime() if err != nil { return err } @@ -161,80 +161,80 @@ func Run(ctx context.Context, containerID, bundlePath, pidFile string) error { return fmt.Errorf("failed to change directory to %s: %s", absBundle, err) } - runcArgs := []string{ - "--root", runcStateDir, + runtimeArgs := []string{ + "--root", runtimeStateDir(), "run", "-b", absBundle, } if pidFile != "" { - runcArgs = append(runcArgs, "--pid-file="+pidFile) + runtimeArgs = append(runtimeArgs, "--pid-file="+pidFile) } - runcArgs = append(runcArgs, containerID) - cmd := exec.Command(runc, runcArgs...) + runtimeArgs = append(runtimeArgs, containerID) + cmd := exec.Command(runtimeBin, runtimeArgs...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr cmd.Stdin = os.Stdout - sylog.Debugf("Calling runc with args %v", runcArgs) + sylog.Debugf("Calling %s with args %v", runtimeBin, runtimeArgs) return cmd.Run() } // Start starts a previously created container func Start(containerID string) error { - runc, err := bin.FindBin("runc") + runtimeBin, err := bin.FindBin("crun") if err != nil { return err } - runcArgs := []string{ - "--root", runcStateDir, + runtimeArgs := []string{ + "--root", runtimeStateDir(), "start", containerID, } - cmd := exec.Command(runc, runcArgs...) + cmd := exec.Command(runtimeBin, runtimeArgs...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr cmd.Stdin = os.Stdout - sylog.Debugf("Calling runc with args %v", runcArgs) + sylog.Debugf("Calling %s with args %v", runtimeBin, runtimeArgs) return cmd.Run() } // State queries container state func State(containerID string) error { - runc, err := bin.FindBin("runc") + runtimeBin, err := runtime() if err != nil { return err } - runcArgs := []string{ - "--root", runcStateDir, + runtimeArgs := []string{ + "--root", runtimeStateDir(), "state", containerID, } - cmd := exec.Command(runc, runcArgs...) + cmd := exec.Command(runtimeBin, runtimeArgs...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr cmd.Stdin = os.Stdout - sylog.Debugf("Calling runc with args %v", runcArgs) + sylog.Debugf("Calling %s with args %v", runtimeBin, runtimeArgs) return cmd.Run() } // Update updates container cgroups resources func Update(containerID, cgFile string) error { - runc, err := bin.FindBin("runc") + runtimeBin, err := runtime() if err != nil { return err } - runcArgs := []string{ - "--root", runcStateDir, + runtimeArgs := []string{ + "--root", runtimeStateDir(), "update", "-r", cgFile, containerID, } - cmd := exec.Command(runc, runcArgs...) + cmd := exec.Command(runtimeBin, runtimeArgs...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr cmd.Stdin = os.Stdout - sylog.Debugf("Calling runc with args %v", runcArgs) + sylog.Debugf("Calling %s with args %v", runtimeBin, runtimeArgs) return cmd.Run() } diff --git a/internal/pkg/util/bin/bin.go b/internal/pkg/util/bin/bin.go index f91a0c2548..5b36df5820 100644 --- a/internal/pkg/util/bin/bin.go +++ b/internal/pkg/util/bin/bin.go @@ -48,6 +48,7 @@ func FindBin(name string) (path string, err error) { // All other executables // We will always search the user's PATH first for these case "conmon", + "crun", "curl", "debootstrap", "dnf", From d5a958f59faace1eb67bfda0d727a09494b51c47 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Tue, 1 Nov 2022 09:28:39 +0000 Subject: [PATCH 016/114] e2e: oci: add subtests, move help to parallel Add subtest information to the large multi-step OCI test functions for easier debugging. Move the test of `oci help` out of the SEQ/ordered section. It can run in parallel, and only once regardless of cgroups managers available. Signed-off-by: Edita Kizinevic --- e2e/oci/oci.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/e2e/oci/oci.go b/e2e/oci/oci.go index bed8aa84de..8845910653 100644 --- a/e2e/oci/oci.go +++ b/e2e/oci/oci.go @@ -93,6 +93,7 @@ func genericOciMount(t *testing.T, c *ctx) (string, func()) { } c.env.RunApptainer( t, + e2e.AsSubtest("mount"), e2e.WithProfile(e2e.RootProfile), e2e.WithCommand("oci mount"), e2e.WithArgs(c.env.ImagePath, bundleDir), @@ -102,6 +103,7 @@ func genericOciMount(t *testing.T, c *ctx) (string, func()) { cleanup := func() { c.env.RunApptainer( t, + e2e.AsSubtest("umount"), e2e.WithProfile(e2e.RootProfile), e2e.WithCommand("oci umount"), e2e.WithArgs(bundleDir), @@ -158,6 +160,7 @@ func (c ctx) testOciAttach(t *testing.T) { c.env.RunApptainer( t, + e2e.AsSubtest("create"), e2e.WithProfile(e2e.RootProfile), e2e.WithCommand("oci create"), e2e.WithArgs("-b", bundleDir, containerID), @@ -176,6 +179,7 @@ func (c ctx) testOciAttach(t *testing.T) { c.env.RunApptainer( t, + e2e.AsSubtest("start"), e2e.WithProfile(e2e.RootProfile), e2e.WithCommand("oci start"), e2e.WithArgs(containerID), @@ -189,6 +193,7 @@ func (c ctx) testOciAttach(t *testing.T) { c.env.RunApptainer( t, + e2e.AsSubtest("attach"), e2e.WithProfile(e2e.RootProfile), e2e.WithCommand("oci attach"), e2e.WithArgs(containerID), @@ -207,6 +212,7 @@ func (c ctx) testOciAttach(t *testing.T) { c.env.RunApptainer( t, + e2e.AsSubtest("delete"), e2e.WithProfile(e2e.RootProfile), e2e.WithCommand("oci delete"), e2e.WithArgs(containerID), @@ -225,6 +231,7 @@ func (c ctx) testOciBasic(t *testing.T) { c.env.RunApptainer( t, + e2e.AsSubtest("create"), e2e.WithProfile(e2e.RootProfile), e2e.WithCommand("oci create"), e2e.WithArgs("-b", bundleDir, containerID), @@ -243,6 +250,7 @@ func (c ctx) testOciBasic(t *testing.T) { c.env.RunApptainer( t, + e2e.AsSubtest("start"), e2e.WithProfile(e2e.RootProfile), e2e.WithCommand("oci start"), e2e.WithArgs(containerID), @@ -256,6 +264,7 @@ func (c ctx) testOciBasic(t *testing.T) { c.env.RunApptainer( t, + e2e.AsSubtest("pause"), e2e.WithProfile(e2e.RootProfile), e2e.WithCommand("oci pause"), e2e.WithArgs(containerID), @@ -273,6 +282,7 @@ func (c ctx) testOciBasic(t *testing.T) { c.env.RunApptainer( t, + e2e.AsSubtest("resume"), e2e.WithProfile(e2e.RootProfile), e2e.WithCommand("oci resume"), e2e.WithArgs(containerID), @@ -290,6 +300,7 @@ func (c ctx) testOciBasic(t *testing.T) { c.env.RunApptainer( t, + e2e.AsSubtest("start again"), e2e.WithProfile(e2e.RootProfile), e2e.WithCommand("oci start"), e2e.WithArgs(containerID), @@ -298,6 +309,7 @@ func (c ctx) testOciBasic(t *testing.T) { c.env.RunApptainer( t, + e2e.AsSubtest("exec"), e2e.WithProfile(e2e.RootProfile), e2e.WithCommand("oci exec"), e2e.WithArgs(containerID, "hostname"), @@ -307,6 +319,7 @@ func (c ctx) testOciBasic(t *testing.T) { c.env.RunApptainer( t, + e2e.AsSubtest("kill"), e2e.WithProfile(e2e.RootProfile), e2e.WithCommand("oci kill"), e2e.WithArgs(containerID, "KILL"), @@ -320,6 +333,7 @@ func (c ctx) testOciBasic(t *testing.T) { c.env.RunApptainer( t, + e2e.AsSubtest("delete"), e2e.WithProfile(e2e.RootProfile), e2e.WithCommand("oci delete"), e2e.WithArgs(containerID), @@ -328,6 +342,7 @@ func (c ctx) testOciBasic(t *testing.T) { c.env.RunApptainer( t, + e2e.AsSubtest("state fail"), e2e.WithProfile(e2e.RootProfile), e2e.WithCommand("oci state"), e2e.WithArgs(containerID), @@ -335,6 +350,7 @@ func (c ctx) testOciBasic(t *testing.T) { ) c.env.RunApptainer( t, + e2e.AsSubtest("kill fail"), e2e.WithProfile(e2e.RootProfile), e2e.WithCommand("oci kill"), e2e.WithArgs(containerID), @@ -342,6 +358,7 @@ func (c ctx) testOciBasic(t *testing.T) { ) c.env.RunApptainer( t, + e2e.AsSubtest("start fail"), e2e.WithProfile(e2e.RootProfile), e2e.WithCommand("oci start"), e2e.WithArgs(containerID), @@ -433,7 +450,7 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { t.Run("basic", c.testOciBasic) t.Run("attach", c.testOciAttach) t.Run("run", c.testOciRun) - t.Run("help", c.testOciHelp) })), + "help": c.testOciHelp, } } From e6e9271c0b9c436d7775b5421ff7138d5f35e7c8 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Tue, 1 Nov 2022 11:36:16 +0000 Subject: [PATCH 017/114] chore(ci): use crun 1.6 The Ubuntu packaged crun is very old. Use a release from GitHub instead. Signed-off-by: Edita Kizinevic --- .github/workflows/ci.yml | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cd748c6677..932a0a6fea 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -238,7 +238,12 @@ jobs: go-version: 1.20.5 - name: Fetch deps - run: sudo apt-get -q update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential squashfs-tools squashfuse fuse-overlayfs fakeroot fuse2fs libseccomp-dev cryptsetup dbus-user-session conmon crun + run: sudo apt-get -q update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential squashfs-tools squashfuse fuse-overlayfs fakeroot fuse2fs libseccomp-dev cryptsetup dbus-user-session conmon + + - name: Install crun + run: | + sudo curl -L -o /usr/local/bin/crun https://github.com/containers/crun/releases/download/1.6/crun-1.6-linux-amd64 + sudo chmod +x /usr/local/bin/crun - name: Build and install Apptainer run: | @@ -268,7 +273,12 @@ jobs: go-version: 1.20.5 - name: Fetch deps - run: sudo apt-get -q update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential squashfs-tools libseccomp-dev cryptsetup dbus-user-session conmon crun + run: sudo apt-get -q update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential squashfs-tools libseccomp-dev cryptsetup dbus-user-session conmon + + - name: Install crun + run: | + sudo curl -L -o /usr/local/bin/crun https://github.com/containers/crun/releases/download/1.6/crun-1.6-linux-amd64 + sudo chmod +x /usr/local/bin/crun - name: Build and install Apptainer run: | @@ -314,7 +324,12 @@ jobs: - name: Fetch deps if: env.run_tests - run: sudo apt-get -q update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential uidmap squashfs-tools squashfuse fuse-overlayfs fakeroot fuse2fs libseccomp-dev cryptsetup dbus-user-session conmon crun + run: sudo apt-get -q update && sudo DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential uidmap squashfs-tools squashfuse fuse-overlayfs fakeroot fuse2fs libseccomp-dev cryptsetup dbus-user-session conmon + + - name: Install crun + run: | + sudo curl -L -o /usr/local/bin/crun https://github.com/containers/crun/releases/download/1.6/crun-1.6-linux-amd64 + sudo chmod +x /usr/local/bin/crun - name: Fetch gocryptfs run: wget -O gocryptfs.tar.gz https://github.com/rfjakob/gocryptfs/releases/download/v2.3/gocryptfs_v2.3_linux-static_amd64.tar.gz && sudo tar xzvf gocryptfs.tar.gz -C /usr/local/bin gocryptfs From 7daac210f5447a4d497713640cb9fecff8f852ad Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Mon, 14 Nov 2022 11:59:36 +0000 Subject: [PATCH 018/114] feat: run OCI image sources via temporary bundle When running containers in `--oci` mode, the argument to run is now an image reference corresponding to a native OCI format handled by containers/image, i.e. * docker:// * docker-archive: * docker-daemon: * oci-archive: * oci: The source image is extracted into a temporary OCI bundle, with a minimally valid configuration that: * Runs the process specified by CMD & ENTRYPOINT only. * Sets the environment specified by the image ENV only. The approach is very naive - we pull through Singularity's OCI blob cache into a temporary oci layout dir, before creating the bundle from it. Auth handling for registries is not yet wired up. There is duplication of various pieces of code from the build / SIF OCI flows as these are not easily exposed to the area we are working in. The intent of the PR, at this stage, is simply to allow e.g. singularity run --oci docker://sylabsio/lolcow Closes sylabs/singularity#1036 Signed-off-by: Edita Kizinevic --- cmd/internal/cli/actions.go | 8 + cmd/internal/cli/oci_linux.go | 2 +- internal/app/apptainer/oci_linux.go | 4 +- .../runtime/launcher/oci/launcher_linux.go | 55 +++- pkg/ocibundle/bundle.go | 5 +- pkg/ocibundle/native/bundle_linux.go | 276 ++++++++++++++++++ pkg/ocibundle/sif/bundle_linux.go | 7 +- pkg/ocibundle/sif/bundle_linux_test.go | 5 +- 8 files changed, 354 insertions(+), 8 deletions(-) create mode 100644 pkg/ocibundle/native/bundle_linux.go diff --git a/cmd/internal/cli/actions.go b/cmd/internal/cli/actions.go index e9f410c527..fc9c470db8 100644 --- a/cmd/internal/cli/actions.go +++ b/cmd/internal/cli/actions.go @@ -145,6 +145,14 @@ func replaceURIWithImage(ctx context.Context, cmd *cobra.Command, args []string) sylog.Fatalf("failed to create a new image cache handle") } + // The OCI runtime launcher will handle OCI image sources directly. + if ociRuntime { + if oci.IsSupported(t) != t { + sylog.Fatalf("OCI runtime only supports OCI image sources. %s is not supported.", t) + } + return + } + switch t { case uri.Library: image, err = handleLibrary(ctx, imgCache, args[0]) diff --git a/cmd/internal/cli/oci_linux.go b/cmd/internal/cli/oci_linux.go index 545a13e907..aa4beb630f 100644 --- a/cmd/internal/cli/oci_linux.go +++ b/cmd/internal/cli/oci_linux.go @@ -322,7 +322,7 @@ var OciMountCmd = &cobra.Command{ DisableFlagsInUseLine: true, PreRun: CheckRoot, Run: func(cmd *cobra.Command, args []string) { - if err := apptainer.OciMount(args[0], args[1]); err != nil { + if err := apptainer.OciMount(cmd.Context(), args[0], args[1]); err != nil { sylog.Fatalf("%s", err) } }, diff --git a/internal/app/apptainer/oci_linux.go b/internal/app/apptainer/oci_linux.go index 9e5f16d7d7..2e1732a202 100644 --- a/internal/app/apptainer/oci_linux.go +++ b/internal/app/apptainer/oci_linux.go @@ -83,12 +83,12 @@ func OciUpdate(containerID string, args *OciArgs) error { } // OciMount mount a SIF image to create an OCI bundle -func OciMount(image string, bundle string) error { +func OciMount(ctx context.Context, image string, bundle string) error { d, err := ocibundle.FromSif(image, bundle, true) if err != nil { return err } - return d.Create(nil) + return d.Create(ctx, nil) } // OciUmount umount SIF and delete OCI bundle diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index 88dbbf3fc3..3c982db75e 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -16,10 +16,17 @@ import ( "context" "errors" "fmt" + "os" "strings" "github.com/apptainer/apptainer/internal/pkg/buildcfg" + "github.com/apptainer/apptainer/internal/pkg/cache" "github.com/apptainer/apptainer/internal/pkg/runtime/launcher" + "github.com/apptainer/apptainer/pkg/ocibundle/native" + "github.com/apptainer/apptainer/pkg/syfs" + "github.com/apptainer/apptainer/pkg/sylog" + useragent "github.com/apptainer/apptainer/pkg/util/user-agent" + "github.com/containers/image/v5/types" "github.com/google/uuid" ) @@ -235,6 +242,7 @@ func checkOpts(lo launcher.Options) error { } // Exec will interactively execute a container via the runc low-level runtime. +// image is a reference to an OCI image, e.g. docker://ubuntu or oci:/tmp/mycontainer func (l *Launcher) Exec(ctx context.Context, image string, cmd string, args []string, instanceName string) error { if instanceName != "" { return fmt.Errorf("%w: instanceName", ErrNotImplemented) @@ -248,9 +256,54 @@ func (l *Launcher) Exec(ctx context.Context, image string, cmd string, args []st return fmt.Errorf("%w: args %v", ErrNotImplemented, args) } + bundleDir, err := os.MkdirTemp("", "oci-bundle") + if err != nil { + return nil + } + defer func() { + sylog.Debugf("Removing OCI bundle at: %s", bundleDir) + if err := os.RemoveAll(bundleDir); err != nil { + sylog.Errorf("Couldn't remove OCI bundle %s: %v", bundleDir, err) + } + }() + + sylog.Debugf("Creating OCI bundle at: %s", bundleDir) + + // TODO - propagate auth config + sysCtx := &types.SystemContext{ + // OCIInsecureSkipTLSVerify: cp.b.Opts.NoHTTPS, + // DockerAuthConfig: cp.b.Opts.DockerAuthConfig, + // DockerDaemonHost: cp.b.Opts.DockerDaemonHost, + OSChoice: "linux", + AuthFilePath: syfs.DockerConf(), + DockerRegistryUserAgent: useragent.Value(), + } + // if cp.b.Opts.NoHTTPS { + // cp.sysCtx.DockerInsecureSkipTLSVerify = types.NewOptionalBool(true) + // } + + var imgCache *cache.Handle + if !l.cfg.CacheDisabled { + imgCache, err = cache.New(cache.Config{ + ParentDir: os.Getenv(cache.DirEnv), + }) + if err != nil { + return err + } + } + + b, err := native.FromImageRef(image, bundleDir, sysCtx, imgCache) + if err != nil { + return err + } + + if err := b.Create(ctx, nil); err != nil { + return err + } + id, err := uuid.NewRandom() if err != nil { return fmt.Errorf("while generating container id: %w", err) } - return Run(ctx, id.String(), image, "") + return Run(ctx, id.String(), b.Path(), "") } diff --git a/pkg/ocibundle/bundle.go b/pkg/ocibundle/bundle.go index 8777c56d1e..a1e7dc9631 100644 --- a/pkg/ocibundle/bundle.go +++ b/pkg/ocibundle/bundle.go @@ -10,11 +10,14 @@ package ocibundle import ( + "context" + "github.com/opencontainers/runtime-spec/specs-go" ) // Bundle defines an OCI bundle interface to create/delete OCI bundles type Bundle interface { - Create(*specs.Spec) error + Create(context.Context, *specs.Spec) error Delete() error + Path() string } diff --git a/pkg/ocibundle/native/bundle_linux.go b/pkg/ocibundle/native/bundle_linux.go new file mode 100644 index 0000000000..fc0010826e --- /dev/null +++ b/pkg/ocibundle/native/bundle_linux.go @@ -0,0 +1,276 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2022, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package native + +import ( + "context" + "encoding/json" + "fmt" + "os" + "path/filepath" + "strings" + + apexlog "github.com/apex/log" + "github.com/apptainer/apptainer/internal/pkg/build/oci" + "github.com/apptainer/apptainer/internal/pkg/cache" + "github.com/apptainer/apptainer/internal/pkg/runtime/engine/config/oci/generate" + "github.com/apptainer/apptainer/pkg/ocibundle" + "github.com/apptainer/apptainer/pkg/ocibundle/tools" + "github.com/apptainer/apptainer/pkg/sylog" + "github.com/containers/image/v5/copy" + "github.com/containers/image/v5/docker" + dockerarchive "github.com/containers/image/v5/docker/archive" + dockerdaemon "github.com/containers/image/v5/docker/daemon" + ociarchive "github.com/containers/image/v5/oci/archive" + ocilayout "github.com/containers/image/v5/oci/layout" + "github.com/containers/image/v5/signature" + "github.com/containers/image/v5/types" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/opencontainers/runtime-spec/specs-go" + "github.com/opencontainers/umoci" + umocilayer "github.com/opencontainers/umoci/oci/layer" + "github.com/opencontainers/umoci/pkg/idtools" +) + +// Bundle is a native OCI bundle, created from imageRef. +type Bundle struct { + // imageRef is the reference to the OCI image source, e.g. docker://ubuntu:latest. + imageRef string + // imageSpec is the OCI image information, CMD, ENTRYPOINT, etc. + imageSpec *imgspecv1.Image + // bundlePath is the location where the OCI bundle will be created. + bundlePath string + // sysCtx provides containers/image transport configuration (auth etc.) + sysCtx *types.SystemContext + // imgCache is a Apptainer image cache, which OCI blobs are pulled through. + // Note that we only use the 'blob' cache section. The 'oci-tmp' cache section holds + // OCI->SIF conversions, which are not used here. + imgCache *cache.Handle + // Generic bundle properties + ocibundle.Bundle +} + +func (b *Bundle) Path() string { + return b.bundlePath +} + +func (b *Bundle) writeConfig(g *generate.Generator) error { + return tools.SaveBundleConfig(b.bundlePath, g) +} + +func (b *Bundle) fetchImage(ctx context.Context, tmpDir string) error { + if b.sysCtx == nil { + return fmt.Errorf("sysctx must be provided") + } + + policy := &signature.Policy{Default: []signature.PolicyRequirement{signature.NewPRInsecureAcceptAnything()}} + policyCtx, err := signature.NewPolicyContext(policy) + if err != nil { + return err + } + + parts := strings.SplitN(b.imageRef, ":", 2) + if len(parts) < 2 { + return fmt.Errorf("could not parse image ref: %s", b.imageRef) + } + var srcRef types.ImageReference + + switch parts[0] { + case "docker": + srcRef, err = docker.ParseReference(parts[1]) + case "docker-archive": + srcRef, err = dockerarchive.ParseReference(parts[1]) + case "docker-daemon": + srcRef, err = dockerdaemon.ParseReference(parts[1]) + case "oci": + srcRef, err = ocilayout.ParseReference(parts[1]) + case "oci-archive": + srcRef, err = ociarchive.ParseReference(parts[1]) + default: + return fmt.Errorf("cannot create an OCI container from %s source", parts[0]) + } + + if err != nil { + return fmt.Errorf("invalid image source: %w", err) + } + + if b.imgCache != nil { + // Grab the modified source ref from the cache + srcRef, err = oci.ConvertReference(ctx, b.imgCache, srcRef, b.sysCtx) + if err != nil { + return err + } + } + + tmpfsRef, err := ocilayout.ParseReference(tmpDir + ":" + "tmp") + if err != nil { + return err + } + + _, err = copy.Image(ctx, policyCtx, tmpfsRef, srcRef, ©.Options{ + ReportWriter: sylog.Writer(), + SourceCtx: b.sysCtx, + }) + if err != nil { + return err + } + + img, err := srcRef.NewImage(ctx, b.sysCtx) + if err != nil { + return err + } + defer img.Close() + + b.imageSpec, err = img.OCIConfig(ctx) + if err != nil { + return err + } + return nil +} + +func (b *Bundle) extractImage(ctx context.Context, tmpDir string) error { + var mapOptions umocilayer.MapOptions + + loggerLevel := sylog.GetLevel() + // set the apex log level, for umoci + if loggerLevel <= int(sylog.ErrorLevel) { + // silent option + apexlog.SetLevel(apexlog.ErrorLevel) + } else if loggerLevel <= int(sylog.LogLevel) { + // quiet option + apexlog.SetLevel(apexlog.WarnLevel) + } else if loggerLevel < int(sylog.DebugLevel) { + // verbose option(s) or default + apexlog.SetLevel(apexlog.InfoLevel) + } else { + // debug option + apexlog.SetLevel(apexlog.DebugLevel) + } + + // Allow unpacking as non-root + if os.Geteuid() != 0 { + mapOptions.Rootless = true + + uidMap, err := idtools.ParseMapping(fmt.Sprintf("0:%d:1", os.Geteuid())) + if err != nil { + return fmt.Errorf("error parsing uidmap: %s", err) + } + mapOptions.UIDMappings = append(mapOptions.UIDMappings, uidMap) + + gidMap, err := idtools.ParseMapping(fmt.Sprintf("0:%d:1", os.Getegid())) + if err != nil { + return fmt.Errorf("error parsing gidmap: %s", err) + } + mapOptions.GIDMappings = append(mapOptions.GIDMappings, gidMap) + } + + engineExt, err := umoci.OpenLayout(tmpDir) + if err != nil { + return fmt.Errorf("error opening layout: %s", err) + } + + // Obtain the manifest + tmpfsRef, err := ocilayout.ParseReference(tmpDir + ":" + "tmp") + if err != nil { + return err + } + imageSource, err := tmpfsRef.NewImageSource(ctx, b.sysCtx) + if err != nil { + return fmt.Errorf("error creating image source: %s", err) + } + manifestData, mediaType, err := imageSource.GetManifest(ctx, nil) + if err != nil { + return fmt.Errorf("error obtaining manifest source: %s", err) + } + if mediaType != imgspecv1.MediaTypeImageManifest { + return fmt.Errorf("error verifying manifest media type: %s", mediaType) + } + var manifest imgspecv1.Manifest + json.Unmarshal(manifestData, &manifest) + + // UnpackRootfs from umoci v0.4.2 expects a path to a non-existing directory + os.RemoveAll(tools.RootFs(b.bundlePath).Path()) + + // Unpack root filesystem + unpackOptions := umocilayer.UnpackOptions{MapOptions: mapOptions} + err = umocilayer.UnpackRootfs(ctx, engineExt, tools.RootFs(b.bundlePath).Path(), manifest, &unpackOptions) + if err != nil { + return fmt.Errorf("error unpacking rootfs: %s", err) + } + + return nil +} + +// Create will created the on-disk structures for the OCI bundle, so that it is ready for execution. +func (b *Bundle) Create(ctx context.Context, ociConfig *specs.Spec) error { + // generate OCI bundle directory and config + g, err := tools.GenerateBundleConfig(b.bundlePath, ociConfig) + if err != nil { + return fmt.Errorf("failed to generate OCI bundle/config: %s", err) + } + // Due to our caching approach for OCI blobs, we need to pull blobs for the image + // out into a separate oci-layout directory. + tmpDir, err := os.MkdirTemp("", "oci-tmp") + if err != nil { + return err + } + defer os.RemoveAll(tmpDir) + // Fetch into temp oci layout (will pull through cache if enabled) + if err := b.fetchImage(ctx, tmpDir); err != nil { + return err + } + // Extract from temp oci layout into bundle rootfs + if err := b.extractImage(ctx, tmpDir); err != nil { + return err + } + // Remove the temp oci layout. + if err := os.RemoveAll(tmpDir); err != nil { + return err + } + + // Set non-root uid/gid per Apptainer defaults + uid := uint32(os.Getuid()) + if uid != 0 { + gid := uint32(os.Getgid()) + g.Config.Process.User.UID = uid + g.Config.Process.User.GID = gid + } + // Set default ENV from image + g.Config.Process.Env = append(g.Config.Process.Env, b.imageSpec.Config.Env...) + // Set default exec from image CMD & Entrypoint + if b.imageSpec == nil { + return fmt.Errorf("imageSpec cannot be nil") + } + args := append(b.imageSpec.Config.Entrypoint, b.imageSpec.Config.Cmd...) + g.SetProcessArgs(args) + return b.writeConfig(g) +} + +// Delete erases OCI bundle created an OCI image ref +func (b *Bundle) Delete() error { + return tools.DeleteBundle(b.bundlePath) +} + +// FromImageRef returns a bundle interface to create/delete an OCI bundle from an OCI image ref. +func FromImageRef(imageRef, bundle string, sysCtx *types.SystemContext, imgCache *cache.Handle) (ocibundle.Bundle, error) { + var err error + + b := &Bundle{ + imageRef: imageRef, + sysCtx: sysCtx, + imgCache: imgCache, + } + b.bundlePath, err = filepath.Abs(bundle) + if err != nil { + return nil, fmt.Errorf("failed to determine bundle path: %s", err) + } + + return b, nil +} diff --git a/pkg/ocibundle/sif/bundle_linux.go b/pkg/ocibundle/sif/bundle_linux.go index 5c15a7d137..5b46feb208 100644 --- a/pkg/ocibundle/sif/bundle_linux.go +++ b/pkg/ocibundle/sif/bundle_linux.go @@ -10,6 +10,7 @@ package sifbundle import ( + "context" "encoding/json" "fmt" "os" @@ -92,7 +93,7 @@ func (s *sifBundle) writeConfig(img *image.Image, g *generate.Generator) error { } // Create creates an OCI bundle from a SIF image -func (s *sifBundle) Create(ociConfig *specs.Spec) error { +func (s *sifBundle) Create(ctx context.Context, ociConfig *specs.Spec) error { if s.image == "" { return fmt.Errorf("image wasn't set, need one to create bundle") } @@ -172,6 +173,10 @@ func (s *sifBundle) Delete() error { return tools.DeleteBundle(s.bundlePath) } +func (s *sifBundle) Path() string { + return s.bundlePath +} + // FromSif returns a bundle interface to create/delete OCI bundle from SIF image func FromSif(image, bundle string, writable bool) (ocibundle.Bundle, error) { var err error diff --git a/pkg/ocibundle/sif/bundle_linux_test.go b/pkg/ocibundle/sif/bundle_linux_test.go index 381340f3ff..3018d52d2a 100644 --- a/pkg/ocibundle/sif/bundle_linux_test.go +++ b/pkg/ocibundle/sif/bundle_linux_test.go @@ -10,6 +10,7 @@ package sifbundle import ( + "context" "os" "runtime" "testing" @@ -48,7 +49,7 @@ func TestFromSif(t *testing.T) { t.Errorf("unexpected success while opening non existent image") } // create OCI bundle from SIF - if err := bundle.Create(nil); err == nil { + if err := bundle.Create(context.Background(), nil); err == nil { // check if cleanup occurred t.Errorf("unexpected success while creating OCI bundle") } @@ -80,7 +81,7 @@ func TestFromSif(t *testing.T) { g.Config.Linux.Seccomp = nil g.SetProcessArgs([]string{tools.RunScript, "id"}) - if err := bundle.Create(g.Config); err != nil { + if err := bundle.Create(context.Background(), g.Config); err != nil { // check if cleanup occurred t.Fatal(err) } From b4a31abc259413a2132326cd07396db91489fd2e Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Tue, 15 Nov 2022 14:19:12 +0000 Subject: [PATCH 019/114] test: Add native OCI bundle verification test Simple test to create a native oci bundle from the supported sources, and verify it is valid with runtime-tools/validate. Signed-off-by: Edita Kizinevic --- pkg/ocibundle/native/bundle_linux_test.go | 140 ++++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 pkg/ocibundle/native/bundle_linux_test.go diff --git a/pkg/ocibundle/native/bundle_linux_test.go b/pkg/ocibundle/native/bundle_linux_test.go new file mode 100644 index 0000000000..6cd0a840d5 --- /dev/null +++ b/pkg/ocibundle/native/bundle_linux_test.go @@ -0,0 +1,140 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2022, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package native + +import ( + "context" + "io" + "log" + "net/http" + "os" + "os/exec" + "testing" + + "github.com/apptainer/apptainer/internal/pkg/cache" + "github.com/containers/image/v5/types" + "github.com/opencontainers/runtime-tools/validate" +) + +const ( + dockerURI = "docker://alpine" + dockerArchiveURI = "https://github.com/apptainer/apptainer/releases/download/v0.1.0/alpine-docker-save.tar" + ociArchiveURI = "https://github.com/apptainer/apptainer/releases/download/v0.1.0/alpine-oci-archive.tar" + dockerDaemonImage = "alpine:latest" +) + +func setupCache(t *testing.T) *cache.Handle { + dir := t.TempDir() + h, err := cache.New(cache.Config{ParentDir: dir}) + if err != nil { + t.Fatalf("failed to create an image cache handle: %s", err) + } + return h +} + +func getTestTar(url string) (path string, err error) { + dl, err := os.CreateTemp("", "oci-test") + if err != nil { + log.Fatal(err) + } + defer dl.Close() + + r, err := http.Get(url) + if err != nil { + return "", err + } + defer r.Body.Close() + + _, err = io.Copy(dl, r.Body) + if err != nil { + return "", err + } + + return dl.Name(), nil +} + +func validateBundle(t *testing.T, bundlePath string) { + v, err := validate.NewValidatorFromPath(bundlePath, false, "linux") + if err != nil { + t.Errorf("Could not create bundle validator: %v", err) + } + if err := v.CheckAll(); err != nil { + t.Errorf("Bundle not valid: %v", err) + } +} + +func TestFromImageRef(t *testing.T) { + if testing.Short() { + t.SkipNow() + } + + // Prepare docker-archive source + dockerArchive, err := getTestTar(dockerArchiveURI) + if err != nil { + t.Fatalf("Could not download docker archive test file: %v", err) + } + defer os.Remove(dockerArchive) + // Prepare docker-daemon source + hasDocker := false + cmd := exec.Command("docker", "ps") + err = cmd.Run() + if err == nil { + hasDocker = true + cmd = exec.Command("docker", "pull", dockerDaemonImage) + err = cmd.Run() + if err != nil { + t.Fatalf("could not docker pull %s %v", dockerDaemonImage, err) + return + } + } + // Prepare oci-archive source + ociArchive, err := getTestTar(ociArchiveURI) + if err != nil { + t.Fatalf("Could not download oci archive test file: %v", err) + } + defer os.Remove(ociArchive) + // Prepare oci source (oci directory layout) + ociLayout := t.TempDir() + cmd = exec.Command("tar", "-C", ociLayout, "-xf", ociArchive) + err = cmd.Run() + if err != nil { + t.Fatalf("Error extracting oci archive to layout: %v", err) + } + + tests := []struct { + name string + imageRef string + needsDocker bool + }{ + {"docker", dockerURI, false}, + {"docker-archive", "docker-archive:" + dockerArchive, false}, + {"docker-daemon", "docker-daemon:" + dockerDaemonImage, true}, + {"oci-archive", "oci-archive:" + ociArchive, false}, + {"oci", "oci:" + ociLayout, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.needsDocker && !hasDocker { + t.Skipf("docker not available") + } + bundleDir := t.TempDir() + b, err := FromImageRef(tt.imageRef, bundleDir, &types.SystemContext{}, setupCache(t)) + if err != nil { + t.Fatalf("While initializing bundle: %s", err) + } + + if err := b.Create(context.Background(), nil); err != nil { + t.Fatalf("While creating bundle: %s", err) + } + + validateBundle(t, bundleDir) + }) + } +} From 365100d923bdd956743f295e286683932256b9f7 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Tue, 15 Nov 2022 14:44:36 +0000 Subject: [PATCH 020/114] e2e: test for run --oci from oci sources Signed-off-by: Edita Kizinevic --- e2e/actions/oci.go | 118 +++++++++++++++++++++++++++------------------ 1 file changed, 72 insertions(+), 46 deletions(-) diff --git a/e2e/actions/oci.go b/e2e/actions/oci.go index c49c9d3f26..afc1e2179f 100644 --- a/e2e/actions/oci.go +++ b/e2e/actions/oci.go @@ -10,73 +10,99 @@ package actions import ( + "io" + "log" + "net/http" "os" + "os/exec" "testing" "github.com/apptainer/apptainer/e2e/internal/e2e" - "github.com/apptainer/apptainer/internal/pkg/test/tool/require" - "github.com/pkg/errors" ) -func (c actionTests) ociBundle(t *testing.T) (string, func()) { - require.Seccomp(t) - require.Filesystem(t, "overlay") +const ( + dockerArchiveURI = "https://github.com/apptainer/apptainer/releases/download/v0.1.0/alpine-docker-save.tar" + ociArchiveURI = "https://github.com/apptainer/apptainer/releases/download/v0.1.0/alpine-oci-archive.tar" +) + +func getTestTar(url string) (path string, err error) { + dl, err := os.CreateTemp("", "oci-test") + if err != nil { + log.Fatal(err) + } + defer dl.Close() - bundleDir, err := os.MkdirTemp(c.env.TestDir, "bundle-") + r, err := http.Get(url) if err != nil { - err = errors.Wrapf(err, "creating temporary bundle directory at %q", c.env.TestDir) - t.Fatalf("failed to create bundle directory: %+v", err) + return "", err } - c.env.RunApptainer( - t, - e2e.WithProfile(e2e.RootProfile), - e2e.WithCommand("oci mount"), - e2e.WithArgs(c.env.ImagePath, bundleDir), - e2e.ExpectExit(0), - ) + defer r.Body.Close() - cleanup := func() { - c.env.RunApptainer( - t, - e2e.WithProfile(e2e.RootProfile), - e2e.WithCommand("oci umount"), - e2e.WithArgs(bundleDir), - e2e.ExpectExit(0), - ) - os.RemoveAll(bundleDir) + _, err = io.Copy(dl, r.Body) + if err != nil { + return "", err } - return bundleDir, cleanup + return dl.Name(), nil } func (c actionTests) actionOciRun(t *testing.T) { - e2e.EnsureImage(t, c.env) - - bundle, cleanup := c.ociBundle(t) - defer cleanup() + // Prepare docker-archive source + dockerArchive, err := getTestTar(dockerArchiveURI) + if err != nil { + t.Fatalf("Could not download docker archive test file: %v", err) + } + defer os.Remove(dockerArchive) + // Prepare oci-archive source + ociArchive, err := getTestTar(ociArchiveURI) + if err != nil { + t.Fatalf("Could not download oci archive test file: %v", err) + } + defer os.Remove(ociArchive) + // Prepare oci source (oci directory layout) + ociLayout := t.TempDir() + cmd := exec.Command("tar", "-C", ociLayout, "-xf", ociArchive) + err = cmd.Run() + if err != nil { + t.Fatalf("Error extracting oci archive to layout: %v", err) + } tests := []struct { - name string - argv []string - exit int + name string + imageRef string + exit int }{ { - name: "NoCommand", - argv: []string{bundle}, - exit: 0, + name: "docker-archive", + imageRef: "docker-archive:" + dockerArchive, + exit: 0, + }, + { + name: "oci-archive", + imageRef: "oci-archive:" + ociArchive, + exit: 0, + }, + { + name: "oci", + imageRef: "oci:" + ociLayout, + exit: 0, }, } - for _, tt := range tests { - c.env.RunApptainer( - t, - e2e.AsSubtest(tt.name), - e2e.WithProfile(e2e.OCIRootProfile), - e2e.WithCommand("run"), - // While we don't support args we are entering a /bin/sh interactively, so we need to exit. - e2e.ConsoleRun(e2e.ConsoleSendLine("exit")), - e2e.WithArgs(tt.argv...), - e2e.ExpectExit(tt.exit), - ) + for _, profile := range []e2e.Profile{e2e.OCIRootProfile, e2e.OCIUserProfile} { + t.Run(profile.String(), func(t *testing.T) { + for _, tt := range tests { + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(e2e.OCIRootProfile), + e2e.WithCommand("run"), + // While we don't support args we are entering a /bin/sh interactively. + e2e.ConsoleRun(e2e.ConsoleSendLine("exit")), + e2e.WithArgs(tt.imageRef), + e2e.ExpectExit(tt.exit), + ) + } + }) } } From 7bd59103591dca54cff75cf5da3d41cf4c7d47a8 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Wed, 16 Nov 2022 13:28:20 +0000 Subject: [PATCH 021/114] chore: refactor native bundle to functional options Signed-off-by: Edita Kizinevic --- .../runtime/launcher/oci/launcher_linux.go | 7 +- pkg/ocibundle/native/bundle_linux.go | 173 +++++++++++------- pkg/ocibundle/native/bundle_linux_test.go | 7 +- 3 files changed, 117 insertions(+), 70 deletions(-) diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index 3c982db75e..d41a8bd9a9 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -292,7 +292,12 @@ func (l *Launcher) Exec(ctx context.Context, image string, cmd string, args []st } } - b, err := native.FromImageRef(image, bundleDir, sysCtx, imgCache) + b, err := native.New( + native.OptBundlePath(bundleDir), + native.OptImageRef(image), + native.OptSysCtx(sysCtx), + native.OptImgCache(imgCache), + ) if err != nil { return err } diff --git a/pkg/ocibundle/native/bundle_linux.go b/pkg/ocibundle/native/bundle_linux.go index fc0010826e..2abe9c4b32 100644 --- a/pkg/ocibundle/native/bundle_linux.go +++ b/pkg/ocibundle/native/bundle_linux.go @@ -57,6 +57,112 @@ type Bundle struct { ocibundle.Bundle } +type Option func(b *Bundle) error + +// OptBundlePath sets the path that the bundle will be created at. +func OptBundlePath(bp string) Option { + return func(b *Bundle) error { + var err error + b.bundlePath, err = filepath.Abs(bp) + if err != nil { + return fmt.Errorf("failed to determine bundle path: %s", err) + } + return nil + } +} + +// OptImageRef sets the image source reference, from which the bundle will be created. +func OptImageRef(ref string) Option { + return func(b *Bundle) error { + b.imageRef = ref + return nil + } +} + +// OptSysCtx sets the OCI client SystemContext holding auth information etc. +func OptSysCtx(sc *types.SystemContext) Option { + return func(b *Bundle) error { + b.sysCtx = sc + return nil + } +} + +// OptImgCache sets the Apptainer image cache used to pull through OCI blobs. +func OptImgCache(ic *cache.Handle) Option { + return func(b *Bundle) error { + b.imgCache = ic + return nil + } +} + +// New returns a bundle interface to create/delete an OCI bundle from an OCI image ref. +func New(opts ...Option) (ocibundle.Bundle, error) { + b := Bundle{ + imageRef: "", + sysCtx: &types.SystemContext{}, + imgCache: nil, + } + + for _, opt := range opts { + if err := opt(&b); err != nil { + return nil, fmt.Errorf("while initializing bundle: %w", err) + } + } + + return &b, nil +} + +// Delete erases OCI bundle created an OCI image ref +func (b *Bundle) Delete() error { + return tools.DeleteBundle(b.bundlePath) +} + +// Create will created the on-disk structures for the OCI bundle, so that it is ready for execution. +func (b *Bundle) Create(ctx context.Context, ociConfig *specs.Spec) error { + // generate OCI bundle directory and config + g, err := tools.GenerateBundleConfig(b.bundlePath, ociConfig) + if err != nil { + return fmt.Errorf("failed to generate OCI bundle/config: %s", err) + } + // Due to our caching approach for OCI blobs, we need to pull blobs for the image + // out into a separate oci-layout directory. + tmpDir, err := os.MkdirTemp("", "oci-tmp") + if err != nil { + return err + } + defer os.RemoveAll(tmpDir) + // Fetch into temp oci layout (will pull through cache if enabled) + if err := b.fetchImage(ctx, tmpDir); err != nil { + return err + } + // Extract from temp oci layout into bundle rootfs + if err := b.extractImage(ctx, tmpDir); err != nil { + return err + } + // Remove the temp oci layout. + if err := os.RemoveAll(tmpDir); err != nil { + return err + } + + // Set non-root uid/gid per Apptainer defaults + uid := uint32(os.Getuid()) + if uid != 0 { + gid := uint32(os.Getgid()) + g.Config.Process.User.UID = uid + g.Config.Process.User.GID = gid + } + // Set default ENV from image + g.Config.Process.Env = append(g.Config.Process.Env, b.imageSpec.Config.Env...) + // Set default exec from image CMD & Entrypoint + if b.imageSpec == nil { + return fmt.Errorf("imageSpec cannot be nil") + } + args := append(b.imageSpec.Config.Entrypoint, b.imageSpec.Config.Cmd...) + g.SetProcessArgs(args) + return b.writeConfig(g) +} + +// Path returns the bundle's path on disk. func (b *Bundle) Path() string { return b.bundlePath } @@ -207,70 +313,3 @@ func (b *Bundle) extractImage(ctx context.Context, tmpDir string) error { return nil } - -// Create will created the on-disk structures for the OCI bundle, so that it is ready for execution. -func (b *Bundle) Create(ctx context.Context, ociConfig *specs.Spec) error { - // generate OCI bundle directory and config - g, err := tools.GenerateBundleConfig(b.bundlePath, ociConfig) - if err != nil { - return fmt.Errorf("failed to generate OCI bundle/config: %s", err) - } - // Due to our caching approach for OCI blobs, we need to pull blobs for the image - // out into a separate oci-layout directory. - tmpDir, err := os.MkdirTemp("", "oci-tmp") - if err != nil { - return err - } - defer os.RemoveAll(tmpDir) - // Fetch into temp oci layout (will pull through cache if enabled) - if err := b.fetchImage(ctx, tmpDir); err != nil { - return err - } - // Extract from temp oci layout into bundle rootfs - if err := b.extractImage(ctx, tmpDir); err != nil { - return err - } - // Remove the temp oci layout. - if err := os.RemoveAll(tmpDir); err != nil { - return err - } - - // Set non-root uid/gid per Apptainer defaults - uid := uint32(os.Getuid()) - if uid != 0 { - gid := uint32(os.Getgid()) - g.Config.Process.User.UID = uid - g.Config.Process.User.GID = gid - } - // Set default ENV from image - g.Config.Process.Env = append(g.Config.Process.Env, b.imageSpec.Config.Env...) - // Set default exec from image CMD & Entrypoint - if b.imageSpec == nil { - return fmt.Errorf("imageSpec cannot be nil") - } - args := append(b.imageSpec.Config.Entrypoint, b.imageSpec.Config.Cmd...) - g.SetProcessArgs(args) - return b.writeConfig(g) -} - -// Delete erases OCI bundle created an OCI image ref -func (b *Bundle) Delete() error { - return tools.DeleteBundle(b.bundlePath) -} - -// FromImageRef returns a bundle interface to create/delete an OCI bundle from an OCI image ref. -func FromImageRef(imageRef, bundle string, sysCtx *types.SystemContext, imgCache *cache.Handle) (ocibundle.Bundle, error) { - var err error - - b := &Bundle{ - imageRef: imageRef, - sysCtx: sysCtx, - imgCache: imgCache, - } - b.bundlePath, err = filepath.Abs(bundle) - if err != nil { - return nil, fmt.Errorf("failed to determine bundle path: %s", err) - } - - return b, nil -} diff --git a/pkg/ocibundle/native/bundle_linux_test.go b/pkg/ocibundle/native/bundle_linux_test.go index 6cd0a840d5..91e195d407 100644 --- a/pkg/ocibundle/native/bundle_linux_test.go +++ b/pkg/ocibundle/native/bundle_linux_test.go @@ -19,7 +19,6 @@ import ( "testing" "github.com/apptainer/apptainer/internal/pkg/cache" - "github.com/containers/image/v5/types" "github.com/opencontainers/runtime-tools/validate" ) @@ -125,7 +124,11 @@ func TestFromImageRef(t *testing.T) { t.Skipf("docker not available") } bundleDir := t.TempDir() - b, err := FromImageRef(tt.imageRef, bundleDir, &types.SystemContext{}, setupCache(t)) + b, err := New( + OptBundlePath(bundleDir), + OptImageRef(tt.imageRef), + OptImgCache(setupCache(t)), + ) if err != nil { t.Fatalf("While initializing bundle: %s", err) } From f28acf784a8029bf57830d8297e62657032659d0 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Wed, 16 Nov 2022 15:30:35 +0000 Subject: [PATCH 022/114] feat: exec / run args support for --oci mode When using `run` or `exec` with the `--oci` runtime mode, accept arguments on the command line. For `run`, the arguments override any CMD specified by the image. For `exec`, the arguments replace ENTRYPOINT/CMD entirely, bypassing the process configuration in the image config. This mirrors the behavior of Singularity images today, via the exec and run runscripts - but is implemented in the OCI bundle config, rather than a script in the container. Closes sylabs/singularity#1024 Closes sylabs/singularity#1092 Signed-off-by: Edita Kizinevic --- internal/pkg/runtime/launcher/launcher.go | 10 +- .../runtime/launcher/native/launcher_linux.go | 4 +- .../runtime/launcher/oci/launcher_linux.go | 11 +- pkg/ocibundle/native/bundle_linux.go | 60 +++++++-- pkg/ocibundle/native/bundle_linux_test.go | 127 ++++++++++++++++++ 5 files changed, 185 insertions(+), 27 deletions(-) diff --git a/internal/pkg/runtime/launcher/launcher.go b/internal/pkg/runtime/launcher/launcher.go index 177dc373b2..e3c316bf04 100644 --- a/internal/pkg/runtime/launcher/launcher.go +++ b/internal/pkg/runtime/launcher/launcher.go @@ -25,9 +25,9 @@ import "context" // It will execute a runtime, such as Apptainer's native runtime (via the starter // binary), or an external OCI runtime (e.g. runc). type Launcher interface { - // Exec will execute the container image 'image', passing arguments 'args' - // the container#s initial process. If instanceName is specified, the - // container must be launched as a background instance, otherwist it must - // run interactively, attached to the console. - Exec(ctx context.Context, image string, cmd string, args []string, instanceName string) error + // Exec will execute the container image 'image', starting 'process', and + // passing arguments 'args'. If instanceName is specified, the container + // must be launched as a background instance, otherwist it must run + // interactively, attached to the console. + Exec(ctx context.Context, image string, process string, args []string, instanceName string) error } diff --git a/internal/pkg/runtime/launcher/native/launcher_linux.go b/internal/pkg/runtime/launcher/native/launcher_linux.go index 255893f044..468ba356ba 100644 --- a/internal/pkg/runtime/launcher/native/launcher_linux.go +++ b/internal/pkg/runtime/launcher/native/launcher_linux.go @@ -107,7 +107,7 @@ func NewLauncher(opts ...launcher.Option) (*Launcher, error) { // This includes interactive containers, instances, and joining an existing instance. // //nolint:maintidx -func (l *Launcher) Exec(ctx context.Context, image string, cmd string, args []string, instanceName string) error { +func (l *Launcher) Exec(ctx context.Context, image string, process string, args []string, instanceName string) error { var err error var fakerootPath string @@ -183,7 +183,7 @@ func (l *Launcher) Exec(ctx context.Context, image string, cmd string, args []st } // Native runtime expects command to execute as arg[0] - args = append([]string{cmd}, args...) + args = append([]string{process}, args...) // Set arguments to pass to contained process. l.generator.SetProcessArgs(args) diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index d41a8bd9a9..567ee61a36 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -243,19 +243,11 @@ func checkOpts(lo launcher.Options) error { // Exec will interactively execute a container via the runc low-level runtime. // image is a reference to an OCI image, e.g. docker://ubuntu or oci:/tmp/mycontainer -func (l *Launcher) Exec(ctx context.Context, image string, cmd string, args []string, instanceName string) error { +func (l *Launcher) Exec(ctx context.Context, image string, process string, args []string, instanceName string) error { if instanceName != "" { return fmt.Errorf("%w: instanceName", ErrNotImplemented) } - if cmd != "" { - return fmt.Errorf("%w: cmd %v", ErrNotImplemented, cmd) - } - - if len(args) > 0 { - return fmt.Errorf("%w: args %v", ErrNotImplemented, args) - } - bundleDir, err := os.MkdirTemp("", "oci-bundle") if err != nil { return nil @@ -297,6 +289,7 @@ func (l *Launcher) Exec(ctx context.Context, image string, cmd string, args []st native.OptImageRef(image), native.OptSysCtx(sysCtx), native.OptImgCache(imgCache), + native.OptProcessArgs(process, args), ) if err != nil { return err diff --git a/pkg/ocibundle/native/bundle_linux.go b/pkg/ocibundle/native/bundle_linux.go index 2abe9c4b32..72d3a5bddc 100644 --- a/pkg/ocibundle/native/bundle_linux.go +++ b/pkg/ocibundle/native/bundle_linux.go @@ -53,6 +53,10 @@ type Bundle struct { // Note that we only use the 'blob' cache section. The 'oci-tmp' cache section holds // OCI->SIF conversions, which are not used here. imgCache *cache.Handle + // process is the command to execute, which may override the image's ENTRYPOINT / CMD. + process string + // args are the command arguments, which may override the image's CMD. + args []string // Generic bundle properties ocibundle.Bundle } @@ -95,6 +99,15 @@ func OptImgCache(ic *cache.Handle) Option { } } +// OptProcessArgs sets the command and arguments to run in the container. +func OptProcessArgs(process string, args []string) Option { + return func(b *Bundle) error { + b.process = process + b.args = args + return nil + } +} + // New returns a bundle interface to create/delete an OCI bundle from an OCI image ref. func New(opts ...Option) (ocibundle.Bundle, error) { b := Bundle{ @@ -144,6 +157,20 @@ func (b *Bundle) Create(ctx context.Context, ociConfig *specs.Spec) error { return err } + b.setProcessArgs(g) + // TODO - Handle custom env and user + b.setProcessEnv(g) + b.setProcessUser(g) + + return b.writeConfig(g) +} + +// Path returns the bundle's path on disk. +func (b *Bundle) Path() string { + return b.bundlePath +} + +func (b *Bundle) setProcessUser(g *generate.Generator) { // Set non-root uid/gid per Apptainer defaults uid := uint32(os.Getuid()) if uid != 0 { @@ -151,20 +178,31 @@ func (b *Bundle) Create(ctx context.Context, ociConfig *specs.Spec) error { g.Config.Process.User.UID = uid g.Config.Process.User.GID = gid } - // Set default ENV from image +} + +func (b *Bundle) setProcessEnv(g *generate.Generator) { + // Set default ENV values from image g.Config.Process.Env = append(g.Config.Process.Env, b.imageSpec.Config.Env...) - // Set default exec from image CMD & Entrypoint - if b.imageSpec == nil { - return fmt.Errorf("imageSpec cannot be nil") - } - args := append(b.imageSpec.Config.Entrypoint, b.imageSpec.Config.Cmd...) - g.SetProcessArgs(args) - return b.writeConfig(g) } -// Path returns the bundle's path on disk. -func (b *Bundle) Path() string { - return b.bundlePath +func (b *Bundle) setProcessArgs(g *generate.Generator) { + var processArgs []string + + if b.process != "" { + processArgs = []string{b.process} + } else { + processArgs = b.imageSpec.Config.Entrypoint + } + + if len(b.args) > 0 { + processArgs = append(processArgs, b.args...) + } else { + if b.process == "" { + processArgs = append(processArgs, b.imageSpec.Config.Cmd...) + } + } + + g.SetProcessArgs(processArgs) } func (b *Bundle) writeConfig(g *generate.Generator) error { diff --git a/pkg/ocibundle/native/bundle_linux_test.go b/pkg/ocibundle/native/bundle_linux_test.go index 91e195d407..77e6b03a2f 100644 --- a/pkg/ocibundle/native/bundle_linux_test.go +++ b/pkg/ocibundle/native/bundle_linux_test.go @@ -16,9 +16,12 @@ import ( "net/http" "os" "os/exec" + "reflect" "testing" "github.com/apptainer/apptainer/internal/pkg/cache" + "github.com/apptainer/apptainer/internal/pkg/runtime/engine/config/oci" + v1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/opencontainers/runtime-tools/validate" ) @@ -141,3 +144,127 @@ func TestFromImageRef(t *testing.T) { }) } } + +func TestSetProcessArgs(t *testing.T) { + tests := []struct { + name string + imgEntrypoint []string + imgCmd []string + bundleProcess string + bundleArgs []string + expectProcessArgs []string + }{ + { + name: "imageEntrypointOnly", + imgEntrypoint: []string{"ENTRYPOINT"}, + imgCmd: []string{}, + bundleProcess: "", + bundleArgs: []string{}, + expectProcessArgs: []string{"ENTRYPOINT"}, + }, + { + name: "imageCmdOnly", + imgEntrypoint: []string{}, + imgCmd: []string{"CMD"}, + bundleProcess: "", + bundleArgs: []string{}, + expectProcessArgs: []string{"CMD"}, + }, + { + name: "imageEntrypointCMD", + imgEntrypoint: []string{"ENTRYPOINT"}, + imgCmd: []string{"CMD"}, + bundleProcess: "", + bundleArgs: []string{}, + expectProcessArgs: []string{"ENTRYPOINT", "CMD"}, + }, + { + name: "ProcessOnly", + imgEntrypoint: []string{}, + imgCmd: []string{}, + bundleProcess: "PROCESS", + bundleArgs: []string{}, + expectProcessArgs: []string{"PROCESS"}, + }, + { + name: "ArgsOnly", + imgEntrypoint: []string{}, + imgCmd: []string{}, + bundleProcess: "", + bundleArgs: []string{"ARGS"}, + expectProcessArgs: []string{"ARGS"}, + }, + { + name: "ProcessArgs", + imgEntrypoint: []string{}, + imgCmd: []string{}, + bundleProcess: "PROCESS", + bundleArgs: []string{"ARGS"}, + expectProcessArgs: []string{"PROCESS", "ARGS"}, + }, + { + name: "overrideEntrypointOnlyProcess", + imgEntrypoint: []string{"ENTRYPOINT"}, + imgCmd: []string{}, + bundleProcess: "PROCESS", + bundleArgs: []string{}, + expectProcessArgs: []string{"PROCESS"}, + }, + { + name: "overrideCmdOnlyArgs", + imgEntrypoint: []string{}, + imgCmd: []string{"CMD"}, + bundleProcess: "", + bundleArgs: []string{"ARGS"}, + expectProcessArgs: []string{"ARGS"}, + }, + { + name: "overrideBothProcess", + imgEntrypoint: []string{"ENTRYPOINT"}, + imgCmd: []string{"CMD"}, + bundleProcess: "PROCESS", + bundleArgs: []string{}, + expectProcessArgs: []string{"PROCESS"}, + }, + { + name: "overrideBothArgs", + imgEntrypoint: []string{"ENTRYPOINT"}, + imgCmd: []string{"CMD"}, + bundleProcess: "", + bundleArgs: []string{"ARGS"}, + expectProcessArgs: []string{"ENTRYPOINT", "ARGS"}, + }, + { + name: "overrideBothProcessArgs", + imgEntrypoint: []string{"ENTRYPOINT"}, + imgCmd: []string{"CMD"}, + bundleProcess: "PROCESS", + bundleArgs: []string{"ARGS"}, + expectProcessArgs: []string{"PROCESS", "ARGS"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + b := Bundle{ + imageSpec: &v1.Image{ + Config: v1.ImageConfig{ + Entrypoint: tt.imgEntrypoint, + Cmd: tt.imgCmd, + }, + }, + process: tt.bundleProcess, + args: tt.bundleArgs, + } + + g, err := oci.DefaultConfig() + if err != nil { + t.Fatal(err) + } + b.setProcessArgs(g) + if !reflect.DeepEqual(g.Config.Process.Args, tt.expectProcessArgs) { + t.Errorf("Expected: %v, Got: %v", tt.expectProcessArgs, g.Config.Process.Args) + } + }) + } +} From e14351951769a35d97c003e9f92b9505bd47faf8 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Wed, 16 Nov 2022 15:50:44 +0000 Subject: [PATCH 023/114] fix: return exit code from OCI launcher Signed-off-by: Edita Kizinevic --- internal/pkg/runtime/launcher/oci/launcher_linux.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index 567ee61a36..045a010061 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -17,6 +17,7 @@ import ( "errors" "fmt" "os" + "os/exec" "strings" "github.com/apptainer/apptainer/internal/pkg/buildcfg" @@ -303,5 +304,10 @@ func (l *Launcher) Exec(ctx context.Context, image string, process string, args if err != nil { return fmt.Errorf("while generating container id: %w", err) } - return Run(ctx, id.String(), b.Path(), "") + + err = Run(ctx, id.String(), b.Path(), "") + if exiterr, ok := err.(*exec.ExitError); ok { + os.Exit(exiterr.ExitCode()) + } + return err } From fcacb62ebb5ccb5b6ea51ebd1afeedc122e3d56e Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Wed, 16 Nov 2022 16:13:50 +0000 Subject: [PATCH 024/114] e2e: test --oci run/exec with args Signed-off-by: Edita Kizinevic --- e2e/actions/actions.go | 3 +- e2e/actions/oci.go | 110 ++++++++++++++++++++++++++------------ e2e/internal/e2e/env.go | 1 + e2e/internal/e2e/image.go | 51 ++++++++++++++++++ e2e/suite.go | 6 +++ 5 files changed, 135 insertions(+), 36 deletions(-) diff --git a/e2e/actions/actions.go b/e2e/actions/actions.go index 8a7a508836..3b9323714e 100644 --- a/e2e/actions/actions.go +++ b/e2e/actions/actions.go @@ -2912,6 +2912,7 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { // // OCI Runtime Mode // - "ociRun": c.actionOciRun, // apptainer run --oci + "ociRun": c.actionOciRun, // apptainer run --oci + "ociExec": c.actionOciExec, // apptainer exec --oci } } diff --git a/e2e/actions/oci.go b/e2e/actions/oci.go index afc1e2179f..2798a24ec1 100644 --- a/e2e/actions/oci.go +++ b/e2e/actions/oci.go @@ -10,11 +10,9 @@ package actions import ( - "io" - "log" - "net/http" "os" "os/exec" + "path/filepath" "testing" "github.com/apptainer/apptainer/e2e/internal/e2e" @@ -22,46 +20,22 @@ import ( const ( dockerArchiveURI = "https://github.com/apptainer/apptainer/releases/download/v0.1.0/alpine-docker-save.tar" - ociArchiveURI = "https://github.com/apptainer/apptainer/releases/download/v0.1.0/alpine-oci-archive.tar" ) -func getTestTar(url string) (path string, err error) { - dl, err := os.CreateTemp("", "oci-test") - if err != nil { - log.Fatal(err) - } - defer dl.Close() - - r, err := http.Get(url) - if err != nil { - return "", err - } - defer r.Body.Close() - - _, err = io.Copy(dl, r.Body) - if err != nil { - return "", err - } - - return dl.Name(), nil -} - func (c actionTests) actionOciRun(t *testing.T) { + e2e.EnsureOCIImage(t, c.env) + // Prepare docker-archive source - dockerArchive, err := getTestTar(dockerArchiveURI) + tmpDir := t.TempDir() + dockerArchive := filepath.Join(tmpDir, "docker-archive.tar") + err := e2e.DownloadFile(dockerArchiveURI, dockerArchive) if err != nil { t.Fatalf("Could not download docker archive test file: %v", err) } defer os.Remove(dockerArchive) - // Prepare oci-archive source - ociArchive, err := getTestTar(ociArchiveURI) - if err != nil { - t.Fatalf("Could not download oci archive test file: %v", err) - } - defer os.Remove(ociArchive) // Prepare oci source (oci directory layout) ociLayout := t.TempDir() - cmd := exec.Command("tar", "-C", ociLayout, "-xf", ociArchive) + cmd := exec.Command("tar", "-C", ociLayout, "-xf", c.env.OCIImagePath) err = cmd.Run() if err != nil { t.Fatalf("Error extracting oci archive to layout: %v", err) @@ -70,6 +44,7 @@ func (c actionTests) actionOciRun(t *testing.T) { tests := []struct { name string imageRef string + argv []string exit int }{ { @@ -79,7 +54,7 @@ func (c actionTests) actionOciRun(t *testing.T) { }, { name: "oci-archive", - imageRef: "oci-archive:" + ociArchive, + imageRef: "oci-archive:" + c.env.OCIImagePath, exit: 0, }, { @@ -87,11 +62,25 @@ func (c actionTests) actionOciRun(t *testing.T) { imageRef: "oci:" + ociLayout, exit: 0, }, + { + name: "true", + imageRef: "oci:" + ociLayout, + argv: []string{"true"}, + exit: 0, + }, + { + name: "false", + imageRef: "oci:" + ociLayout, + argv: []string{"false"}, + exit: 1, + }, } for _, profile := range []e2e.Profile{e2e.OCIRootProfile, e2e.OCIUserProfile} { t.Run(profile.String(), func(t *testing.T) { for _, tt := range tests { + cmdArgs := []string{tt.imageRef} + cmdArgs = append(cmdArgs, tt.argv...) c.env.RunApptainer( t, e2e.AsSubtest(tt.name), @@ -99,10 +88,61 @@ func (c actionTests) actionOciRun(t *testing.T) { e2e.WithCommand("run"), // While we don't support args we are entering a /bin/sh interactively. e2e.ConsoleRun(e2e.ConsoleSendLine("exit")), - e2e.WithArgs(tt.imageRef), + e2e.WithArgs(cmdArgs...), e2e.ExpectExit(tt.exit), ) } }) } } + +// exec tests min fuctionality for apptainer exec +func (c actionTests) actionOciExec(t *testing.T) { + e2e.EnsureOCIImage(t, c.env) + + imageRef := "oci-archive:" + c.env.OCIImagePath + + tests := []struct { + name string + argv []string + exit int + }{ + { + name: "NoCommand", + argv: []string{imageRef}, + exit: 1, + }, + { + name: "True", + argv: []string{imageRef, "true"}, + exit: 0, + }, + { + name: "TrueAbsPAth", + argv: []string{imageRef, "/bin/true"}, + exit: 0, + }, + { + name: "False", + argv: []string{imageRef, "false"}, + exit: 1, + }, + { + name: "FalseAbsPath", + argv: []string{imageRef, "/bin/false"}, + exit: 1, + }, + } + + for _, tt := range tests { + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(e2e.UserProfile), + e2e.WithCommand("exec"), + e2e.WithDir("/tmp"), + e2e.WithArgs(tt.argv...), + e2e.ExpectExit(tt.exit), + ) + } +} diff --git a/e2e/internal/e2e/env.go b/e2e/internal/e2e/env.go index da4fec7d6f..f67f826e79 100644 --- a/e2e/internal/e2e/env.go +++ b/e2e/internal/e2e/env.go @@ -18,6 +18,7 @@ type TestEnv struct { SingularityImagePath string // Path to a Singularity image for legacy tests DebianImagePath string // Path to an image containing a Debian distribution with libc compatible to the host libc OrasTestImage string // URI to SIF image pushed into local registry with ORAS + OCIImagePath string TestDir string // Path to the directory from which an Apptainer command needs to be executed TestRegistry string // Host:Port of local registry TestRegistryImage string // URI to OCI image pushed into local registry diff --git a/e2e/internal/e2e/image.go b/e2e/internal/e2e/image.go index eb171c55af..367de54140 100644 --- a/e2e/internal/e2e/image.go +++ b/e2e/internal/e2e/image.go @@ -13,6 +13,7 @@ import ( "context" "fmt" "io" + "net/http" "os" "os/exec" "path/filepath" @@ -33,6 +34,8 @@ import ( "github.com/containers/image/v5/types" ) +const ociArchiveURI = "https://github.com/apptainer/apptainer/releases/download/v0.1.0/alpine-oci-archive.tar" + var ( ensureMutex sync.Mutex pullMutex sync.Mutex @@ -306,3 +309,51 @@ func parseRef(refString string) (ref types.ImageReference, err error) { return ref, err } + +func DownloadFile(url string, path string) error { + dl, err := os.Create(path) + if err != nil { + return err + } + defer dl.Close() + + r, err := http.Get(url) + if err != nil { + return err + } + defer r.Body.Close() + + _, err = io.Copy(dl, r.Body) + if err != nil { + return err + } + return nil +} + +// EnsureImage checks if e2e OCI test image is available, and fetches +// it otherwise. +func EnsureOCIImage(t *testing.T, env TestEnv) { + ensureMutex.Lock() + defer ensureMutex.Unlock() + + switch _, err := os.Stat(env.OCIImagePath); { + case err == nil: + // OK: file exists, return + return + + case os.IsNotExist(err): + // OK: file does not exist, continue + + default: + // FATAL: something else is wrong + t.Fatalf("Failed when checking image %q: %+v\n", + env.OCIImagePath, + err) + } + + // Prepare oci-archive source + err := DownloadFile(ociArchiveURI, env.OCIImagePath) + if err != nil { + t.Fatalf("Could not download oci archive test file: %v", err) + } +} diff --git a/e2e/suite.go b/e2e/suite.go index a4b657c07c..4c30147665 100644 --- a/e2e/suite.go +++ b/e2e/suite.go @@ -164,6 +164,12 @@ func Run(t *testing.T) { testenv.SingularityImagePath = path.Join(name, "test-singularity.sif") defer os.Remove(testenv.SingularityImagePath) + // OCI Test image + ociImagePath := path.Join(name, "oci.tar") + t.Log("Path to test OCI image:", ociImagePath) + testenv.OCIImagePath = ociImagePath + defer os.Remove(ociImagePath) + testenv.DebianImagePath = path.Join(name, "test-debian.sif") defer os.Remove(testenv.DebianImagePath) From 24c9783830fff1215215ef88cae432e243339594 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Fri, 18 Nov 2022 15:05:03 +0000 Subject: [PATCH 025/114] feat: enable shell for --oci mode Enable `singularity shell --oci ...` with behavior matching native runtime, i.e. * Run shell set with SINGULARITY_SHELL or --shell * If not set, try /bin/bash --norc * If not available, use /bin/sh Closes sylabs/singularity#1025 Signed-off-by: Edita Kizinevic --- cmd/internal/cli/actions.go | 15 +++++-- e2e/actions/actions.go | 5 ++- e2e/actions/oci.go | 50 +++++++++++++++++++++++ internal/pkg/runtime/launcher/launcher.go | 2 +- 4 files changed, 66 insertions(+), 6 deletions(-) diff --git a/cmd/internal/cli/actions.go b/cmd/internal/cli/actions.go index fc9c470db8..5fd30b3dc5 100644 --- a/cmd/internal/cli/actions.go +++ b/cmd/internal/cli/actions.go @@ -243,10 +243,19 @@ var ShellCmd = &cobra.Command{ image := args[0] containerCmd := "/.singularity.d/actions/shell" containerArgs := []string{} - // OCI runtime does not use an action script + // OCI runtime does not use an action script, but must match behavior. + // See - internal/pkg/util/fs/files/action_scripts.go (case shell). if ociRuntime { - // TODO - needs to have bash -> sh fallback logic implemented somewhere. - containerCmd = "/bin/sh" + // APPTAINER_SHELL or --shell has priority + if shellPath != "" { + containerCmd = shellPath + // Clear the shellPath - not handled internally by the OCI runtime, as we exec directly without an action script. + shellPath = "" + } else { + // Otherwise try to exec /bin/bash --norc, falling back to /bin/sh + containerCmd = "/bin/sh" + containerArgs = []string{"-c", "test -x /bin/bash && PS1='Apptainer> ' exec /bin/bash --norc || PS1='Apptainer> ' exec /bin/sh"} + } } setVM(cmd) if vm { diff --git a/e2e/actions/actions.go b/e2e/actions/actions.go index 3b9323714e..165469e7bc 100644 --- a/e2e/actions/actions.go +++ b/e2e/actions/actions.go @@ -2912,7 +2912,8 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { // // OCI Runtime Mode // - "ociRun": c.actionOciRun, // apptainer run --oci - "ociExec": c.actionOciExec, // apptainer exec --oci + "ociRun": c.actionOciRun, // apptainer run --oci + "ociExec": c.actionOciExec, // apptainer exec --oci + "ociShell": c.actionOciShell, // apptainer shell --oci } } diff --git a/e2e/actions/oci.go b/e2e/actions/oci.go index 2798a24ec1..67d51e38f3 100644 --- a/e2e/actions/oci.go +++ b/e2e/actions/oci.go @@ -146,3 +146,53 @@ func (c actionTests) actionOciExec(t *testing.T) { ) } } + +// Shell interaction tests +func (c actionTests) actionOciShell(t *testing.T) { + e2e.EnsureOCIImage(t, c.env) + + tests := []struct { + name string + argv []string + consoleOps []e2e.ApptainerConsoleOp + exit int + }{ + { + name: "ShellExit", + argv: []string{"oci-archive:" + c.env.OCIImagePath}, + consoleOps: []e2e.ApptainerConsoleOp{ + // "cd /" to work around issue where a long + // working directory name causes the test + // to fail because the "Apptainer" that + // we are looking for is chopped from the + // front. + // TODO(mem): This test was added back in 491a71716013654acb2276e4b37c2e015d2dfe09 + e2e.ConsoleSendLine("cd /"), + e2e.ConsoleExpect("Apptainer"), + e2e.ConsoleSendLine("exit"), + }, + exit: 0, + }, + { + name: "ShellBadCommand", + argv: []string{"oci-archive:" + c.env.OCIImagePath}, + consoleOps: []e2e.ApptainerConsoleOp{ + e2e.ConsoleSendLine("_a_fake_command"), + e2e.ConsoleSendLine("exit"), + }, + exit: 127, + }, + } + + for _, tt := range tests { + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(e2e.OCIUserProfile), + e2e.WithCommand("shell"), + e2e.WithArgs(tt.argv...), + e2e.ConsoleRun(tt.consoleOps...), + e2e.ExpectExit(tt.exit), + ) + } +} diff --git a/internal/pkg/runtime/launcher/launcher.go b/internal/pkg/runtime/launcher/launcher.go index e3c316bf04..812349835d 100644 --- a/internal/pkg/runtime/launcher/launcher.go +++ b/internal/pkg/runtime/launcher/launcher.go @@ -27,7 +27,7 @@ import "context" type Launcher interface { // Exec will execute the container image 'image', starting 'process', and // passing arguments 'args'. If instanceName is specified, the container - // must be launched as a background instance, otherwist it must run + // must be launched as a background instance, otherwise it must run // interactively, attached to the console. Exec(ctx context.Context, image string, process string, args []string, instanceName string) error } From 92f2ddbf631646d7842c35d387d294d49b1c933c Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Mon, 21 Nov 2022 09:46:10 +0000 Subject: [PATCH 026/114] fix: correct uid/gid non-root mapping Ensure e2e tests for oci actions use user profile. Set uid/gid mappings explicitly. We need to do this anyone, going forward, but here it works around: https://github.com/containers/crun/issues/1072 Signed-off-by: Edita Kizinevic --- cmd/internal/cli/build_linux.go | 5 +- e2e/actions/oci.go | 2 +- internal/app/apptainer/overlay_create.go | 5 +- internal/pkg/build/stage.go | 4 +- .../pkg/{fakeroot => fakefake}/fakefake.go | 2 +- .../runtime/engine/apptainer/prepare_linux.go | 3 +- .../runtime/engine/apptainer/process_linux.go | 6 +- .../runtime/launcher/native/launcher_linux.go | 9 +- pkg/ocibundle/native/bundle_linux.go | 93 ++++++++++++++++++- 9 files changed, 111 insertions(+), 18 deletions(-) rename internal/pkg/{fakeroot => fakefake}/fakefake.go (99%) diff --git a/cmd/internal/cli/build_linux.go b/cmd/internal/cli/build_linux.go index 099164213d..de6c8ac9fa 100644 --- a/cmd/internal/cli/build_linux.go +++ b/cmd/internal/cli/build_linux.go @@ -27,6 +27,7 @@ import ( "github.com/apptainer/apptainer/internal/pkg/build" "github.com/apptainer/apptainer/internal/pkg/buildcfg" "github.com/apptainer/apptainer/internal/pkg/cache" + "github.com/apptainer/apptainer/internal/pkg/fakefake" "github.com/apptainer/apptainer/internal/pkg/fakeroot" "github.com/apptainer/apptainer/internal/pkg/remote/endpoint" fakerootConfig "github.com/apptainer/apptainer/internal/pkg/runtime/engine/fakeroot/config" @@ -95,7 +96,7 @@ func fakerootExec(isDeffile, unprivEncrypt bool) { if buildArgs.ignoreUserns { err = errors.New("could not start root-mapped namespace because --ignore-userns is set") } else { - err = fakeroot.UnshareRootMapped(args) + err = fakefake.UnshareRootMapped(args) } if err == nil { // All the work has been done by the child process @@ -173,7 +174,7 @@ func runBuild(cmd *cobra.Command, args []string) { if buildArgs.ignoreFakerootCmd { err = errors.New("fakeroot command is ignored because of --ignore-fakeroot-command") } else { - fakerootPath, err = fakeroot.FindFake() + fakerootPath, err = fakefake.FindFake() } if err != nil { sylog.Infof("fakeroot command not found") diff --git a/e2e/actions/oci.go b/e2e/actions/oci.go index 67d51e38f3..bd811021b4 100644 --- a/e2e/actions/oci.go +++ b/e2e/actions/oci.go @@ -84,7 +84,7 @@ func (c actionTests) actionOciRun(t *testing.T) { c.env.RunApptainer( t, e2e.AsSubtest(tt.name), - e2e.WithProfile(e2e.OCIRootProfile), + e2e.WithProfile(e2e.OCIUserProfile), e2e.WithCommand("run"), // While we don't support args we are entering a /bin/sh interactively. e2e.ConsoleRun(e2e.ConsoleSendLine("exit")), diff --git a/internal/app/apptainer/overlay_create.go b/internal/app/apptainer/overlay_create.go index 23a03f0dde..3e9dd1789c 100644 --- a/internal/app/apptainer/overlay_create.go +++ b/internal/app/apptainer/overlay_create.go @@ -18,6 +18,7 @@ import ( "runtime" "strings" + "github.com/apptainer/apptainer/internal/pkg/fakefake" "github.com/apptainer/apptainer/internal/pkg/fakeroot" "github.com/apptainer/apptainer/internal/pkg/util/bin" "github.com/apptainer/apptainer/pkg/image" @@ -232,7 +233,7 @@ func OverlayCreate(size int, imgPath string, overlaySparse bool, isFakeroot bool // the fakeroot command (in suid flow with no user // namespaces), using the --fakeroot option here // prevents overlay from working, most unfortunately. - err = fakeroot.UnshareRootMapped([]string{"/bin/true"}) + err = fakefake.UnshareRootMapped([]string{"/bin/true"}) if err != nil { sylog.Debugf("UnshareRootMapped failed: %v", err) if isFakeroot { @@ -248,7 +249,7 @@ func OverlayCreate(size int, imgPath string, overlaySparse bool, isFakeroot bool if isFakeroot { sylog.Debugf("Trying root-mapped namespace") - err = fakeroot.UnshareRootMapped(os.Args) + err = fakefake.UnshareRootMapped(os.Args) if err == nil { // everything was done by the child os.Exit(0) diff --git a/internal/pkg/build/stage.go b/internal/pkg/build/stage.go index 18cad349a3..2bc8c74122 100644 --- a/internal/pkg/build/stage.go +++ b/internal/pkg/build/stage.go @@ -19,7 +19,7 @@ import ( "github.com/apptainer/apptainer/internal/pkg/build/files" "github.com/apptainer/apptainer/internal/pkg/buildcfg" - "github.com/apptainer/apptainer/internal/pkg/fakeroot" + "github.com/apptainer/apptainer/internal/pkg/fakefake" "github.com/apptainer/apptainer/pkg/build/types" "github.com/apptainer/apptainer/pkg/sylog" ) @@ -99,7 +99,7 @@ func (s *stage) runPostScript(sessionResolv, sessionHosts string) error { // the nested apptainer will run fakeroot if it isn't // started and pass down the components and environment // to nested apptainers. - fakerootBinds, err = fakeroot.GetFakeBinds(s.b.Opts.FakerootPath) + fakerootBinds, err = fakefake.GetFakeBinds(s.b.Opts.FakerootPath) if err != nil { return fmt.Errorf("while getting fakeroot bindpoints: %v", err) } diff --git a/internal/pkg/fakeroot/fakefake.go b/internal/pkg/fakefake/fakefake.go similarity index 99% rename from internal/pkg/fakeroot/fakefake.go rename to internal/pkg/fakefake/fakefake.go index 10816aff23..2a9e642b6e 100644 --- a/internal/pkg/fakeroot/fakefake.go +++ b/internal/pkg/fakefake/fakefake.go @@ -9,7 +9,7 @@ // This file is for "fake fakeroot", that is, root-mapped unprivileged // user namespaces (unshare -r) and the fakeroot command -package fakeroot +package fakefake import ( "bufio" diff --git a/internal/pkg/runtime/engine/apptainer/prepare_linux.go b/internal/pkg/runtime/engine/apptainer/prepare_linux.go index 33ba28a29e..95868811b5 100644 --- a/internal/pkg/runtime/engine/apptainer/prepare_linux.go +++ b/internal/pkg/runtime/engine/apptainer/prepare_linux.go @@ -25,6 +25,7 @@ import ( "github.com/ProtonMail/go-crypto/openpgp" "github.com/apptainer/apptainer/internal/pkg/buildcfg" "github.com/apptainer/apptainer/internal/pkg/cgroups" + "github.com/apptainer/apptainer/internal/pkg/fakefake" fakerootutil "github.com/apptainer/apptainer/internal/pkg/fakeroot" "github.com/apptainer/apptainer/internal/pkg/image/driver" "github.com/apptainer/apptainer/internal/pkg/instance" @@ -98,7 +99,7 @@ func (e *EngineOperations) PrepareConfig(starterConfig *starter.Config) error { if fakerootPath := e.EngineConfig.GetFakerootPath(); fakerootPath != "" { // look for fakeroot again because the PATH used is // more restricted at this point than it was earlier - newPath, err := fakerootutil.FindFake() + newPath, err := fakefake.FindFake() if err != nil { return fmt.Errorf("error finding fakeroot in privileged PATH: %v", err) } diff --git a/internal/pkg/runtime/engine/apptainer/process_linux.go b/internal/pkg/runtime/engine/apptainer/process_linux.go index f0dce42fa9..26ccf07a49 100644 --- a/internal/pkg/runtime/engine/apptainer/process_linux.go +++ b/internal/pkg/runtime/engine/apptainer/process_linux.go @@ -32,7 +32,7 @@ import ( "unsafe" "github.com/apptainer/apptainer/internal/pkg/checkpoint/dmtcp" - "github.com/apptainer/apptainer/internal/pkg/fakeroot" + "github.com/apptainer/apptainer/internal/pkg/fakefake" "github.com/apptainer/apptainer/internal/pkg/instance" "github.com/apptainer/apptainer/internal/pkg/plugin" "github.com/apptainer/apptainer/internal/pkg/security" @@ -889,7 +889,7 @@ func runActionScript(engineConfig *apptainerConfig.EngineConfig) ([]string, []st } } - fakeargs := fakeroot.GetFakeArgs() + fakeargs := fakefake.GetFakeArgs() fakerootPath := fakeargs[0] _, err = os.Stat(fakerootPath) if err == nil && getEnvVal(penv, "FAKEROOTKEY") == "" { @@ -906,7 +906,7 @@ func runActionScript(engineConfig *apptainerConfig.EngineConfig) ([]string, []st if engineConfig.GetFakerootPath() == "" { // Must be joining an instance, so also set BIND // variables for nesting - fakebinds, _ := fakeroot.GetFakeBinds(fakerootPath) + fakebinds, _ := fakefake.GetFakeBinds(fakerootPath) bindval := strings.Join(fakebinds, ",") for _, pfx := range env.ApptainerPrefixes { bindvar := pfx + "BIND=" diff --git a/internal/pkg/runtime/launcher/native/launcher_linux.go b/internal/pkg/runtime/launcher/native/launcher_linux.go index 468ba356ba..4a3ef4541b 100644 --- a/internal/pkg/runtime/launcher/native/launcher_linux.go +++ b/internal/pkg/runtime/launcher/native/launcher_linux.go @@ -26,6 +26,7 @@ import ( "github.com/apptainer/apptainer/internal/pkg/buildcfg" "github.com/apptainer/apptainer/internal/pkg/cgroups" "github.com/apptainer/apptainer/internal/pkg/checkpoint/dmtcp" + "github.com/apptainer/apptainer/internal/pkg/fakefake" "github.com/apptainer/apptainer/internal/pkg/fakeroot" "github.com/apptainer/apptainer/internal/pkg/image/driver" "github.com/apptainer/apptainer/internal/pkg/image/unpacker" @@ -134,7 +135,7 @@ func (l *Launcher) Exec(ctx context.Context, image string, process string, args if l.cfg.IgnoreFakerootCmd { err = errors.New("fakeroot command is ignored because of --ignore-fakeroot-command") } else { - fakerootPath, err = fakeroot.FindFake() + fakerootPath, err = fakefake.FindFake() } if err != nil { sylog.Infof("fakeroot command not found, using only root-mapped namespace") @@ -148,7 +149,7 @@ func (l *Launcher) Exec(ctx context.Context, image string, process string, args if l.cfg.IgnoreUserns { err = errors.New("could not start root-mapped namespace because --ignore-userns is set") } else { - err = fakeroot.UnshareRootMapped(os.Args) + err = fakefake.UnshareRootMapped(os.Args) } if err == nil { // All good @@ -158,7 +159,7 @@ func (l *Launcher) Exec(ctx context.Context, image string, process string, args if l.cfg.IgnoreFakerootCmd { err = errors.New("fakeroot command is ignored because of --ignore-fakeroot-command") } else { - fakerootPath, err = fakeroot.FindFake() + fakerootPath, err = fakefake.FindFake() } if err != nil { sylog.Fatalf("--fakeroot requires either being in %v, unprivileged user namespaces, or the fakeroot command", fakeroot.SubUIDFile) @@ -670,7 +671,7 @@ func (l *Launcher) setBinds(fakerootPath string) error { if fakerootPath != "" { l.engineConfig.SetFakerootPath(fakerootPath) // Add binds for fakeroot command - fakebindPaths, err := fakeroot.GetFakeBinds(fakerootPath) + fakebindPaths, err := fakefake.GetFakeBinds(fakerootPath) if err != nil { return fmt.Errorf("while getting fakeroot bindpoints: %w", err) } diff --git a/pkg/ocibundle/native/bundle_linux.go b/pkg/ocibundle/native/bundle_linux.go index 72d3a5bddc..52ec40358b 100644 --- a/pkg/ocibundle/native/bundle_linux.go +++ b/pkg/ocibundle/native/bundle_linux.go @@ -20,6 +20,7 @@ import ( apexlog "github.com/apex/log" "github.com/apptainer/apptainer/internal/pkg/build/oci" "github.com/apptainer/apptainer/internal/pkg/cache" + "github.com/apptainer/apptainer/internal/pkg/fakeroot" "github.com/apptainer/apptainer/internal/pkg/runtime/engine/config/oci/generate" "github.com/apptainer/apptainer/pkg/ocibundle" "github.com/apptainer/apptainer/pkg/ocibundle/tools" @@ -160,7 +161,9 @@ func (b *Bundle) Create(ctx context.Context, ociConfig *specs.Spec) error { b.setProcessArgs(g) // TODO - Handle custom env and user b.setProcessEnv(g) - b.setProcessUser(g) + if err := b.setProcessUser(g); err != nil { + return err + } return b.writeConfig(g) } @@ -170,14 +173,100 @@ func (b *Bundle) Path() string { return b.bundlePath } -func (b *Bundle) setProcessUser(g *generate.Generator) { +func (b *Bundle) setProcessUser(g *generate.Generator) error { // Set non-root uid/gid per Apptainer defaults uid := uint32(os.Getuid()) if uid != 0 { gid := uint32(os.Getgid()) g.Config.Process.User.UID = uid g.Config.Process.User.GID = gid + // Get user's configured subuid & subgid ranges + subuidRange, err := fakeroot.GetIDRange(fakeroot.SubUIDFile, uid) + if err != nil { + return err + } + // We must be able to map at least 0->65535 inside the container + if subuidRange.Size < 65536 { + return fmt.Errorf("subuid range size (%d) must be at least 65536", subuidRange.Size) + } + subgidRange, err := fakeroot.GetIDRange(fakeroot.SubGIDFile, uid) + if err != nil { + return err + } + if subgidRange.Size <= gid { + return fmt.Errorf("subuid range size (%d) must be at least 65536", subgidRange.Size) + } + + // Preserve own uid container->host, map everything else to subuid range. + if uid < 65536 { + g.Config.Linux.UIDMappings = []specs.LinuxIDMapping{ + { + ContainerID: 0, + HostID: subuidRange.HostID, + Size: uid, + }, + { + ContainerID: uid, + HostID: uid, + Size: 1, + }, + { + ContainerID: uid + 1, + HostID: subuidRange.HostID + uid, + Size: subuidRange.Size - uid, + }, + } + } else { + g.Config.Linux.UIDMappings = []specs.LinuxIDMapping{ + { + ContainerID: 0, + HostID: subuidRange.HostID, + Size: 65536, + }, + { + ContainerID: uid, + HostID: uid, + Size: 1, + }, + } + } + + // Preserve own gid container->host, map everything else to subgid range. + if gid < 65536 { + g.Config.Linux.GIDMappings = []specs.LinuxIDMapping{ + { + ContainerID: 0, + HostID: subgidRange.HostID, + Size: gid, + }, + { + ContainerID: gid, + HostID: gid, + Size: 1, + }, + { + ContainerID: gid + 1, + HostID: subgidRange.HostID + gid, + Size: subgidRange.Size - gid, + }, + } + } else { + g.Config.Linux.GIDMappings = []specs.LinuxIDMapping{ + { + ContainerID: 0, + HostID: subgidRange.HostID, + Size: 65536, + }, + { + ContainerID: gid, + HostID: gid, + Size: 1, + }, + } + } + g.Config.Linux.Namespaces = append(g.Config.Linux.Namespaces, specs.LinuxNamespace{Type: "user"}) } + return nil } func (b *Bundle) setProcessEnv(g *generate.Generator) { From f5e51c347c4496cdd1c94d1018e4a11a6c047105 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Mon, 21 Nov 2022 10:12:59 +0000 Subject: [PATCH 027/114] chore: Update deps for CI, packages, docs runc -> crun Need uidmap on Debian Signed-off-by: Edita Kizinevic --- INSTALL.md | 12 +++++++++--- dist/debian/control | 4 +++- dist/rpm/apptainer.spec.in | 3 +++ scripts/ci-deb-build-test | 4 +++- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index eb5e687f89..cf083b4922 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -28,7 +28,8 @@ sudo apt-get install -y \ fuse-overlayfs \ fakeroot \ cryptsetup \ - curl wget git + curl wget git \ + conmon crun ``` On CentOS/RHEL: @@ -47,7 +48,8 @@ sudo yum install -y \ fakeroot \ /usr/*bin/fuse2fs \ cryptsetup \ - wget git + wget git \ + conmon crun ``` On SLE/openSUSE @@ -59,9 +61,13 @@ sudo zypper install -y \ libuuid-devel \ openssl-devel \ cryptsetup sysuser-tools \ - gcc go + gcc go \ + conmon crun ``` +_Note - `crun` can be ommitted if you will not use the `apptainer oci` +commands, or the `--oci` execution mode._ + ## Install Go Apptainer is written in Go, and may require a newer version of Go than is diff --git a/dist/debian/control b/dist/debian/control index 4867a0c046..2d89cb9c6e 100644 --- a/dist/debian/control +++ b/dist/debian/control @@ -36,7 +36,9 @@ Depends: squashfuse, fuse2fs, fuse-overlayfs, - fakeroot + fakeroot, + conmon, + crun Conflicts: singularity-container Description: container platform focused on supporting "Mobility of Compute" formerly known as Singularity Mobility of Compute encapsulates the development to compute model diff --git a/dist/rpm/apptainer.spec.in b/dist/rpm/apptainer.spec.in index 0ea8f3a128..93c45ab1e4 100644 --- a/dist/rpm/apptainer.spec.in +++ b/dist/rpm/apptainer.spec.in @@ -107,8 +107,11 @@ BuildRequires: zlib-devel %if "%{_target_vendor}" == "suse" Requires: squashfs %else +Requires: shadow-utils Requires: squashfs-tools %endif +Requires: conmon +Requires: crun Requires: squashfuse Requires: fakeroot Requires: fuse-overlayfs diff --git a/scripts/ci-deb-build-test b/scripts/ci-deb-build-test index 6380922827..b8294e6cfd 100755 --- a/scripts/ci-deb-build-test +++ b/scripts/ci-deb-build-test @@ -30,7 +30,9 @@ apt-get install -y \ libssl-dev \ python2 \ uuid-dev \ - golang-go + golang-go \ + conmon \ + crun # for squashfuse_ll build apt-get install -y autoconf automake libtool pkg-config libfuse-dev zlib1g-dev From af57b656af09adb1f196b0f55f326e127f29873f Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Mon, 21 Nov 2022 10:29:54 +0000 Subject: [PATCH 028/114] e2e: use root and user OCI profiles in action tests Signed-off-by: Edita Kizinevic --- e2e/actions/oci.go | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/e2e/actions/oci.go b/e2e/actions/oci.go index bd811021b4..e586587d25 100644 --- a/e2e/actions/oci.go +++ b/e2e/actions/oci.go @@ -84,7 +84,7 @@ func (c actionTests) actionOciRun(t *testing.T) { c.env.RunApptainer( t, e2e.AsSubtest(tt.name), - e2e.WithProfile(e2e.OCIUserProfile), + e2e.WithProfile(profile), e2e.WithCommand("run"), // While we don't support args we are entering a /bin/sh interactively. e2e.ConsoleRun(e2e.ConsoleSendLine("exit")), @@ -184,15 +184,19 @@ func (c actionTests) actionOciShell(t *testing.T) { }, } - for _, tt := range tests { - c.env.RunApptainer( - t, - e2e.AsSubtest(tt.name), - e2e.WithProfile(e2e.OCIUserProfile), - e2e.WithCommand("shell"), - e2e.WithArgs(tt.argv...), - e2e.ConsoleRun(tt.consoleOps...), - e2e.ExpectExit(tt.exit), - ) + for _, profile := range []e2e.Profile{e2e.OCIRootProfile, e2e.OCIUserProfile} { + t.Run(profile.String(), func(t *testing.T) { + for _, tt := range tests { + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(profile), + e2e.WithCommand("shell"), + e2e.WithArgs(tt.argv...), + e2e.ConsoleRun(tt.consoleOps...), + e2e.ExpectExit(tt.exit), + ) + } + }) } } From 45632429e219e29ab997267af889a1309c43e5be Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Mon, 28 Nov 2022 11:38:39 +0000 Subject: [PATCH 029/114] pkg: Update / tidy rpm spec file Tidy up the rpm spec file. * Remove the manual handling of build root, GOPATH etc. Not needed as we are using go modules now. * Remove redundant explicit deps. * Fix crun -> runc dep for EL7. * Use rpm make_xxx macros instead of direct make calls. * Ensure all directories created are owned by package. Fixes sylabs/singularity#1142 Fixes sylabs/singularity#1141 Signed-off-by: Edita Kizinevic --- INSTALL.md | 4 +++- dist/rpm/apptainer.rpmlintrc | 7 +++++++ dist/rpm/apptainer.spec.in | 25 +++++++++++++++++++------ 3 files changed, 29 insertions(+), 7 deletions(-) create mode 100644 dist/rpm/apptainer.rpmlintrc diff --git a/INSTALL.md b/INSTALL.md index cf083b4922..d1dff6561c 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -52,6 +52,8 @@ sudo yum install -y \ conmon crun ``` +_Note - use `runc` instead of `crun` on CentOS/RHEL 7._ + On SLE/openSUSE ```sh @@ -65,7 +67,7 @@ sudo zypper install -y \ conmon crun ``` -_Note - `crun` can be ommitted if you will not use the `apptainer oci` +_Note - `crun` / `runc` can be ommitted if you will not use the `apptainer oci` commands, or the `--oci` execution mode._ ## Install Go diff --git a/dist/rpm/apptainer.rpmlintrc b/dist/rpm/apptainer.rpmlintrc new file mode 100644 index 0000000000..bac4de24ba --- /dev/null +++ b/dist/rpm/apptainer.rpmlintrc @@ -0,0 +1,7 @@ +addFilter(r'setuid-binary /usr/libexec/apptainer/bin/starter-suid') +addFilter(r'non-standard-executable-perm /usr/libexec/apptainer/bin/starter-suid') +addFilter(r'zero-length /etc/apptainer/capability.json') +addFilter(r'zero-length /etc/apptainer/global-pgp-public') +addFilter(r'readelf-failed /usr/bin/apptainer 'utf-8' codec can't decode byte 0xc2') +addFilter(r'readelf-failed /usr/libexec/apptainer/bin/starter 'utf-8' codec can't decode byte 0xc2') +addFilter(r'readelf-failed /usr/libexec/apptainer/bin/starter-suid 'utf-8' codec can't decode byte 0xc2') diff --git a/dist/rpm/apptainer.spec.in b/dist/rpm/apptainer.spec.in index 93c45ab1e4..54062229be 100644 --- a/dist/rpm/apptainer.spec.in +++ b/dist/rpm/apptainer.spec.in @@ -41,15 +41,17 @@ # The last singularity version number in EPEL/Fedora %global last_singularity_version 3.8.7-3 -Summary: Application and environment virtualization formerly known as Singularity Name: apptainer Version: @PACKAGE_RPM_VERSION@ Release: @PACKAGE_RELEASE@%{?dist} +Summary: Application and environment virtualization formerly known as Singularity + # See LICENSE.md for first party code (BSD-3-Clause and LBNL BSD) # See LICENSE_THIRD_PARTY.md for incorporated code (ASL 2.0) # See LICENSE_DEPENDENCIES.md for dependencies # License identifiers taken from: https://fedoraproject.org/wiki/Licensing License: BSD and LBNL BSD and ASL 2.0 + URL: https://apptainer.org Source: https://github.com/%{name}/%{name}/releases/download/v%{package_version}/%{name}-%{package_version}.tar.gz @PACKAGE_GOLANG_SOURCE@ @@ -91,11 +93,20 @@ Conflicts: sif-runtime BuildRequires: binutils-gold %endif BuildRequires: golang -BuildRequires: git BuildRequires: gcc BuildRequires: make -BuildRequires: libseccomp-devel +# Paths to runtime dependencies detected by mconfig, so must be present at build time. BuildRequires: cryptsetup +# Required for building bundled conmon +BuildRequires: libseccomp-devel +Requires: conmon +# crun requirement not satisfied on EL7 or SLES default repos - use runc there. +%if "%{_target_vendor}" == "suse" || 0%{?rhel} > 7 +Requires: crun +%else +Requires: runc +%endif +Requires: cryptsetup %if "%{?squashfuse_version}" != "" BuildRequires: autoconf BuildRequires: automake @@ -110,8 +121,6 @@ Requires: squashfs Requires: shadow-utils Requires: squashfs-tools %endif -Requires: conmon -Requires: crun Requires: squashfuse Requires: fakeroot Requires: fuse-overlayfs @@ -297,6 +306,7 @@ fi %if "%{?gocryptfs_version}" != "" %{_libexecdir}/%{name}/bin/gocryptfs %endif +%dir %{_libexecdir}/%{name}/cni %if "%{?squashfuse_version}" != "" %{_libexecdir}/%{name}/bin/squashfuse_ll %endif @@ -304,6 +314,9 @@ fi %{_libexecdir}/%{name}/lib %dir %{_sysconfdir}/%{name} %config(noreplace) %{_sysconfdir}/%{name}/* +%dir %{_sysconfdir}/%{name}/cgroups +%dir %{_sysconfdir}/%{name}/network +%dir %{_sysconfdir}/%{name}/seccomp-profiles %{_datadir}/bash-completion/completions/* %dir %{_sharedstatedir}/%{name} %dir %{_sharedstatedir}/%{name}/mnt @@ -315,9 +328,9 @@ fi %license LICENSE_DEPENDENCIES.md %doc README.md %doc CHANGELOG.md +%doc CONTRIBUTING.md %files suid %attr(4755, root, root) %{_libexecdir}/%{name}/bin/starter-suid %changelog - From b8d648dc2f65f86a5b63bca841e5afe4e32335e8 Mon Sep 17 00:00:00 2001 From: Dave Trudgian Date: Mon, 28 Nov 2022 15:22:34 +0000 Subject: [PATCH 030/114] Update INSTALL.md - fix typo from PR review. Co-authored-by: Mike Frisch Signed-off-by: Edita Kizinevic --- INSTALL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/INSTALL.md b/INSTALL.md index d1dff6561c..456d9c0f88 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -67,7 +67,7 @@ sudo zypper install -y \ conmon crun ``` -_Note - `crun` / `runc` can be ommitted if you will not use the `apptainer oci` +_Note - `crun` / `runc` can be omitted if you will not use the `apptainer oci` commands, or the `--oci` execution mode._ ## Install Go From 3e3e6fd8334070d9219ea8b2feae78b70a8e8989 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Mon, 28 Nov 2022 15:47:41 +0000 Subject: [PATCH 031/114] pkg: fix Ubuntu 18.04 deb build If `crun` is not available, require `runc`. Signed-off-by: Edita Kizinevic --- INSTALL.md | 6 ++++++ dist/debian/control | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/INSTALL.md b/INSTALL.md index 456d9c0f88..9abd960b1a 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -32,6 +32,12 @@ sudo apt-get install -y \ conmon crun ``` +_Note_: on Ubuntu 18.04 or Debian 10 leave out `conmon`, `crun`, and +`fuse-overlayfs` because they are not available, or install them from another +source. Leaving out the first two will prevent the `--oci` option from working +and leaving out the third will prevent `--overlay` and `--writable-tmpfs` +options from working without suid mode. + On CentOS/RHEL: ```sh diff --git a/dist/debian/control b/dist/debian/control index 2d89cb9c6e..3943922b2b 100644 --- a/dist/debian/control +++ b/dist/debian/control @@ -38,7 +38,7 @@ Depends: fuse-overlayfs, fakeroot, conmon, - crun + crun | runc Conflicts: singularity-container Description: container platform focused on supporting "Mobility of Compute" formerly known as Singularity Mobility of Compute encapsulates the development to compute model From f4c677c2537414383c501d3dd715de5b8961b4a7 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Mon, 21 Nov 2022 15:03:44 +0000 Subject: [PATCH 032/114] oci: move some code to launcher, use explicit config Move ID mapping and process setup code to the launcher, out of the oci bundle package. Add a minimal config for `--oci` mode, rather than starting with the default OCI config. Add explicit configuration of mounts, with `tmpfs` mount locations matching the native runtime with `--containall / --compat`. Ensure ID mapping is always explicit (including for root). Add tests touching the tmpfs locations. Signed-off-by: Edita Kizinevic --- e2e/actions/oci.go | 40 +++-- .../runtime/launcher/oci/launcher_linux.go | 26 ++- .../pkg/runtime/launcher/oci/mounts_linux.go | 146 +++++++++++++++++ .../pkg/runtime/launcher/oci/process_linux.go | 150 ++++++++++++++++++ .../pkg/runtime/launcher/oci/spec_linux.go | 88 ++++++++++ pkg/ocibundle/native/bundle_linux.go | 111 +------------ 6 files changed, 441 insertions(+), 120 deletions(-) create mode 100644 internal/pkg/runtime/launcher/oci/mounts_linux.go create mode 100644 internal/pkg/runtime/launcher/oci/process_linux.go create mode 100644 internal/pkg/runtime/launcher/oci/spec_linux.go diff --git a/e2e/actions/oci.go b/e2e/actions/oci.go index e586587d25..b2bf4cd9a3 100644 --- a/e2e/actions/oci.go +++ b/e2e/actions/oci.go @@ -132,18 +132,36 @@ func (c actionTests) actionOciExec(t *testing.T) { argv: []string{imageRef, "/bin/false"}, exit: 1, }, + { + name: "TouchTmp", + argv: []string{imageRef, "/bin/touch", "/tmp/test"}, + exit: 0, + }, + { + name: "TouchVarTmp", + argv: []string{imageRef, "/bin/touch", "/var/tmp/test"}, + exit: 0, + }, + { + name: "TouchHome", + argv: []string{imageRef, "/bin/sh", "-c", "touch $HOME"}, + exit: 0, + }, } - - for _, tt := range tests { - c.env.RunApptainer( - t, - e2e.AsSubtest(tt.name), - e2e.WithProfile(e2e.UserProfile), - e2e.WithCommand("exec"), - e2e.WithDir("/tmp"), - e2e.WithArgs(tt.argv...), - e2e.ExpectExit(tt.exit), - ) + for _, profile := range []e2e.Profile{e2e.OCIRootProfile, e2e.OCIUserProfile} { + t.Run(profile.String(), func(t *testing.T) { + for _, tt := range tests { + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(e2e.UserProfile), + e2e.WithCommand("exec"), + e2e.WithDir("/tmp"), + e2e.WithArgs(tt.argv...), + e2e.ExpectExit(tt.exit), + ) + } + }) } } diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index 045a010061..ee94ad2af6 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -285,6 +285,30 @@ func (l *Launcher) Exec(ctx context.Context, image string, process string, args } } + spec, err := MinimalSpec() + if err != nil { + return err + } + + spec.Process.User = l.getProcessUser() + uidMap, gidMap, err := l.getIDMaps() + if err != nil { + return err + } + spec.Linux.UIDMappings = uidMap + spec.Linux.GIDMappings = gidMap + cwd, err := l.getProcessCwd() + if err != nil { + return err + } + spec.Process.Cwd = cwd + + mounts, err := l.getMounts() + if err != nil { + return err + } + spec.Mounts = mounts + b, err := native.New( native.OptBundlePath(bundleDir), native.OptImageRef(image), @@ -296,7 +320,7 @@ func (l *Launcher) Exec(ctx context.Context, image string, process string, args return err } - if err := b.Create(ctx, nil); err != nil { + if err := b.Create(ctx, spec); err != nil { return err } diff --git a/internal/pkg/runtime/launcher/oci/mounts_linux.go b/internal/pkg/runtime/launcher/oci/mounts_linux.go new file mode 100644 index 0000000000..4720f7a28b --- /dev/null +++ b/internal/pkg/runtime/launcher/oci/mounts_linux.go @@ -0,0 +1,146 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2022, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +// Package oci implements a Launcher that will configure and launch a container +// with an OCI runtime. It also provides implementations of OCI state +// transitions that can be called directly, Create/Start/Kill etc. +package oci + +import ( + "fmt" + "strconv" + + "github.com/apptainer/apptainer/internal/pkg/util/user" + "github.com/opencontainers/runtime-spec/specs-go" +) + +// getMounts returns a mount list for the container's OCI runtime spec. +func (l *Launcher) getMounts() ([]specs.Mount, error) { + mounts := &[]specs.Mount{} + l.addProcMount(mounts) + l.addSysMount(mounts) + err := addDevMounts(mounts) + if err != nil { + return nil, fmt.Errorf("while configuring devpts mount: %w", err) + } + l.addTmpMounts(mounts) + err = l.addHomeMount(mounts) + if err != nil { + return nil, fmt.Errorf("while configuring home mount: %w", err) + } + return *mounts, nil +} + +// addBindMount adds a bind mount from src on host, to dst in container. +func (l *Launcher) addBindMount(mounts *[]specs.Mount, src, dst string) { + *mounts = append(*mounts, + specs.Mount{ + Source: src, + Destination: dst, + Type: "none", + Options: []string{"rbind", "nosuid", "nodev"}, + }) +} + +// addTmpMounts adds tmpfs mounts for /tmp and /var/tmp in the container. +func (l *Launcher) addTmpMounts(mounts *[]specs.Mount) { + *mounts = append(*mounts, + specs.Mount{ + Destination: "/tmp", + Type: "tmpfs", + Source: "tmpfs", + Options: []string{"nosuid", "relatime", "mode=777", "size=65536k"}, + }, + specs.Mount{ + Destination: "/tmp", + Type: "tmpfs", + Source: "tmpfs", + Options: []string{"nosuid", "relatime", "mode=777", "size=65536k"}, + }) +} + +// addDevMounts adds mounts to assemble a minimal /dev in the container. +func addDevMounts(mounts *[]specs.Mount) error { + group, err := user.GetGrNam("tty") + if err != nil { + return fmt.Errorf("while identifying tty gid: %w", err) + } + + *mounts = append(*mounts, + specs.Mount{ + Destination: "/dev", + Type: "tmpfs", + Source: "tmpfs", + Options: []string{"nosuid", "strictatime", "mode=755", "size=65536k"}, + }, + specs.Mount{ + Destination: "/dev/pts", + Type: "devpts", + Source: "devpts", + Options: []string{"nosuid", "noexec", "newinstance", "ptmxmode=0666", "mode=0620", "gid=" + strconv.Itoa(int(group.GID))}, + }, + specs.Mount{ + Destination: "/dev/shm", + Type: "tmpfs", + Source: "shm", + Options: []string{"nosuid", "noexec", "nodev", "mode=1777", "size=65536k"}, + }, + specs.Mount{ + Destination: "/dev/mqueue", + Type: "mqueue", + Source: "mqueue", + Options: []string{"nosuid", "noexec", "nodev"}, + }, + ) + + return nil +} + +// addProcMount adds the /proc tree in the container. +func (l *Launcher) addProcMount(mounts *[]specs.Mount) { + if l.cfg.Namespaces.PID { + *mounts = append(*mounts, + specs.Mount{ + Source: "proc", + Destination: "/proc", + Type: "proc", + }) + } else { + l.addBindMount(mounts, "/proc", "/proc") + } +} + +// addSysMount adds the /sys tree in the container. +func (l *Launcher) addSysMount(mounts *[]specs.Mount) { + *mounts = append(*mounts, + specs.Mount{ + Source: "sysfs", + Destination: "/sys", + Type: "sysfs", + Options: []string{"nosuid", "noexec", "nodev", "ro"}, + }) +} + +// addHomeMount adds a user home directory as a tmpfs mount. We are currently +// emulating `--compat` / `--containall`, so the user must specifically bind in +// their home directory from the host for it to be available. +func (l *Launcher) addHomeMount(mounts *[]specs.Mount) error { + pw, err := user.CurrentOriginal() + if err != nil { + return err + } + *mounts = append(*mounts, + specs.Mount{ + Destination: pw.Dir, + Type: "tmpfs", + Source: "tmpfs", + Options: []string{"nosuid", "relatime", "mode=755", "size=65536k", "uid=" + strconv.Itoa(int(pw.UID)), "gid=" + strconv.Itoa(int(pw.GID))}, + }) + return nil +} diff --git a/internal/pkg/runtime/launcher/oci/process_linux.go b/internal/pkg/runtime/launcher/oci/process_linux.go new file mode 100644 index 0000000000..4a770bece3 --- /dev/null +++ b/internal/pkg/runtime/launcher/oci/process_linux.go @@ -0,0 +1,150 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2022, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package oci + +import ( + "fmt" + "os" + + "github.com/apptainer/apptainer/internal/pkg/fakeroot" + "github.com/apptainer/apptainer/internal/pkg/util/user" + "github.com/opencontainers/runtime-spec/specs-go" +) + +// getProcessUser computes the uid/gid(s) to be set on process execution. +// Currently this only supports the same uid / primary gid as on the host. +// TODO - expand for fakeroot, and arbitrary mapped user. +func (l *Launcher) getProcessUser() specs.User { + return specs.User{ + UID: uint32(os.Getuid()), + GID: uint32(os.Getgid()), + } +} + +// getProcessCwd computes the Cwd that the container process should start in. +// Currently this is the user's tmpfs home directory (see --containall). +func (l *Launcher) getProcessCwd() (dir string, err error) { + pw, err := user.CurrentOriginal() + if err != nil { + return "", err + } + return pw.Dir, nil +} + +// getIDMaps returns uid and gid mappings appropriate for a non-root user, if required. +func (l *Launcher) getIDMaps() (uidMap, gidMap []specs.LinuxIDMapping, err error) { + uid := uint32(os.Getuid()) + // Root user gets pass-through mapping + if uid == 0 { + uidMap = []specs.LinuxIDMapping{ + { + ContainerID: 0, + HostID: 0, + Size: 65536, + }, + } + gidMap = []specs.LinuxIDMapping{ + { + ContainerID: 0, + HostID: 0, + Size: 65536, + }, + } + return uidMap, gidMap, nil + } + // Set non-root uid/gid per Apptainer defaults + gid := uint32(os.Getgid()) + // Get user's configured subuid & subgid ranges + subuidRange, err := fakeroot.GetIDRange(fakeroot.SubUIDFile, uid) + if err != nil { + return nil, nil, err + } + // We must be able to map at least 0->65535 inside the container + if subuidRange.Size < 65536 { + return nil, nil, fmt.Errorf("subuid range size (%d) must be at least 65536", subuidRange.Size) + } + subgidRange, err := fakeroot.GetIDRange(fakeroot.SubGIDFile, uid) + if err != nil { + return nil, nil, err + } + if subgidRange.Size <= gid { + return nil, nil, fmt.Errorf("subuid range size (%d) must be at least 65536", subgidRange.Size) + } + + // Preserve own uid container->host, map everything else to subuid range. + if uid < 65536 { + uidMap = []specs.LinuxIDMapping{ + { + ContainerID: 0, + HostID: subuidRange.HostID, + Size: uid, + }, + { + ContainerID: uid, + HostID: uid, + Size: 1, + }, + { + ContainerID: uid + 1, + HostID: subuidRange.HostID + uid, + Size: subuidRange.Size - uid, + }, + } + } else { + uidMap = []specs.LinuxIDMapping{ + { + ContainerID: 0, + HostID: subuidRange.HostID, + Size: 65536, + }, + { + ContainerID: uid, + HostID: uid, + Size: 1, + }, + } + } + + // Preserve own gid container->host, map everything else to subgid range. + if gid < 65536 { + gidMap = []specs.LinuxIDMapping{ + { + ContainerID: 0, + HostID: subgidRange.HostID, + Size: gid, + }, + { + ContainerID: gid, + HostID: gid, + Size: 1, + }, + { + ContainerID: gid + 1, + HostID: subgidRange.HostID + gid, + Size: subgidRange.Size - gid, + }, + } + } else { + gidMap = []specs.LinuxIDMapping{ + { + ContainerID: 0, + HostID: subgidRange.HostID, + Size: 65536, + }, + { + ContainerID: gid, + HostID: gid, + Size: 1, + }, + } + } + + return uidMap, gidMap, nil +} diff --git a/internal/pkg/runtime/launcher/oci/spec_linux.go b/internal/pkg/runtime/launcher/oci/spec_linux.go new file mode 100644 index 0000000000..90a557d021 --- /dev/null +++ b/internal/pkg/runtime/launcher/oci/spec_linux.go @@ -0,0 +1,88 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2022, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package oci + +import ( + "github.com/opencontainers/runtime-spec/specs-go" +) + +// MinimalSpec returns an OCI runtime spec with a minimal OCI configuration that +// is a starting point for compatibility with Apptainer's native launcher in +// `--compat` mode. +func MinimalSpec() (*specs.Spec, error) { + config := specs.Spec{ + Version: specs.Version, + } + config.Root = &specs.Root{ + Path: "rootfs", + // TODO - support writable-tmpfs / writable + Readonly: true, + } + config.Process = &specs.Process{ + Terminal: true, + // Default fallback to a shell at / - will generally be overwritten by + // the launcher. + Args: []string{"sh"}, + Cwd: "/", + } + config.Process.User = specs.User{} + config.Process.Env = []string{ + "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", + } + + // TODO - these are appropriate minimum for rootless. We need to tie into + // Apptainer's cap-add / cap-drop mechanism. + config.Process.Capabilities = &specs.LinuxCapabilities{ + Bounding: []string{ + "CAP_NET_BIND_SERVICE", + "CAP_KILL", + "CAP_AUDIT_WRITE", + }, + Permitted: []string{ + "CAP_NET_BIND_SERVICE", + "CAP_KILL", + "CAP_AUDIT_WRITE", + }, + Inheritable: []string{}, + Effective: []string{ + "CAP_NET_BIND_SERVICE", + "CAP_KILL", + "CAP_AUDIT_WRITE", + }, + Ambient: []string{ + "CAP_NET_BIND_SERVICE", + "CAP_KILL", + "CAP_AUDIT_WRITE", + }, + } + + // All mounts are added by the launcher, as it must handle flags. + config.Mounts = []specs.Mount{} + + config.Linux = &specs.Linux{ + // Minimum namespaces matching native runtime with --compat / --containall. + // TODO: ßAdditional namespaces can be added by launcher. + Namespaces: []specs.LinuxNamespace{ + { + Type: "ipc", + }, + { + Type: "pid", + }, + { + Type: "mount", + }, + { + Type: "user", + }, + }, + } + return &config, nil +} diff --git a/pkg/ocibundle/native/bundle_linux.go b/pkg/ocibundle/native/bundle_linux.go index 52ec40358b..6573396a43 100644 --- a/pkg/ocibundle/native/bundle_linux.go +++ b/pkg/ocibundle/native/bundle_linux.go @@ -20,7 +20,6 @@ import ( apexlog "github.com/apex/log" "github.com/apptainer/apptainer/internal/pkg/build/oci" "github.com/apptainer/apptainer/internal/pkg/cache" - "github.com/apptainer/apptainer/internal/pkg/fakeroot" "github.com/apptainer/apptainer/internal/pkg/runtime/engine/config/oci/generate" "github.com/apptainer/apptainer/pkg/ocibundle" "github.com/apptainer/apptainer/pkg/ocibundle/tools" @@ -157,13 +156,10 @@ func (b *Bundle) Create(ctx context.Context, ociConfig *specs.Spec) error { if err := os.RemoveAll(tmpDir); err != nil { return err } - + // ProcessArgs are set here, rather than in the launcher spec generation, as we need to + // consult the image Config to handle combining ENTRYPOINT/CMD with user + // provided args. b.setProcessArgs(g) - // TODO - Handle custom env and user - b.setProcessEnv(g) - if err := b.setProcessUser(g); err != nil { - return err - } return b.writeConfig(g) } @@ -173,107 +169,6 @@ func (b *Bundle) Path() string { return b.bundlePath } -func (b *Bundle) setProcessUser(g *generate.Generator) error { - // Set non-root uid/gid per Apptainer defaults - uid := uint32(os.Getuid()) - if uid != 0 { - gid := uint32(os.Getgid()) - g.Config.Process.User.UID = uid - g.Config.Process.User.GID = gid - // Get user's configured subuid & subgid ranges - subuidRange, err := fakeroot.GetIDRange(fakeroot.SubUIDFile, uid) - if err != nil { - return err - } - // We must be able to map at least 0->65535 inside the container - if subuidRange.Size < 65536 { - return fmt.Errorf("subuid range size (%d) must be at least 65536", subuidRange.Size) - } - subgidRange, err := fakeroot.GetIDRange(fakeroot.SubGIDFile, uid) - if err != nil { - return err - } - if subgidRange.Size <= gid { - return fmt.Errorf("subuid range size (%d) must be at least 65536", subgidRange.Size) - } - - // Preserve own uid container->host, map everything else to subuid range. - if uid < 65536 { - g.Config.Linux.UIDMappings = []specs.LinuxIDMapping{ - { - ContainerID: 0, - HostID: subuidRange.HostID, - Size: uid, - }, - { - ContainerID: uid, - HostID: uid, - Size: 1, - }, - { - ContainerID: uid + 1, - HostID: subuidRange.HostID + uid, - Size: subuidRange.Size - uid, - }, - } - } else { - g.Config.Linux.UIDMappings = []specs.LinuxIDMapping{ - { - ContainerID: 0, - HostID: subuidRange.HostID, - Size: 65536, - }, - { - ContainerID: uid, - HostID: uid, - Size: 1, - }, - } - } - - // Preserve own gid container->host, map everything else to subgid range. - if gid < 65536 { - g.Config.Linux.GIDMappings = []specs.LinuxIDMapping{ - { - ContainerID: 0, - HostID: subgidRange.HostID, - Size: gid, - }, - { - ContainerID: gid, - HostID: gid, - Size: 1, - }, - { - ContainerID: gid + 1, - HostID: subgidRange.HostID + gid, - Size: subgidRange.Size - gid, - }, - } - } else { - g.Config.Linux.GIDMappings = []specs.LinuxIDMapping{ - { - ContainerID: 0, - HostID: subgidRange.HostID, - Size: 65536, - }, - { - ContainerID: gid, - HostID: gid, - Size: 1, - }, - } - } - g.Config.Linux.Namespaces = append(g.Config.Linux.Namespaces, specs.LinuxNamespace{Type: "user"}) - } - return nil -} - -func (b *Bundle) setProcessEnv(g *generate.Generator) { - // Set default ENV values from image - g.Config.Process.Env = append(g.Config.Process.Env, b.imageSpec.Config.Env...) -} - func (b *Bundle) setProcessArgs(g *generate.Generator) { var processArgs []string From 2cdf179150e81bf9e31a381b29fa18a94bf9cc8a Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Mon, 21 Nov 2022 15:58:20 +0000 Subject: [PATCH 033/114] oci: Add --fakeroot support to --oci mode Initial --fakeroot support for --oci mode. Mirrors behavior with --compat / --contain. Closes sylabs/singularity#1035 Signed-off-by: Edita Kizinevic --- cmd/internal/cli/oci_linux.go | 8 + e2e/actions/oci.go | 6 +- .../runtime/engine/fakeroot/config/config.go | 1 + .../runtime/engine/fakeroot/engine_linux.go | 6 +- internal/pkg/runtime/launcher/oci/README.md | 141 ++++++++++++++++++ .../runtime/launcher/oci/launcher_linux.go | 32 ++-- .../pkg/runtime/launcher/oci/mounts_linux.go | 84 ++++++----- .../runtime/launcher/oci/oci_conmon_linux.go | 8 +- .../pkg/runtime/launcher/oci/oci_linux.go | 14 +- .../runtime/launcher/oci/oci_runc_linux.go | 110 ++++++++++++-- .../pkg/runtime/launcher/oci/process_linux.go | 71 ++++----- .../pkg/runtime/launcher/oci/spec_linux.go | 37 ++--- internal/pkg/util/starter/starter.go | 2 +- 13 files changed, 388 insertions(+), 132 deletions(-) create mode 100644 internal/pkg/runtime/launcher/oci/README.md diff --git a/cmd/internal/cli/oci_linux.go b/cmd/internal/cli/oci_linux.go index aa4beb630f..2d242ce5a5 100644 --- a/cmd/internal/cli/oci_linux.go +++ b/cmd/internal/cli/oci_linux.go @@ -10,6 +10,10 @@ package cli import ( + "errors" + "os" + "os/exec" + "github.com/apptainer/apptainer/docs" "github.com/apptainer/apptainer/internal/app/apptainer" "github.com/apptainer/apptainer/internal/pkg/runtime/launcher/oci" @@ -154,6 +158,10 @@ var OciRunCmd = &cobra.Command{ PreRun: CheckRoot, Run: func(cmd *cobra.Command, args []string) { if err := apptainer.OciRun(cmd.Context(), args[0], &ociArgs); err != nil { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + os.Exit(exitErr.ExitCode()) + } sylog.Fatalf("%s", err) } }, diff --git a/e2e/actions/oci.go b/e2e/actions/oci.go index b2bf4cd9a3..2b3cd490cc 100644 --- a/e2e/actions/oci.go +++ b/e2e/actions/oci.go @@ -76,7 +76,7 @@ func (c actionTests) actionOciRun(t *testing.T) { }, } - for _, profile := range []e2e.Profile{e2e.OCIRootProfile, e2e.OCIUserProfile} { + for _, profile := range e2e.OCIProfiles { t.Run(profile.String(), func(t *testing.T) { for _, tt := range tests { cmdArgs := []string{tt.imageRef} @@ -148,7 +148,7 @@ func (c actionTests) actionOciExec(t *testing.T) { exit: 0, }, } - for _, profile := range []e2e.Profile{e2e.OCIRootProfile, e2e.OCIUserProfile} { + for _, profile := range e2e.OCIProfiles { t.Run(profile.String(), func(t *testing.T) { for _, tt := range tests { c.env.RunApptainer( @@ -202,7 +202,7 @@ func (c actionTests) actionOciShell(t *testing.T) { }, } - for _, profile := range []e2e.Profile{e2e.OCIRootProfile, e2e.OCIUserProfile} { + for _, profile := range e2e.OCIProfiles { t.Run(profile.String(), func(t *testing.T) { for _, tt := range tests { c.env.RunApptainer( diff --git a/internal/pkg/runtime/engine/fakeroot/config/config.go b/internal/pkg/runtime/engine/fakeroot/config/config.go index 833d31d831..c7fede2169 100644 --- a/internal/pkg/runtime/engine/fakeroot/config/config.go +++ b/internal/pkg/runtime/engine/fakeroot/config/config.go @@ -19,4 +19,5 @@ type EngineConfig struct { Envs []string `json:"envs"` Home string `json:"home"` BuildEnv bool `json:"buildEnv"` + NoPIDNS bool `json:"NoPIDNS"` } diff --git a/internal/pkg/runtime/engine/fakeroot/engine_linux.go b/internal/pkg/runtime/engine/fakeroot/engine_linux.go index 7f2cd1d108..6951360d7f 100644 --- a/internal/pkg/runtime/engine/fakeroot/engine_linux.go +++ b/internal/pkg/runtime/engine/fakeroot/engine_linux.go @@ -101,7 +101,11 @@ func (e *EngineOperations) PrepareConfig(starterConfig *starter.Config) error { g.AddOrReplaceLinuxNamespace(specs.UserNamespace, "") g.AddOrReplaceLinuxNamespace(specs.MountNamespace, "") - g.AddOrReplaceLinuxNamespace(specs.PIDNamespace, "") + + // If we enter a PID NS in the --oci action -> oci run flow, then crun / runc will fail. + if !e.EngineConfig.NoPIDNS { + g.AddOrReplaceLinuxNamespace(specs.PIDNamespace, "") + } uid := uint32(os.Getuid()) gid := uint32(os.Getgid()) diff --git a/internal/pkg/runtime/launcher/oci/README.md b/internal/pkg/runtime/launcher/oci/README.md new file mode 100644 index 0000000000..1c3275108d --- /dev/null +++ b/internal/pkg/runtime/launcher/oci/README.md @@ -0,0 +1,141 @@ +# internal/pkg/runtime/launcher/oci + +This package contains routines that configure and launch a container in an OCI +bundle format, using a low-level OCI runtime, either `crun` or `runc` at this +time. `crun` is currently preferred. `runc` is used where `crun` is not +available. + +**Note** - at present, all functionality works with either `crun` or `runc`. +However, in future `crun` may be required for all functionality, as `runc` does +not support some limited ID mappings etc. that may be beneficial in an HPC +scenario. + +The package contrasts with `internal/pkg/runtime/launcher/native` which executes +Apptainer format containers (SIF/Sandbox/squashfs/ext3), using one of our own +runtime engines (`internal/pkg/runtime/engine/*`). + +There are two flows that are implemented here. + +* Basic OCI runtime operations agains an existing bundle, which will be executed + via the `apptainer oci` command group. These are not widely used by + end-users of apptainer. +* A `Launcher`, that implements an `Exec` function that will be called by + 'actions' (run/shell/exec) in `--oci` mode, and will: + * Prepare an OCI bundle according to `launcher.Options` passed through from + the CLI layer. + * Execute the bundle, interactively, via the OCI Run operation. + +**Note** - this area of code is under heavy development for experimental. +It is likely that it will be heavily refactored, and split, in future. + +## Basic OCI Operations + +The following files implement basic OCI operations on a runtime bundle: + +### `oci_linux.go` + +Defines constants, path resolution, and minimal bundle locking functions. + +### `oci_runc_linux.go` + +Holds implementations of the Run / Start / Exec / Kill / Delete / Pause / Resume +/ State OCI runtime operations. + +See + + +These functions are thin wrappers around the `runc`/`crun` operations of the +same name. + +### `oci_conmon_linux.go` + +Hold an implementation of the Create OCI runtime operation. This calls out to +`conmon`, which in turn calls `crun` or `runc`. + +`conmon` is used to manage logging and console streams for containers that are +started backgrounded, so we don't have to do that ourselves. + +### `oci_attach_linux.go` + +Implements an `Attach` function, which can attach the user's console to the +streams of a container running in the background, which is being monitored by +conmon. + +### Testing + +End-to-end flows of basic OCI operations on an existing bundle are tested in the +OCI group of the e2e suite, `e2e/oci`. + +## Launcher Flow + +The `Launcher` type connects the standard apptainer CLI actions +(run/shell/exec), to execution of an OCI container in a native bundle. Invoked +with the `--oci` flag, this is in contrast to running a Apptainer format +container, with Apptainer's own runtime engine. + +### `spec_linux.go` + +Provides a minimal OCI runtime spec, that will form the basis of container +execution that is roughly comparable to running a native apptainer container +with `--compat` (`--containall`). + +### `mounts_linux.go` + +Provides code handling the addition of required mounts to the OCI runtime spec. + +### `process_linux.go` + +Provides code handling configuration of container process execution, including +user mapping. + +### `launcher_linux.go` + +Implements `Launcher.Exec`, which is called from the CLI layer. It will: + +* Create a temporary bundle directory. +* Use `pkg/ocibundle/native` to retrieve the specified image, and extract it in + the temporary bundle. +* Configure the container by creating an appropriate runtime spec. +* Call the interactive OCI Run function to execute the container with `crun` or + `runc`. + +### Namespace Considerations + +An OCI container started via `Launch.Exec` as a non-root user always uses at +least one user namespace. + +The user namespace is created *prior to* calling `runc` or `crun`, so we'll call +it an *outer* user namespace. + +Creation of this outer user namespace is via using the `RunNS` function, instead +of `Run`. The `RunNS` function executes the Apptainer `starter` binary, with a +minimal configuration of the fakeroot engine ( +`internal/pkg/runtime/engine/fakeroot/config`). + +The `starter` will create a user namespace and ID mapping, and will then execute +`apptainer oci run` to perform the basic OCI Run operation against the bundle +that the `Launcher.Exec` function has prepared. + +The outer user namespace from which `runc` or `crun` is called *always* maps the +host user id to root inside the userns. + +When a container is run in `--fakeroot` mode, the outer user namespace is the +only user namespace. The OCI runtime config does not request any additional +userns or ID mapping be performed by `crun` / `runc`. + +When a container is **not** run in `--fakeroot` mode, the OCI runtime config for +the bundle requests that `crun` / `runc`: + +* Create another, inner, user namespace for the container. +* Apply an ID mapping which reverses the 'fakeroot' outer ID mapping. + +I.E. when a container runs without `--fakeroot`, the ID mapping is: + +* User ID on host (1001) +* Root in outer user namespace (0) +* User ID in container (1001) + +### Testing + +End-to-end testing of the launcher flow is via the `e2e/actions` suite. Tests +prefixed `oci`. diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index ee94ad2af6..911f122700 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -204,9 +204,6 @@ func checkOpts(lo launcher.Options) error { badOpt = append(badOpt, "PwdPath") } - if lo.Fakeroot { - badOpt = append(badOpt, "Fakeroot") - } if lo.Boot { badOpt = append(badOpt, "Boot") } @@ -291,12 +288,18 @@ func (l *Launcher) Exec(ctx context.Context, image string, process string, args } spec.Process.User = l.getProcessUser() - uidMap, gidMap, err := l.getIDMaps() - if err != nil { - return err + + // If we are *not* requesting fakeroot, then we need to map the container + // uid back to host uid, through the initial fakeroot userns. + if !l.cfg.Fakeroot && os.Getuid() != 0 { + uidMap, gidMap, err := l.getReverseUserMaps() + if err != nil { + return err + } + spec.Linux.UIDMappings = uidMap + spec.Linux.GIDMappings = gidMap } - spec.Linux.UIDMappings = uidMap - spec.Linux.GIDMappings = gidMap + cwd, err := l.getProcessCwd() if err != nil { return err @@ -329,9 +332,16 @@ func (l *Launcher) Exec(ctx context.Context, image string, process string, args return fmt.Errorf("while generating container id: %w", err) } - err = Run(ctx, id.String(), b.Path(), "") - if exiterr, ok := err.(*exec.ExitError); ok { - os.Exit(exiterr.ExitCode()) + if os.Getuid() == 0 { + // Direct execution of runc/crun run. + err = Run(ctx, id.String(), b.Path(), "") + } else { + // Reexec apptainer oci run in a userns with mappings. + err = RunNS(ctx, id.String(), b.Path(), "") + } + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + os.Exit(exitErr.ExitCode()) } return err } diff --git a/internal/pkg/runtime/launcher/oci/mounts_linux.go b/internal/pkg/runtime/launcher/oci/mounts_linux.go index 4720f7a28b..b43d6e7ee3 100644 --- a/internal/pkg/runtime/launcher/oci/mounts_linux.go +++ b/internal/pkg/runtime/launcher/oci/mounts_linux.go @@ -14,6 +14,7 @@ package oci import ( "fmt" + "os" "strconv" "github.com/apptainer/apptainer/internal/pkg/util/user" @@ -37,17 +38,6 @@ func (l *Launcher) getMounts() ([]specs.Mount, error) { return *mounts, nil } -// addBindMount adds a bind mount from src on host, to dst in container. -func (l *Launcher) addBindMount(mounts *[]specs.Mount, src, dst string) { - *mounts = append(*mounts, - specs.Mount{ - Source: src, - Destination: dst, - Type: "none", - Options: []string{"rbind", "nosuid", "nodev"}, - }) -} - // addTmpMounts adds tmpfs mounts for /tmp and /var/tmp in the container. func (l *Launcher) addTmpMounts(mounts *[]specs.Mount) { *mounts = append(*mounts, @@ -67,9 +57,19 @@ func (l *Launcher) addTmpMounts(mounts *[]specs.Mount) { // addDevMounts adds mounts to assemble a minimal /dev in the container. func addDevMounts(mounts *[]specs.Mount) error { - group, err := user.GetGrNam("tty") - if err != nil { - return fmt.Errorf("while identifying tty gid: %w", err) + ptsMount := specs.Mount{ + Destination: "/dev/pts", + Type: "devpts", + Source: "devpts", + Options: []string{"nosuid", "noexec", "newinstance", "ptmxmode=0666", "mode=0620"}, + } + + if os.Getuid() == 0 { + group, err := user.GetGrNam("tty") + if err != nil { + return fmt.Errorf("while identifying tty gid: %w", err) + } + ptsMount.Options = append(ptsMount.Options, fmt.Sprintf("gid=%d", group.GID)) } *mounts = append(*mounts, @@ -79,12 +79,7 @@ func addDevMounts(mounts *[]specs.Mount) error { Source: "tmpfs", Options: []string{"nosuid", "strictatime", "mode=755", "size=65536k"}, }, - specs.Mount{ - Destination: "/dev/pts", - Type: "devpts", - Source: "devpts", - Options: []string{"nosuid", "noexec", "newinstance", "ptmxmode=0666", "mode=0620", "gid=" + strconv.Itoa(int(group.GID))}, - }, + ptsMount, specs.Mount{ Destination: "/dev/shm", Type: "tmpfs", @@ -104,33 +99,50 @@ func addDevMounts(mounts *[]specs.Mount) error { // addProcMount adds the /proc tree in the container. func (l *Launcher) addProcMount(mounts *[]specs.Mount) { - if l.cfg.Namespaces.PID { + *mounts = append(*mounts, + specs.Mount{ + Source: "proc", + Destination: "/proc", + Type: "proc", + }) +} + +// addSysMount adds the /sys tree in the container. +func (l *Launcher) addSysMount(mounts *[]specs.Mount) { + if os.Getuid() == 0 { *mounts = append(*mounts, specs.Mount{ - Source: "proc", - Destination: "/proc", - Type: "proc", + Source: "sysfs", + Destination: "/sys", + Type: "sysfs", + Options: []string{"nosuid", "noexec", "nodev", "ro"}, }) } else { - l.addBindMount(mounts, "/proc", "/proc") + *mounts = append(*mounts, + specs.Mount{ + Source: "/sys", + Destination: "/sys", + Type: "none", + Options: []string{"rbind", "nosuid", "noexec", "nodev", "ro"}, + }) } } -// addSysMount adds the /sys tree in the container. -func (l *Launcher) addSysMount(mounts *[]specs.Mount) { - *mounts = append(*mounts, - specs.Mount{ - Source: "sysfs", - Destination: "/sys", - Type: "sysfs", - Options: []string{"nosuid", "noexec", "nodev", "ro"}, - }) -} - // addHomeMount adds a user home directory as a tmpfs mount. We are currently // emulating `--compat` / `--containall`, so the user must specifically bind in // their home directory from the host for it to be available. func (l *Launcher) addHomeMount(mounts *[]specs.Mount) error { + if l.cfg.Fakeroot { + *mounts = append(*mounts, + specs.Mount{ + Destination: "/root", + Type: "tmpfs", + Source: "tmpfs", + Options: []string{"nosuid", "relatime", "mode=755", "size=65536k"}, + }) + return nil + } + pw, err := user.CurrentOriginal() if err != nil { return err diff --git a/internal/pkg/runtime/launcher/oci/oci_conmon_linux.go b/internal/pkg/runtime/launcher/oci/oci_conmon_linux.go index 0649c83172..ba69214991 100644 --- a/internal/pkg/runtime/launcher/oci/oci_conmon_linux.go +++ b/internal/pkg/runtime/launcher/oci/oci_conmon_linux.go @@ -91,6 +91,12 @@ func Create(containerID, bundlePath string) error { defer startParent.Close() apptainerBin := filepath.Join(buildcfg.BINDIR, "apptainer") + + rsd, err := runtimeStateDir() + if err != nil { + return err + } + cmdArgs := []string{ "--api-version", "1", "--cid", containerID, @@ -101,7 +107,7 @@ func Create(containerID, bundlePath string) error { "--container-pidfile", path.Join(sd, containerPidFile), "--log-path", path.Join(sd, containerLogFile), "--runtime-arg", "--root", - "--runtime-arg", runtimeStateDir(), + "--runtime-arg", rsd, "--runtime-arg", "--log", "--runtime-arg", path.Join(sd, runcLogFile), "--full-attach", diff --git a/internal/pkg/runtime/launcher/oci/oci_linux.go b/internal/pkg/runtime/launcher/oci/oci_linux.go index af219dcb4d..f78d9e0cd7 100644 --- a/internal/pkg/runtime/launcher/oci/oci_linux.go +++ b/internal/pkg/runtime/launcher/oci/oci_linux.go @@ -56,12 +56,16 @@ func runtime() (path string, err error) { } // runtimeStateDir returns path to use for crun/runc's state handling. -func runtimeStateDir() string { - uid := os.Getuid() - if uid == 0 { - return "/run/apptainer-oci" +func runtimeStateDir() (path string, err error) { + // Ensure we get correct uid for host if we were re-exec'd in id mapped userns + pw, err := user.CurrentOriginal() + if err != nil { + return "", err + } + if pw.UID == 0 { + return "/run/apptainer-oci", nil } - return fmt.Sprintf("/run/user/%d/apptainer-oci", uid) + return fmt.Sprintf("/run/user/%d/apptainer-oci", pw.UID), nil } // stateDir returns the path to container state handled by conmon/apptainer diff --git a/internal/pkg/runtime/launcher/oci/oci_runc_linux.go b/internal/pkg/runtime/launcher/oci/oci_runc_linux.go index cb1fff97b9..cc3a5d693b 100644 --- a/internal/pkg/runtime/launcher/oci/oci_runc_linux.go +++ b/internal/pkg/runtime/launcher/oci/oci_runc_linux.go @@ -18,8 +18,12 @@ import ( "os" "os/exec" "path/filepath" + "strings" - "github.com/apptainer/apptainer/internal/pkg/util/bin" + "github.com/apptainer/apptainer/internal/pkg/buildcfg" + fakerootConfig "github.com/apptainer/apptainer/internal/pkg/runtime/engine/fakeroot/config" + "github.com/apptainer/apptainer/internal/pkg/util/starter" + "github.com/apptainer/apptainer/pkg/runtime/engine/config" "github.com/apptainer/apptainer/pkg/sylog" ) @@ -29,8 +33,13 @@ func Delete(ctx context.Context, containerID string) error { if err != nil { return err } + rsd, err := runtimeStateDir() + if err != nil { + return err + } + runtimeArgs := []string{ - "--root", runtimeStateDir(), + "--root", rsd, "delete", containerID, } @@ -71,8 +80,13 @@ func Exec(containerID string, cmdArgs []string) error { if err != nil { return err } + rsd, err := runtimeStateDir() + if err != nil { + return err + } + runtimeArgs := []string{ - "--root", runtimeStateDir(), + "--root", rsd, "exec", containerID, } @@ -91,8 +105,13 @@ func Kill(containerID string, killSignal string) error { if err != nil { return err } + rsd, err := runtimeStateDir() + if err != nil { + return err + } + runtimeArgs := []string{ - "--root", runtimeStateDir(), + "--root", rsd, "kill", containerID, killSignal, @@ -112,8 +131,13 @@ func Pause(containerID string) error { if err != nil { return err } + rsd, err := runtimeStateDir() + if err != nil { + return err + } + runtimeArgs := []string{ - "--root", runtimeStateDir(), + "--root", rsd, "pause", containerID, } @@ -132,8 +156,13 @@ func Resume(containerID string) error { if err != nil { return err } + rsd, err := runtimeStateDir() + if err != nil { + return err + } + runtimeArgs := []string{ - "--root", runtimeStateDir(), + "--root", rsd, "resume", containerID, } @@ -161,8 +190,13 @@ func Run(ctx context.Context, containerID, bundlePath, pidFile string) error { return fmt.Errorf("failed to change directory to %s: %s", absBundle, err) } + rsd, err := runtimeStateDir() + if err != nil { + return err + } + runtimeArgs := []string{ - "--root", runtimeStateDir(), + "--root", rsd, "run", "-b", absBundle, } @@ -178,14 +212,58 @@ func Run(ctx context.Context, containerID, bundlePath, pidFile string) error { return cmd.Run() } +// RunNS reexecs apptainer in a user namespace, with supplied uid/gid mapping, calling oci run. +func RunNS(ctx context.Context, containerID, bundlePath, pidFile string) error { + absBundle, err := filepath.Abs(bundlePath) + if err != nil { + return fmt.Errorf("failed to determine bundle absolute path: %s", err) + } + + if err := os.Chdir(absBundle); err != nil { + return fmt.Errorf("failed to change directory to %s: %s", absBundle, err) + } + + args := []string{ + filepath.Join(buildcfg.BINDIR, "apptainer"), + "oci", + "run", + "-b", absBundle, + containerID, + } + if pidFile != "" { + args = append(args, "--pid-file="+pidFile) + } + + sylog.Debugf("Calling fakeroot engine to execute %q", strings.Join(args, " ")) + + cfg := &config.Common{ + EngineName: fakerootConfig.Name, + ContainerID: "fakeroot", + EngineConfig: &fakerootConfig.EngineConfig{Args: args, NoPIDNS: true}, + } + + return starter.Run( + "Apptainer oci userns", + cfg, + starter.WithStdin(os.Stdin), + starter.WithStdout(os.Stdout), + starter.WithStderr(os.Stderr), + ) +} + // Start starts a previously created container func Start(containerID string) error { - runtimeBin, err := bin.FindBin("crun") + runtimeBin, err := runtime() if err != nil { return err } + rsd, err := runtimeStateDir() + if err != nil { + return err + } + runtimeArgs := []string{ - "--root", runtimeStateDir(), + "--root", rsd, "start", containerID, } @@ -204,8 +282,13 @@ func State(containerID string) error { if err != nil { return err } + rsd, err := runtimeStateDir() + if err != nil { + return err + } + runtimeArgs := []string{ - "--root", runtimeStateDir(), + "--root", rsd, "state", containerID, } @@ -224,8 +307,13 @@ func Update(containerID, cgFile string) error { if err != nil { return err } + rsd, err := runtimeStateDir() + if err != nil { + return err + } + runtimeArgs := []string{ - "--root", runtimeStateDir(), + "--root", rsd, "update", "-r", cgFile, containerID, diff --git a/internal/pkg/runtime/launcher/oci/process_linux.go b/internal/pkg/runtime/launcher/oci/process_linux.go index 4a770bece3..21dd98fdc5 100644 --- a/internal/pkg/runtime/launcher/oci/process_linux.go +++ b/internal/pkg/runtime/launcher/oci/process_linux.go @@ -22,6 +22,12 @@ import ( // Currently this only supports the same uid / primary gid as on the host. // TODO - expand for fakeroot, and arbitrary mapped user. func (l *Launcher) getProcessUser() specs.User { + if l.cfg.Fakeroot { + return specs.User{ + UID: 0, + GID: 0, + } + } return specs.User{ UID: uint32(os.Getuid()), GID: uint32(os.Getgid()), @@ -31,6 +37,10 @@ func (l *Launcher) getProcessUser() specs.User { // getProcessCwd computes the Cwd that the container process should start in. // Currently this is the user's tmpfs home directory (see --containall). func (l *Launcher) getProcessCwd() (dir string, err error) { + if l.cfg.Fakeroot { + return "/root", nil + } + pw, err := user.CurrentOriginal() if err != nil { return "", err @@ -38,35 +48,20 @@ func (l *Launcher) getProcessCwd() (dir string, err error) { return pw.Dir, nil } -// getIDMaps returns uid and gid mappings appropriate for a non-root user, if required. -func (l *Launcher) getIDMaps() (uidMap, gidMap []specs.LinuxIDMapping, err error) { +// getReverseUserMaps returns uid and gid mappings that re-map container uid to host +// uid. This 'reverses' the host user to container root mapping in the initial +// userns from which the OCI runtime is launched. +// +// host 1001 -> fakeroot userns 0 -> container 1001 +func (l *Launcher) getReverseUserMaps() (uidMap, gidMap []specs.LinuxIDMapping, err error) { uid := uint32(os.Getuid()) - // Root user gets pass-through mapping - if uid == 0 { - uidMap = []specs.LinuxIDMapping{ - { - ContainerID: 0, - HostID: 0, - Size: 65536, - }, - } - gidMap = []specs.LinuxIDMapping{ - { - ContainerID: 0, - HostID: 0, - Size: 65536, - }, - } - return uidMap, gidMap, nil - } - // Set non-root uid/gid per Apptainer defaults gid := uint32(os.Getgid()) // Get user's configured subuid & subgid ranges subuidRange, err := fakeroot.GetIDRange(fakeroot.SubUIDFile, uid) if err != nil { return nil, nil, err } - // We must be able to map at least 0->65535 inside the container + // We must always be able to map at least 0->65535 inside the container, so we cover 'nobody'. if subuidRange.Size < 65536 { return nil, nil, fmt.Errorf("subuid range size (%d) must be at least 65536", subuidRange.Size) } @@ -74,26 +69,25 @@ func (l *Launcher) getIDMaps() (uidMap, gidMap []specs.LinuxIDMapping, err error if err != nil { return nil, nil, err } - if subgidRange.Size <= gid { + if subgidRange.Size < 65536 { return nil, nil, fmt.Errorf("subuid range size (%d) must be at least 65536", subgidRange.Size) } - // Preserve own uid container->host, map everything else to subuid range. - if uid < 65536 { + if uid < subuidRange.Size { uidMap = []specs.LinuxIDMapping{ { ContainerID: 0, - HostID: subuidRange.HostID, + HostID: 1, Size: uid, }, { ContainerID: uid, - HostID: uid, + HostID: 0, Size: 1, }, { ContainerID: uid + 1, - HostID: subuidRange.HostID + uid, + HostID: uid + 1, Size: subuidRange.Size - uid, }, } @@ -101,33 +95,32 @@ func (l *Launcher) getIDMaps() (uidMap, gidMap []specs.LinuxIDMapping, err error uidMap = []specs.LinuxIDMapping{ { ContainerID: 0, - HostID: subuidRange.HostID, - Size: 65536, + HostID: 1, + Size: subuidRange.Size, }, { ContainerID: uid, - HostID: uid, + HostID: 0, Size: 1, }, } } - // Preserve own gid container->host, map everything else to subgid range. - if gid < 65536 { + if gid < subgidRange.Size { gidMap = []specs.LinuxIDMapping{ { ContainerID: 0, - HostID: subgidRange.HostID, + HostID: 1, Size: gid, }, { ContainerID: gid, - HostID: gid, + HostID: 0, Size: 1, }, { ContainerID: gid + 1, - HostID: subgidRange.HostID + gid, + HostID: gid + 1, Size: subgidRange.Size - gid, }, } @@ -135,12 +128,12 @@ func (l *Launcher) getIDMaps() (uidMap, gidMap []specs.LinuxIDMapping, err error gidMap = []specs.LinuxIDMapping{ { ContainerID: 0, - HostID: subgidRange.HostID, - Size: 65536, + HostID: 1, + Size: subgidRange.Size, }, { ContainerID: gid, - HostID: gid, + HostID: 0, Size: 1, }, } diff --git a/internal/pkg/runtime/launcher/oci/spec_linux.go b/internal/pkg/runtime/launcher/oci/spec_linux.go index 90a557d021..ebd23afbd9 100644 --- a/internal/pkg/runtime/launcher/oci/spec_linux.go +++ b/internal/pkg/runtime/launcher/oci/spec_linux.go @@ -41,25 +41,17 @@ func MinimalSpec() (*specs.Spec, error) { // Apptainer's cap-add / cap-drop mechanism. config.Process.Capabilities = &specs.LinuxCapabilities{ Bounding: []string{ - "CAP_NET_BIND_SERVICE", - "CAP_KILL", - "CAP_AUDIT_WRITE", - }, - Permitted: []string{ - "CAP_NET_BIND_SERVICE", + "CAP_CHOWN", + "CAP_DAC_OVERRIDE", + "CAP_FOWNER", + "CAP_FSETID", "CAP_KILL", - "CAP_AUDIT_WRITE", - }, - Inheritable: []string{}, - Effective: []string{ "CAP_NET_BIND_SERVICE", - "CAP_KILL", - "CAP_AUDIT_WRITE", - }, - Ambient: []string{ - "CAP_NET_BIND_SERVICE", - "CAP_KILL", - "CAP_AUDIT_WRITE", + "CAP_SETFCAP", + "CAP_SETGID", + "CAP_SETPCAP", + "CAP_SETUID", + "CAP_SYS_CHROOT", }, } @@ -68,19 +60,16 @@ func MinimalSpec() (*specs.Spec, error) { config.Linux = &specs.Linux{ // Minimum namespaces matching native runtime with --compat / --containall. - // TODO: ßAdditional namespaces can be added by launcher. + // TODO: Additional namespaces to be added by launcher. Namespaces: []specs.LinuxNamespace{ { - Type: "ipc", - }, - { - Type: "pid", + Type: specs.IPCNamespace, }, { - Type: "mount", + Type: specs.PIDNamespace, }, { - Type: "user", + Type: specs.MountNamespace, }, }, } diff --git a/internal/pkg/util/starter/starter.go b/internal/pkg/util/starter/starter.go index 6b19ef6ec8..55c5d8b75c 100644 --- a/internal/pkg/util/starter/starter.go +++ b/internal/pkg/util/starter/starter.go @@ -118,7 +118,7 @@ func Run(name string, config *config.Common, ops ...CommandOp) error { cmd.Stderr = c.stderr if err := cmd.Run(); err != nil { - return fmt.Errorf("while running %s: %s", c.path, err) + return fmt.Errorf("while running %s: %w", c.path, err) } return nil } From 5f196e9e44656141aad43b726863c52a60022b7c Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Tue, 29 Nov 2022 16:28:47 +0000 Subject: [PATCH 034/114] fix: use conf file value for oci tmpfs size Use the configuration file sessiondir max size value for --oci mode tmpfs mounts. Increase the default from 16M -> 64M. The 16M default is very low, and has periodically caused issues running programs that create even small amounts of temporary data on --contained filesystems. Fixes sylabs/singularity#1140 Signed-off-by: Edita Kizinevic --- CHANGELOG.md | 2 + .../pkg/confgen/testdata/test_1.out.correct | 7 +-- internal/pkg/confgen/testdata/test_2.in | 7 +-- .../pkg/confgen/testdata/test_2.out.correct | 7 +-- internal/pkg/confgen/testdata/test_3.in | 7 +-- .../pkg/confgen/testdata/test_3.out.correct | 7 +-- .../pkg/confgen/testdata/test_default.tmpl | 7 +-- .../runtime/launcher/oci/launcher_linux.go | 12 ++++- .../launcher/oci/launcher_linux_test.go | 11 +++- .../pkg/runtime/launcher/oci/mounts_linux.go | 53 +++++++++++++++---- pkg/util/apptainerconf/config.go | 8 +-- 11 files changed, 92 insertions(+), 36 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f8696354ae..4e5674c888 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,8 @@ For older changes see the [archived Singularity change log](https://github.com/a `runc` to manage containers. - The `apptainer oci` flags `--sync-socket`, `--empty-process`, and `--timeout` have been removed. +- `sessiondir maxsize` in `apptainer.conf` now defaults to 64 MiB for new + installations. This is an increase from 16 MiB in prior versions. ### New Features & Functionality diff --git a/internal/pkg/confgen/testdata/test_1.out.correct b/internal/pkg/confgen/testdata/test_1.out.correct index 0cfec2ec2d..7fbea721a8 100644 --- a/internal/pkg/confgen/testdata/test_1.out.correct +++ b/internal/pkg/confgen/testdata/test_1.out.correct @@ -146,9 +146,10 @@ mount slave = yes # SESSIONDIR MAXSIZE: [STRING] # DEFAULT: 64 -# This specifies how large the default sessiondir should be (in MB) and it will -# only affect users who use the "--contain" options and don't also specify a -# location to do default read/writes to (e.g. "--workdir" or "--home"). +# This specifies how large the default tmpfs sessiondir should be (in MB). +# The sessiondir is used to hold data written to isolated directories when +# running with --contain and ephemeral changes when running with --writable-tmpfs. +# In --oci mode, each tmpfs mount in the container can be up to this size. sessiondir max size = 64 diff --git a/internal/pkg/confgen/testdata/test_2.in b/internal/pkg/confgen/testdata/test_2.in index e42de0ba5d..110e0fe7c2 100644 --- a/internal/pkg/confgen/testdata/test_2.in +++ b/internal/pkg/confgen/testdata/test_2.in @@ -146,9 +146,10 @@ mount slave = yes # SESSIONDIR MAXSIZE: [STRING] # DEFAULT: 64 -# This specifies how large the default sessiondir should be (in MB) and it will -# only affect users who use the "--contain" options and don't also specify a -# location to do default read/writes to (e.g. "--workdir" or "--home"). +# This specifies how large the default tmpfs sessiondir should be (in MB). +# The sessiondir is used to hold data written to isolated directories when +# running with --contain and ephemeral changes when running with --writable-tmpfs, +# and the equivalent data in --oci mode. sessiondir max size = 64 diff --git a/internal/pkg/confgen/testdata/test_2.out.correct b/internal/pkg/confgen/testdata/test_2.out.correct index e9c044bcaf..4c980da8ff 100644 --- a/internal/pkg/confgen/testdata/test_2.out.correct +++ b/internal/pkg/confgen/testdata/test_2.out.correct @@ -146,9 +146,10 @@ mount slave = yes # SESSIONDIR MAXSIZE: [STRING] # DEFAULT: 64 -# This specifies how large the default sessiondir should be (in MB) and it will -# only affect users who use the "--contain" options and don't also specify a -# location to do default read/writes to (e.g. "--workdir" or "--home"). +# This specifies how large the default tmpfs sessiondir should be (in MB). +# The sessiondir is used to hold data written to isolated directories when +# running with --contain and ephemeral changes when running with --writable-tmpfs. +# In --oci mode, each tmpfs mount in the container can be up to this size. sessiondir max size = 64 diff --git a/internal/pkg/confgen/testdata/test_3.in b/internal/pkg/confgen/testdata/test_3.in index e2c70b9066..c61f15800f 100644 --- a/internal/pkg/confgen/testdata/test_3.in +++ b/internal/pkg/confgen/testdata/test_3.in @@ -137,9 +137,10 @@ mount slave = yes # SESSIONDIR MAXSIZE: [STRING] # DEFAULT: 64 -# This specifies how large the default sessiondir should be (in MB) and it will -# only affect users who use the "--contain" options and don't also specify a -# location to do default read/writes to (e.g. "--workdir" or "--home"). +# This specifies how large the default tmpfs sessiondir should be (in MB). +# The sessiondir is used to hold data written to isolated directories when +# running with --contain and ephemeral changes when running with --writable-tmpfs, +# and the equivalent data in --oci mode. sessiondir max size = 64 diff --git a/internal/pkg/confgen/testdata/test_3.out.correct b/internal/pkg/confgen/testdata/test_3.out.correct index e9c044bcaf..4c980da8ff 100644 --- a/internal/pkg/confgen/testdata/test_3.out.correct +++ b/internal/pkg/confgen/testdata/test_3.out.correct @@ -146,9 +146,10 @@ mount slave = yes # SESSIONDIR MAXSIZE: [STRING] # DEFAULT: 64 -# This specifies how large the default sessiondir should be (in MB) and it will -# only affect users who use the "--contain" options and don't also specify a -# location to do default read/writes to (e.g. "--workdir" or "--home"). +# This specifies how large the default tmpfs sessiondir should be (in MB). +# The sessiondir is used to hold data written to isolated directories when +# running with --contain and ephemeral changes when running with --writable-tmpfs. +# In --oci mode, each tmpfs mount in the container can be up to this size. sessiondir max size = 64 diff --git a/internal/pkg/confgen/testdata/test_default.tmpl b/internal/pkg/confgen/testdata/test_default.tmpl index d13b50c400..4579403e2d 100644 --- a/internal/pkg/confgen/testdata/test_default.tmpl +++ b/internal/pkg/confgen/testdata/test_default.tmpl @@ -148,9 +148,10 @@ mount slave = {{ if eq .MountSlave true }}yes{{ else }}no{{ end }} # SESSIONDIR MAXSIZE: [STRING] # DEFAULT: 64 -# This specifies how large the default sessiondir should be (in MB) and it will -# only affect users who use the "--contain" options and don't also specify a -# location to do default read/writes to (e.g. "--workdir" or "--home"). +# This specifies how large the default tmpfs sessiondir should be (in MB). +# The sessiondir is used to hold data written to isolated directories when +# running with --contain and ephemeral changes when running with --writable-tmpfs. +# In --oci mode, each tmpfs mount in the container can be up to this size. sessiondir max size = {{ .SessiondirMaxSize }} diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index 911f122700..e375829085 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -26,6 +26,7 @@ import ( "github.com/apptainer/apptainer/pkg/ocibundle/native" "github.com/apptainer/apptainer/pkg/syfs" "github.com/apptainer/apptainer/pkg/sylog" + "github.com/apptainer/apptainer/pkg/util/apptainerconf" useragent "github.com/apptainer/apptainer/pkg/util/user-agent" "github.com/containers/image/v5/types" "github.com/google/uuid" @@ -39,7 +40,8 @@ var ( // Launcher will holds configuration for, and will launch a container using an // OCI runtime. type Launcher struct { - cfg launcher.Options + cfg launcher.Options + apptainerConf *apptainerconf.File } // NewLauncher returns a oci.Launcher with an initial configuration set by opts. @@ -54,7 +56,13 @@ func NewLauncher(opts ...launcher.Option) (*Launcher, error) { if err := checkOpts(lo); err != nil { return nil, err } - return &Launcher{lo}, nil + + c := apptainerconf.GetCurrentConfig() + if c == nil { + return nil, fmt.Errorf("apptainer configuration is not initialized") + } + + return &Launcher{cfg: lo, apptainerConf: c}, nil } // checkOpts ensures that options set are supported by the oci.Launcher. diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux_test.go b/internal/pkg/runtime/launcher/oci/launcher_linux_test.go index a64b87e1be..e1e4db355b 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux_test.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux_test.go @@ -14,9 +14,16 @@ import ( "testing" "github.com/apptainer/apptainer/internal/pkg/runtime/launcher" + "github.com/apptainer/apptainer/pkg/util/apptainerconf" ) func TestNewLauncher(t *testing.T) { + sc, err := apptainerconf.GetConfig(nil) + if err != nil { + t.Fatalf("while initializing apptainerconf: %s", err) + } + apptainerconf.SetCurrentConfig(sc) + tests := []struct { name string opts []launcher.Option @@ -25,7 +32,7 @@ func TestNewLauncher(t *testing.T) { }{ { name: "default", - want: &Launcher{}, + want: &Launcher{apptainerConf: sc}, wantErr: false, }, { @@ -33,7 +40,7 @@ func TestNewLauncher(t *testing.T) { opts: []launcher.Option{ launcher.OptHome("/home/test", false, false), }, - want: &Launcher{cfg: launcher.Options{HomeDir: "/home/test"}}, + want: &Launcher{cfg: launcher.Options{HomeDir: "/home/test"}, apptainerConf: sc}, }, { name: "unsupportedOption", diff --git a/internal/pkg/runtime/launcher/oci/mounts_linux.go b/internal/pkg/runtime/launcher/oci/mounts_linux.go index b43d6e7ee3..434672bd26 100644 --- a/internal/pkg/runtime/launcher/oci/mounts_linux.go +++ b/internal/pkg/runtime/launcher/oci/mounts_linux.go @@ -15,7 +15,6 @@ package oci import ( "fmt" "os" - "strconv" "github.com/apptainer/apptainer/internal/pkg/util/user" "github.com/opencontainers/runtime-spec/specs-go" @@ -26,7 +25,7 @@ func (l *Launcher) getMounts() ([]specs.Mount, error) { mounts := &[]specs.Mount{} l.addProcMount(mounts) l.addSysMount(mounts) - err := addDevMounts(mounts) + err := l.addDevMounts(mounts) if err != nil { return nil, fmt.Errorf("while configuring devpts mount: %w", err) } @@ -41,22 +40,33 @@ func (l *Launcher) getMounts() ([]specs.Mount, error) { // addTmpMounts adds tmpfs mounts for /tmp and /var/tmp in the container. func (l *Launcher) addTmpMounts(mounts *[]specs.Mount) { *mounts = append(*mounts, + specs.Mount{ Destination: "/tmp", Type: "tmpfs", Source: "tmpfs", - Options: []string{"nosuid", "relatime", "mode=777", "size=65536k"}, + Options: []string{ + "nosuid", + "relatime", + "mode=777", + fmt.Sprintf("size=%dm", l.apptainerConf.SessiondirMaxSize), + }, }, specs.Mount{ - Destination: "/tmp", + Destination: "/var/tmp", Type: "tmpfs", Source: "tmpfs", - Options: []string{"nosuid", "relatime", "mode=777", "size=65536k"}, + Options: []string{ + "nosuid", + "relatime", + "mode=777", + fmt.Sprintf("size=%dm", l.apptainerConf.SessiondirMaxSize), + }, }) } // addDevMounts adds mounts to assemble a minimal /dev in the container. -func addDevMounts(mounts *[]specs.Mount) error { +func (l *Launcher) addDevMounts(mounts *[]specs.Mount) error { ptsMount := specs.Mount{ Destination: "/dev/pts", Type: "devpts", @@ -77,14 +87,25 @@ func addDevMounts(mounts *[]specs.Mount) error { Destination: "/dev", Type: "tmpfs", Source: "tmpfs", - Options: []string{"nosuid", "strictatime", "mode=755", "size=65536k"}, + Options: []string{ + "nosuid", + "strictatime", + "mode=755", + fmt.Sprintf("size=%dm", l.apptainerConf.SessiondirMaxSize), + }, }, ptsMount, specs.Mount{ Destination: "/dev/shm", Type: "tmpfs", Source: "shm", - Options: []string{"nosuid", "noexec", "nodev", "mode=1777", "size=65536k"}, + Options: []string{ + "nosuid", + "noexec", + "nodev", + "mode=1777", + fmt.Sprintf("size=%dm", l.apptainerConf.SessiondirMaxSize), + }, }, specs.Mount{ Destination: "/dev/mqueue", @@ -138,7 +159,12 @@ func (l *Launcher) addHomeMount(mounts *[]specs.Mount) error { Destination: "/root", Type: "tmpfs", Source: "tmpfs", - Options: []string{"nosuid", "relatime", "mode=755", "size=65536k"}, + Options: []string{ + "nosuid", + "relatime", + "mode=755", + fmt.Sprintf("size=%dm", l.apptainerConf.SessiondirMaxSize), + }, }) return nil } @@ -152,7 +178,14 @@ func (l *Launcher) addHomeMount(mounts *[]specs.Mount) error { Destination: pw.Dir, Type: "tmpfs", Source: "tmpfs", - Options: []string{"nosuid", "relatime", "mode=755", "size=65536k", "uid=" + strconv.Itoa(int(pw.UID)), "gid=" + strconv.Itoa(int(pw.GID))}, + Options: []string{ + "nosuid", + "relatime", + "mode=755", + fmt.Sprintf("size=%dm", l.apptainerConf.SessiondirMaxSize), + fmt.Sprintf("uid=%d", pw.UID), + fmt.Sprintf("gid=%d", pw.GID), + }, }) return nil } diff --git a/pkg/util/apptainerconf/config.go b/pkg/util/apptainerconf/config.go index bf8623dc26..f9acd171a2 100644 --- a/pkg/util/apptainerconf/config.go +++ b/pkg/util/apptainerconf/config.go @@ -293,10 +293,10 @@ mount slave = {{ if eq .MountSlave true }}yes{{ else }}no{{ end }} # SESSIONDIR MAXSIZE: [STRING] # DEFAULT: 64 -# This specifies how large the default sessiondir should be (in MB). It will -# affect users who use the "--contain" options and don't also specify a -# location to do default read/writes to (e.g. "--workdir" or "--home") and -# it will also affect users of "--writable-tmpfs". +# This specifies how large the default tmpfs sessiondir should be (in MB). +# The sessiondir is used to hold data written to isolated directories when +# running with --contain and ephemeral changes when running with --writable-tmpfs. +# In --oci mode, each tmpfs mount in the container can be up to this size. sessiondir max size = {{ .SessiondirMaxSize }} # LIMIT CONTAINER OWNERS: [STRING] From 8d8de038cd045755cef7a2989d2e5dc1732b4bcc Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Thu, 1 Dec 2022 12:26:01 +0000 Subject: [PATCH 035/114] feat: oci: support namespace flags Support namespace request CLI options. * --ipc - no effect, always used in --oci mode. * --net - only supported with --network none. * --pid - no effect, always used in --oci mode. * -u / --userns - only effective for root, non-root always uses user ns. * --uts Add info logging where the option is redundant. Closes sylabs/singularity#1026 Signed-off-by: Edita Kizinevic --- .../runtime/launcher/oci/launcher_linux.go | 57 +++++++++++++------ 1 file changed, 39 insertions(+), 18 deletions(-) diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index e375829085..1def813970 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -30,6 +30,7 @@ import ( useragent "github.com/apptainer/apptainer/pkg/util/user-agent" "github.com/containers/image/v5/types" "github.com/google/uuid" + "github.com/opencontainers/runtime-spec/specs-go" ) var ( @@ -143,25 +144,10 @@ func checkOpts(lo launcher.Options) error { badOpt = append(badOpt, "NoEval") } - if lo.Namespaces.IPC { - badOpt = append(badOpt, "Namespaces.IPC") - } - if lo.Namespaces.Net { - badOpt = append(badOpt, "Namespaces.Net") - } - if lo.Namespaces.PID { - badOpt = append(badOpt, "Namespaces.PID") - } - if lo.Namespaces.UTS { - badOpt = append(badOpt, "Namespaces.UTS") - } - if lo.Namespaces.User { - badOpt = append(badOpt, "Namespaces.User") - } - // Network always set in CLI layer even if network namespace not requested. - if lo.Namespaces.Net && lo.Network != "" { - badOpt = append(badOpt, "Network") + // We only support isolation at present + if lo.Namespaces.Net && lo.Network != "none" { + badOpt = append(badOpt, "Network (except none)") } if len(lo.NetworkArgs) > 0 { @@ -314,6 +300,41 @@ func (l *Launcher) Exec(ctx context.Context, image string, process string, args } spec.Process.Cwd = cwd + if l.cfg.Namespaces.IPC { + sylog.Infof("--oci runtime always uses an IPC namespace, ipc flag is redundant.") + } + + // Currently supports only `--network none`, i.e. isolated loopback only. + // Launcher.checkopts enforces this. + if l.cfg.Namespaces.Net { + spec.Linux.Namespaces = append( + spec.Linux.Namespaces, + specs.LinuxNamespace{Type: specs.NetworkNamespace}, + ) + } + + if l.cfg.Namespaces.PID { + sylog.Infof("--oci runtime always uses a PID namespace, pid flag is redundant.") + } + + if l.cfg.Namespaces.User { + if os.Getuid() == 0 { + spec.Linux.Namespaces = append( + spec.Linux.Namespaces, + specs.LinuxNamespace{Type: specs.UserNamespace}, + ) + } else { + sylog.Infof("--oci runtime always uses a user namespace when run as a non-root userns, user flag is redundant.") + } + } + + if l.cfg.Namespaces.UTS { + spec.Linux.Namespaces = append( + spec.Linux.Namespaces, + specs.LinuxNamespace{Type: specs.UTSNamespace}, + ) + } + mounts, err := l.getMounts() if err != nil { return err From de885a3f1f595633e1dd18932cb551aa93c82608 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Thu, 1 Dec 2022 13:22:01 +0000 Subject: [PATCH 036/114] chore: refactor for easier testing Signed-off-by: Edita Kizinevic --- .../runtime/launcher/oci/launcher_linux.go | 100 +++++++----------- .../pkg/runtime/launcher/oci/spec_linux.go | 78 +++++++++++--- 2 files changed, 99 insertions(+), 79 deletions(-) diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index 1def813970..3f0adf3d15 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -233,6 +233,42 @@ func checkOpts(lo launcher.Options) error { return nil } +// createSpec produces an OCI runtime specification, suitable to launch a +// container. This spec excludes ProcessArgs, as these have to be computed where +// the image config is available, to account for the image's CMD / ENTRYPOINT. +func (l *Launcher) createSpec() (*specs.Spec, error) { + spec := minimalSpec() + + spec.Process.User = l.getProcessUser() + + // If we are *not* requesting fakeroot, then we need to map the container + // uid back to host uid, through the initial fakeroot userns. + if !l.cfg.Fakeroot && os.Getuid() != 0 { + uidMap, gidMap, err := l.getReverseUserMaps() + if err != nil { + return nil, err + } + spec.Linux.UIDMappings = uidMap + spec.Linux.GIDMappings = gidMap + } + + spec = addNamespaces(spec, l.cfg.Namespaces) + + cwd, err := l.getProcessCwd() + if err != nil { + return nil, err + } + spec.Process.Cwd = cwd + + mounts, err := l.getMounts() + if err != nil { + return nil, err + } + spec.Mounts = mounts + + return &spec, nil +} + // Exec will interactively execute a container via the runc low-level runtime. // image is a reference to an OCI image, e.g. docker://ubuntu or oci:/tmp/mycontainer func (l *Launcher) Exec(ctx context.Context, image string, process string, args []string, instanceName string) error { @@ -276,71 +312,11 @@ func (l *Launcher) Exec(ctx context.Context, image string, process string, args } } - spec, err := MinimalSpec() + spec, err := l.createSpec() if err != nil { - return err + return fmt.Errorf("while creating OCI spec: %w", err) } - spec.Process.User = l.getProcessUser() - - // If we are *not* requesting fakeroot, then we need to map the container - // uid back to host uid, through the initial fakeroot userns. - if !l.cfg.Fakeroot && os.Getuid() != 0 { - uidMap, gidMap, err := l.getReverseUserMaps() - if err != nil { - return err - } - spec.Linux.UIDMappings = uidMap - spec.Linux.GIDMappings = gidMap - } - - cwd, err := l.getProcessCwd() - if err != nil { - return err - } - spec.Process.Cwd = cwd - - if l.cfg.Namespaces.IPC { - sylog.Infof("--oci runtime always uses an IPC namespace, ipc flag is redundant.") - } - - // Currently supports only `--network none`, i.e. isolated loopback only. - // Launcher.checkopts enforces this. - if l.cfg.Namespaces.Net { - spec.Linux.Namespaces = append( - spec.Linux.Namespaces, - specs.LinuxNamespace{Type: specs.NetworkNamespace}, - ) - } - - if l.cfg.Namespaces.PID { - sylog.Infof("--oci runtime always uses a PID namespace, pid flag is redundant.") - } - - if l.cfg.Namespaces.User { - if os.Getuid() == 0 { - spec.Linux.Namespaces = append( - spec.Linux.Namespaces, - specs.LinuxNamespace{Type: specs.UserNamespace}, - ) - } else { - sylog.Infof("--oci runtime always uses a user namespace when run as a non-root userns, user flag is redundant.") - } - } - - if l.cfg.Namespaces.UTS { - spec.Linux.Namespaces = append( - spec.Linux.Namespaces, - specs.LinuxNamespace{Type: specs.UTSNamespace}, - ) - } - - mounts, err := l.getMounts() - if err != nil { - return err - } - spec.Mounts = mounts - b, err := native.New( native.OptBundlePath(bundleDir), native.OptImageRef(image), diff --git a/internal/pkg/runtime/launcher/oci/spec_linux.go b/internal/pkg/runtime/launcher/oci/spec_linux.go index ebd23afbd9..243a8fd8d9 100644 --- a/internal/pkg/runtime/launcher/oci/spec_linux.go +++ b/internal/pkg/runtime/launcher/oci/spec_linux.go @@ -10,13 +10,30 @@ package oci import ( + "os" + + "github.com/apptainer/apptainer/internal/pkg/runtime/launcher" + "github.com/apptainer/apptainer/pkg/sylog" "github.com/opencontainers/runtime-spec/specs-go" ) -// MinimalSpec returns an OCI runtime spec with a minimal OCI configuration that +// defaultNamespaces matching native runtime with --compat / --containall. +var defaultNamespaces = []specs.LinuxNamespace{ + { + Type: specs.IPCNamespace, + }, + { + Type: specs.PIDNamespace, + }, + { + Type: specs.MountNamespace, + }, +} + +// minimalSpec returns an OCI runtime spec with a minimal OCI configuration that // is a starting point for compatibility with Apptainer's native launcher in // `--compat` mode. -func MinimalSpec() (*specs.Spec, error) { +func minimalSpec() specs.Spec { config := specs.Spec{ Version: specs.Version, } @@ -58,20 +75,47 @@ func MinimalSpec() (*specs.Spec, error) { // All mounts are added by the launcher, as it must handle flags. config.Mounts = []specs.Mount{} - config.Linux = &specs.Linux{ - // Minimum namespaces matching native runtime with --compat / --containall. - // TODO: Additional namespaces to be added by launcher. - Namespaces: []specs.LinuxNamespace{ - { - Type: specs.IPCNamespace, - }, - { - Type: specs.PIDNamespace, - }, - { - Type: specs.MountNamespace, - }, - }, + config.Linux = &specs.Linux{Namespaces: defaultNamespaces} + return config +} + +// addNamespaces adds requested namespace, if appropriate, to an existing spec. +// It is assumed that spec contains at least the defaultNamespaces. +func addNamespaces(spec specs.Spec, ns launcher.Namespaces) specs.Spec { + if ns.IPC { + sylog.Infof("--oci runtime always uses an IPC namespace, ipc flag is redundant.") + } + + // Currently supports only `--network none`, i.e. isolated loopback only. + // Launcher.checkopts enforces this. + if ns.Net { + spec.Linux.Namespaces = append( + spec.Linux.Namespaces, + specs.LinuxNamespace{Type: specs.NetworkNamespace}, + ) } - return &config, nil + + if ns.PID { + sylog.Infof("--oci runtime always uses a PID namespace, pid flag is redundant.") + } + + if ns.User { + if os.Getuid() == 0 { + spec.Linux.Namespaces = append( + spec.Linux.Namespaces, + specs.LinuxNamespace{Type: specs.UserNamespace}, + ) + } else { + sylog.Infof("--oci runtime always uses a user namespace when run as a non-root userns, user flag is redundant.") + } + } + + if ns.UTS { + spec.Linux.Namespaces = append( + spec.Linux.Namespaces, + specs.LinuxNamespace{Type: specs.UTSNamespace}, + ) + } + + return spec } From 14405c8c143f554cb0f87ef1952c030610b80ea4 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Thu, 1 Dec 2022 13:51:38 +0000 Subject: [PATCH 037/114] test: oci: Test_addNamespaces Signed-off-by: Edita Kizinevic --- .../launcher/oci/launcher_linux_test.go | 4 ++ .../runtime/launcher/oci/spec_linux_test.go | 72 +++++++++++++++++++ 2 files changed, 76 insertions(+) create mode 100644 internal/pkg/runtime/launcher/oci/spec_linux_test.go diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux_test.go b/internal/pkg/runtime/launcher/oci/launcher_linux_test.go index e1e4db355b..25ff92f791 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux_test.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux_test.go @@ -14,10 +14,14 @@ import ( "testing" "github.com/apptainer/apptainer/internal/pkg/runtime/launcher" + "github.com/apptainer/apptainer/internal/pkg/test" "github.com/apptainer/apptainer/pkg/util/apptainerconf" ) func TestNewLauncher(t *testing.T) { + test.DropPrivilege(t) + defer test.ResetPrivilege(t) + sc, err := apptainerconf.GetConfig(nil) if err != nil { t.Fatalf("while initializing apptainerconf: %s", err) diff --git a/internal/pkg/runtime/launcher/oci/spec_linux_test.go b/internal/pkg/runtime/launcher/oci/spec_linux_test.go new file mode 100644 index 0000000000..7a1d4759dc --- /dev/null +++ b/internal/pkg/runtime/launcher/oci/spec_linux_test.go @@ -0,0 +1,72 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2022, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package oci + +import ( + "reflect" + "testing" + + "github.com/apptainer/apptainer/internal/pkg/runtime/launcher" + "github.com/apptainer/apptainer/internal/pkg/test" + "github.com/opencontainers/runtime-spec/specs-go" +) + +func Test_addNamespaces(t *testing.T) { + test.DropPrivilege(t) + defer test.ResetPrivilege(t) + + tests := []struct { + name string + ns launcher.Namespaces + wantNS []specs.LinuxNamespace + }{ + { + name: "none", + ns: launcher.Namespaces{}, + wantNS: defaultNamespaces, + }, + { + name: "pid", + ns: launcher.Namespaces{PID: true}, + wantNS: defaultNamespaces, + }, + { + name: "ipc", + ns: launcher.Namespaces{IPC: true}, + wantNS: defaultNamespaces, + }, + { + name: "user", + ns: launcher.Namespaces{User: true}, + wantNS: defaultNamespaces, + }, + { + name: "net", + ns: launcher.Namespaces{Net: true}, + wantNS: append(defaultNamespaces, specs.LinuxNamespace{Type: specs.NetworkNamespace}), + }, + { + name: "uts", + ns: launcher.Namespaces{UTS: true}, + wantNS: append(defaultNamespaces, specs.LinuxNamespace{Type: specs.UTSNamespace}), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + spec := minimalSpec() + newSpec := addNamespaces(spec, tt.ns) + newNS := newSpec.Linux.Namespaces + if !reflect.DeepEqual(newNS, tt.wantNS) { + t.Errorf("addNamespaces() got %v, want %v", newNS, tt.wantNS) + } + }) + } +} From 53ac7f8d4517da7a583430f73b488e2168fa8e6a Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Thu, 1 Dec 2022 13:59:16 +0000 Subject: [PATCH 038/114] e2e: minimal --oci namespace request tests Signed-off-by: Edita Kizinevic --- e2e/actions/actions.go | 7 +++-- e2e/actions/oci.go | 65 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 3 deletions(-) diff --git a/e2e/actions/actions.go b/e2e/actions/actions.go index 165469e7bc..d61d00ac44 100644 --- a/e2e/actions/actions.go +++ b/e2e/actions/actions.go @@ -2912,8 +2912,9 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { // // OCI Runtime Mode // - "ociRun": c.actionOciRun, // apptainer run --oci - "ociExec": c.actionOciExec, // apptainer exec --oci - "ociShell": c.actionOciShell, // apptainer shell --oci + "ociRun": c.actionOciRun, // apptainer run --oci + "ociExec": c.actionOciExec, // apptainer exec --oci + "ociShell": c.actionOciShell, // apptainer shell --oci + "ociNetwork": c.actionOciNetwork, // apptainer exec --oci --net } } diff --git a/e2e/actions/oci.go b/e2e/actions/oci.go index 2b3cd490cc..a65b7b7c65 100644 --- a/e2e/actions/oci.go +++ b/e2e/actions/oci.go @@ -147,6 +147,11 @@ func (c actionTests) actionOciExec(t *testing.T) { argv: []string{imageRef, "/bin/sh", "-c", "touch $HOME"}, exit: 0, }, + { + name: "UTSNamespace", + argv: []string{"--uts", imageRef, "true"}, + exit: 0, + }, } for _, profile := range e2e.OCIProfiles { t.Run(profile.String(), func(t *testing.T) { @@ -218,3 +223,63 @@ func (c actionTests) actionOciShell(t *testing.T) { }) } } + +func (c actionTests) actionOciNetwork(t *testing.T) { + e2e.EnsureOCIImage(t, c.env) + imageRef := "oci-archive:" + c.env.OCIImagePath + + tests := []struct { + name string + profile e2e.Profile + netType string + expectExit int + }{ + { + name: "InvalidNetworkRoot", + profile: e2e.OCIRootProfile, + netType: "bridge", + expectExit: 255, + }, + { + name: "InvalidNetworkUser", + profile: e2e.OCIUserProfile, + netType: "bridge", + expectExit: 255, + }, + { + name: "InvalidNetworkFakeroot", + profile: e2e.OCIFakerootProfile, + netType: "bridge", + expectExit: 255, + }, + { + name: "NoneNetworkRoot", + profile: e2e.OCIRootProfile, + netType: "none", + expectExit: 0, + }, + { + name: "NoneNetworkUser", + profile: e2e.OCIUserProfile, + netType: "none", + expectExit: 0, + }, + { + name: "NoneNetworkFakeRoot", + profile: e2e.OCIFakerootProfile, + netType: "none", + expectExit: 0, + }, + } + + for _, tt := range tests { + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(tt.profile), + e2e.WithCommand("exec"), + e2e.WithArgs("--net", "--network", tt.netType, imageRef, "id"), + e2e.ExpectExit(tt.expectExit), + ) + } +} From f73e2081c9bc9b8181757860feee335ce02529a6 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Thu, 1 Dec 2022 16:07:57 +0000 Subject: [PATCH 039/114] fix: Don't set Process.Terminal in oci launcher if no term If we are running such that stdin is not a terminal, then the OCI runtime config should have Process.Terminal=false to avoid errors. https://github.com/opencontainers/runc/blob/main/docs/terminals.md#issues Signed-off-by: Edita Kizinevic --- internal/pkg/runtime/launcher/oci/launcher_linux.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index 3f0adf3d15..3454a346da 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -19,6 +19,7 @@ import ( "os" "os/exec" "strings" + "syscall" "github.com/apptainer/apptainer/internal/pkg/buildcfg" "github.com/apptainer/apptainer/internal/pkg/cache" @@ -31,6 +32,7 @@ import ( "github.com/containers/image/v5/types" "github.com/google/uuid" "github.com/opencontainers/runtime-spec/specs-go" + "golang.org/x/term" ) var ( @@ -239,6 +241,11 @@ func checkOpts(lo launcher.Options) error { func (l *Launcher) createSpec() (*specs.Spec, error) { spec := minimalSpec() + // Override the default Process.Terminal to false if our stdin is not a terminal. + if !term.IsTerminal(syscall.Stdin) { + spec.Process.Terminal = false + } + spec.Process.User = l.getProcessUser() // If we are *not* requesting fakeroot, then we need to map the container From 930836bbecd21066a6da1c20db748cbdfea8b959 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Fri, 2 Dec 2022 12:20:50 +0000 Subject: [PATCH 040/114] chore: move BindPath code out of runtime package Signed-off-by: Edita Kizinevic --- internal/pkg/checkpoint/dmtcp/checkpoint.go | 8 ++++---- internal/pkg/runtime/launcher/native/launcher_linux.go | 7 ++++--- pkg/runtime/engine/apptainer/config/config.go | 7 ++++--- .../engine/apptainer/config => util/bind}/bind.go | 4 ++-- .../engine/apptainer/config => util/bind}/bind_test.go | 4 ++-- .../engine/apptainer/config => util/bind}/mount.go | 2 +- .../engine/apptainer/config => util/bind}/mount_test.go | 4 ++-- 7 files changed, 19 insertions(+), 17 deletions(-) rename pkg/{runtime/engine/apptainer/config => util/bind}/bind.go (98%) rename pkg/{runtime/engine/apptainer/config => util/bind}/bind_test.go (98%) rename pkg/{runtime/engine/apptainer/config => util/bind}/mount.go (99%) rename pkg/{runtime/engine/apptainer/config => util/bind}/mount_test.go (98%) diff --git a/internal/pkg/checkpoint/dmtcp/checkpoint.go b/internal/pkg/checkpoint/dmtcp/checkpoint.go index 2f46df3c31..4556acb142 100644 --- a/internal/pkg/checkpoint/dmtcp/checkpoint.go +++ b/internal/pkg/checkpoint/dmtcp/checkpoint.go @@ -14,7 +14,7 @@ import ( "os" "path/filepath" - apptainerConfig "github.com/apptainer/apptainer/pkg/runtime/engine/apptainer/config" + "github.com/apptainer/apptainer/pkg/util/bind" ) type Entry struct { @@ -38,11 +38,11 @@ func (e *Entry) CoordinatorPort() (string, error) { return s.Text(), nil } -func (e *Entry) BindPath() apptainerConfig.BindPath { - return apptainerConfig.BindPath{ +func (e *Entry) BindPath() bind.BindPath { + return bind.BindPath{ Source: e.path, Destination: containerStatepath, - Options: map[string]*apptainerConfig.BindOption{ + Options: map[string]*bind.BindOption{ "rw": {}, }, } diff --git a/internal/pkg/runtime/launcher/native/launcher_linux.go b/internal/pkg/runtime/launcher/native/launcher_linux.go index 4a3ef4541b..9784996fe1 100644 --- a/internal/pkg/runtime/launcher/native/launcher_linux.go +++ b/internal/pkg/runtime/launcher/native/launcher_linux.go @@ -51,6 +51,7 @@ import ( "github.com/apptainer/apptainer/pkg/runtime/engine/config" "github.com/apptainer/apptainer/pkg/sylog" "github.com/apptainer/apptainer/pkg/util/apptainerconf" + "github.com/apptainer/apptainer/pkg/util/bind" "github.com/apptainer/apptainer/pkg/util/capabilities" "github.com/apptainer/apptainer/pkg/util/cryptkey" "github.com/apptainer/apptainer/pkg/util/fs/proc" @@ -654,14 +655,14 @@ func (l *Launcher) useSuid(insideUserNs bool) (useSuid bool) { // setBinds sets engine configuration for requested bind mounts. func (l *Launcher) setBinds(fakerootPath string) error { // First get binds from -B/--bind and env var - binds, err := apptainerConfig.ParseBindPath(l.cfg.BindPaths) + binds, err := bind.ParseBindPath(l.cfg.BindPaths) if err != nil { return fmt.Errorf("while parsing bind path: %w", err) } // Now add binds from one or more --mount and env var. // Note that these do not get exported for nested containers for _, m := range l.cfg.Mounts { - bps, err := apptainerConfig.ParseMountString(m) + bps, err := bind.ParseMountString(m) if err != nil { return fmt.Errorf("while parsing mount %q: %w", m, err) } @@ -675,7 +676,7 @@ func (l *Launcher) setBinds(fakerootPath string) error { if err != nil { return fmt.Errorf("while getting fakeroot bindpoints: %w", err) } - fakebinds, err := apptainerConfig.ParseBindPath(fakebindPaths) + fakebinds, err := bind.ParseBindPath(fakebindPaths) if err != nil { return fmt.Errorf("while parsing fakeroot bind paths: %w", err) } diff --git a/pkg/runtime/engine/apptainer/config/config.go b/pkg/runtime/engine/apptainer/config/config.go index 63f5938148..e91c943b41 100644 --- a/pkg/runtime/engine/apptainer/config/config.go +++ b/pkg/runtime/engine/apptainer/config/config.go @@ -17,6 +17,7 @@ import ( "github.com/apptainer/apptainer/internal/pkg/runtime/engine/config/oci" "github.com/apptainer/apptainer/pkg/image" "github.com/apptainer/apptainer/pkg/util/apptainerconf" + "github.com/apptainer/apptainer/pkg/util/bind" ) // Name is the name of the runtime. @@ -89,7 +90,7 @@ type JSONConfig struct { LibrariesPath []string `json:"librariesPath,omitempty"` FuseMount []FuseMount `json:"fuseMount,omitempty"` ImageList []image.Image `json:"imageList,omitempty"` - BindPath []BindPath `json:"bindpath,omitempty"` + BindPath []bind.BindPath `json:"bindpath,omitempty"` ApptainerEnv map[string]string `json:"apptainerEnv,omitempty"` UnixSocketPair [2]int `json:"unixSocketPair,omitempty"` OpenFd []int `json:"openFd,omitempty"` @@ -304,12 +305,12 @@ func (e *EngineConfig) GetCustomHome() bool { } // SetBindPath sets the paths to bind into container. -func (e *EngineConfig) SetBindPath(bindpath []BindPath) { +func (e *EngineConfig) SetBindPath(bindpath []bind.BindPath) { e.JSON.BindPath = bindpath } // GetBindPath retrieves the bind paths. -func (e *EngineConfig) GetBindPath() []BindPath { +func (e *EngineConfig) GetBindPath() []bind.BindPath { return e.JSON.BindPath } diff --git a/pkg/runtime/engine/apptainer/config/bind.go b/pkg/util/bind/bind.go similarity index 98% rename from pkg/runtime/engine/apptainer/config/bind.go rename to pkg/util/bind/bind.go index 24cb738669..86f1f37d47 100644 --- a/pkg/runtime/engine/apptainer/config/bind.go +++ b/pkg/util/bind/bind.go @@ -1,9 +1,9 @@ -// Copyright (c) 2019-2021, Sylabs Inc. All rights reserved. +// Copyright (c) 2019-2022, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. -package apptainer +package bind import ( "fmt" diff --git a/pkg/runtime/engine/apptainer/config/bind_test.go b/pkg/util/bind/bind_test.go similarity index 98% rename from pkg/runtime/engine/apptainer/config/bind_test.go rename to pkg/util/bind/bind_test.go index 6431886b41..a1cc61f20a 100644 --- a/pkg/runtime/engine/apptainer/config/bind_test.go +++ b/pkg/util/bind/bind_test.go @@ -1,9 +1,9 @@ -// Copyright (c) 2021, Sylabs Inc. All rights reserved. +// Copyright (c) 2021-2022, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. -package apptainer +package bind import ( "reflect" diff --git a/pkg/runtime/engine/apptainer/config/mount.go b/pkg/util/bind/mount.go similarity index 99% rename from pkg/runtime/engine/apptainer/config/mount.go rename to pkg/util/bind/mount.go index d4e2b35f78..959e40706f 100644 --- a/pkg/runtime/engine/apptainer/config/mount.go +++ b/pkg/util/bind/mount.go @@ -3,7 +3,7 @@ // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. -package apptainer +package bind import ( "encoding/csv" diff --git a/pkg/runtime/engine/apptainer/config/mount_test.go b/pkg/util/bind/mount_test.go similarity index 98% rename from pkg/runtime/engine/apptainer/config/mount_test.go rename to pkg/util/bind/mount_test.go index 46820cb732..a0ffce702a 100644 --- a/pkg/runtime/engine/apptainer/config/mount_test.go +++ b/pkg/util/bind/mount_test.go @@ -1,9 +1,9 @@ -// Copyright (c) 2021, Sylabs Inc. All rights reserved. +// Copyright (c) 2022, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. -package apptainer +package bind import ( "reflect" From f2a4cf2def005db1b9b2014cfb51173a041e9390 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Fri, 2 Dec 2022 13:58:56 +0000 Subject: [PATCH 041/114] feat: oci: enable bind mounts via --bind, --mount Implement support for bind mounts (rw & ro) specified using -B/--bind and --mount on the singularity command line. Fixes sylabs/singularity#1027 Signed-off-by: Edita Kizinevic --- .../runtime/launcher/oci/launcher_linux.go | 7 +- .../pkg/runtime/launcher/oci/mounts_linux.go | 69 ++++- .../runtime/launcher/oci/mounts_linux_test.go | 268 ++++++++++++++++++ 3 files changed, 334 insertions(+), 10 deletions(-) create mode 100644 internal/pkg/runtime/launcher/oci/mounts_linux_test.go diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index 3454a346da..67a00bd536 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -100,15 +100,10 @@ func checkOpts(lo launcher.Options) error { badOpt = append(badOpt, "NoHome") } - if len(lo.BindPaths) > 0 { - badOpt = append(badOpt, "BindPaths") - } if len(lo.FuseMount) > 0 { badOpt = append(badOpt, "FuseMount") } - if len(lo.Mounts) > 0 { - badOpt = append(badOpt, "Mounts") - } + if len(lo.NoMount) > 0 { badOpt = append(badOpt, "NoMount") } diff --git a/internal/pkg/runtime/launcher/oci/mounts_linux.go b/internal/pkg/runtime/launcher/oci/mounts_linux.go index 434672bd26..f2278247c7 100644 --- a/internal/pkg/runtime/launcher/oci/mounts_linux.go +++ b/internal/pkg/runtime/launcher/oci/mounts_linux.go @@ -15,8 +15,11 @@ package oci import ( "fmt" "os" + "path/filepath" "github.com/apptainer/apptainer/internal/pkg/util/user" + "github.com/apptainer/apptainer/pkg/sylog" + "github.com/apptainer/apptainer/pkg/util/bind" "github.com/opencontainers/runtime-spec/specs-go" ) @@ -25,15 +28,16 @@ func (l *Launcher) getMounts() ([]specs.Mount, error) { mounts := &[]specs.Mount{} l.addProcMount(mounts) l.addSysMount(mounts) - err := l.addDevMounts(mounts) - if err != nil { + if err := l.addDevMounts(mounts); err != nil { return nil, fmt.Errorf("while configuring devpts mount: %w", err) } l.addTmpMounts(mounts) - err = l.addHomeMount(mounts) - if err != nil { + if err := l.addHomeMount(mounts); err != nil { return nil, fmt.Errorf("while configuring home mount: %w", err) } + if err := l.addBindMounts(mounts); err != nil { + return nil, fmt.Errorf("while configuring bind mount(s): %w", err) + } return *mounts, nil } @@ -189,3 +193,60 @@ func (l *Launcher) addHomeMount(mounts *[]specs.Mount) error { }) return nil } + +func (l *Launcher) addBindMounts(mounts *[]specs.Mount) error { + // First get binds from -B/--bind and env var + binds, err := bind.ParseBindPath(l.cfg.BindPaths) + if err != nil { + return fmt.Errorf("while parsing bind path: %w", err) + } + // Now add binds from one or more --mount and env var. + for _, m := range l.cfg.Mounts { + bps, err := bind.ParseMountString(m) + if err != nil { + return fmt.Errorf("while parsing mount %q: %w", m, err) + } + binds = append(binds, bps...) + } + + for _, b := range binds { + if !l.apptainerConf.UserBindControl { + sylog.Warningf("Ignoring bind mount request: user bind control disabled by system administrator") + return nil + } + if err := addBindMount(mounts, b); err != nil { + return fmt.Errorf("while adding mount %q: %w", b.Source, err) + } + } + return nil +} + +func addBindMount(mounts *[]specs.Mount, b bind.BindPath) error { + if b.ID() != "" || b.ImageSrc() != "" { + return fmt.Errorf("image binds are not yet supported by the OCI runtime") + } + + opts := []string{"rbind", "nosuid", "nodev"} + if b.Readonly() { + opts = append(opts, "ro") + } + + absSource, err := filepath.Abs(b.Source) + if err != nil { + return fmt.Errorf("cannot determine absolute path of %s: %w", b.Source, err) + } + if _, err := os.Stat(absSource); err != nil { + return fmt.Errorf("cannot stat bind source %s: %w", b.Source, err) + } + + sylog.Debugf("Adding bind of %s to %s, with options %v", absSource, b.Destination, opts) + + *mounts = append(*mounts, + specs.Mount{ + Source: absSource, + Destination: b.Destination, + Type: "none", + Options: opts, + }) + return nil +} diff --git a/internal/pkg/runtime/launcher/oci/mounts_linux_test.go b/internal/pkg/runtime/launcher/oci/mounts_linux_test.go new file mode 100644 index 0000000000..2ee3b58f7d --- /dev/null +++ b/internal/pkg/runtime/launcher/oci/mounts_linux_test.go @@ -0,0 +1,268 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2022, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +// Package oci implements a Launcher that will configure and launch a container +// with an OCI runtime. It also provides implementations of OCI state +// transitions that can be called directly, Create/Start/Kill etc. +package oci + +import ( + "reflect" + "testing" + + "github.com/apptainer/apptainer/internal/pkg/runtime/launcher" + "github.com/apptainer/apptainer/pkg/util/apptainerconf" + "github.com/apptainer/apptainer/pkg/util/bind" + "github.com/opencontainers/runtime-spec/specs-go" +) + +func Test_addBindMount(t *testing.T) { + tests := []struct { + name string + b bind.BindPath + wantMounts *[]specs.Mount + wantErr bool + }{ + { + name: "Valid", + b: bind.BindPath{ + Source: "/tmp", + Destination: "/tmp", + }, + wantMounts: &[]specs.Mount{ + { + Source: "/tmp", + Destination: "/tmp", + Type: "none", + Options: []string{"rbind", "nosuid", "nodev"}, + }, + }, + }, + { + name: "ValidRO", + b: bind.BindPath{ + Source: "/tmp", + Destination: "/tmp", + Options: map[string]*bind.BindOption{"ro": {}}, + }, + wantMounts: &[]specs.Mount{ + { + Source: "/tmp", + Destination: "/tmp", + Type: "none", + Options: []string{"rbind", "nosuid", "nodev", "ro"}, + }, + }, + }, + { + name: "BadSource", + b: bind.BindPath{ + Source: "doesnotexist!", + Destination: "/mnt", + }, + wantMounts: &[]specs.Mount{}, + wantErr: true, + }, + { + name: "ImageID", + b: bind.BindPath{ + Source: "/myimage.sif", + Destination: "/mnt", + Options: map[string]*bind.BindOption{"id": {Value: "4"}}, + }, + wantMounts: &[]specs.Mount{}, + wantErr: true, + }, + { + name: "ImageSrc", + b: bind.BindPath{ + Source: "/myimage.sif", + Destination: "/mnt", + Options: map[string]*bind.BindOption{"img-src": {Value: "/test"}}, + }, + wantMounts: &[]specs.Mount{}, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mounts := &[]specs.Mount{} + err := addBindMount(mounts, tt.b) + if (err != nil) != tt.wantErr { + t.Errorf("addBindMount() error = %v, wantErr %v", err, tt.wantErr) + } + if !reflect.DeepEqual(mounts, tt.wantMounts) { + t.Errorf("addBindMount() want %v, got %v", tt.wantMounts, mounts) + } + }) + } +} + +func TestLauncher_addBindMounts(t *testing.T) { + tests := []struct { + name string + cfg launcher.Options + userbind bool + wantMounts *[]specs.Mount + wantErr bool + }{ + { + name: "Disabled", + cfg: launcher.Options{ + BindPaths: []string{"/tmp"}, + }, + wantMounts: &[]specs.Mount{}, + wantErr: false, + }, + { + name: "ValidBindSrc", + cfg: launcher.Options{ + BindPaths: []string{"/tmp"}, + }, + userbind: true, + wantMounts: &[]specs.Mount{ + { + Source: "/tmp", + Destination: "/tmp", + Type: "none", + Options: []string{"rbind", "nosuid", "nodev"}, + }, + }, + wantErr: false, + }, + { + name: "ValidBindSrcDst", + cfg: launcher.Options{ + BindPaths: []string{"/tmp:/mnt"}, + }, + userbind: true, + wantMounts: &[]specs.Mount{ + { + Source: "/tmp", + Destination: "/mnt", + Type: "none", + Options: []string{"rbind", "nosuid", "nodev"}, + }, + }, + wantErr: false, + }, + { + name: "ValidBindRO", + cfg: launcher.Options{ + BindPaths: []string{"/tmp:/mnt:ro"}, + }, + userbind: true, + wantMounts: &[]specs.Mount{ + { + Source: "/tmp", + Destination: "/mnt", + Type: "none", + Options: []string{"rbind", "nosuid", "nodev", "ro"}, + }, + }, + wantErr: false, + }, + { + name: "InvalidBindSrc", + cfg: launcher.Options{ + BindPaths: []string{"!doesnotexist"}, + }, + userbind: true, + wantMounts: &[]specs.Mount{}, + wantErr: true, + }, + { + name: "UnsupportedBindID", + cfg: launcher.Options{ + BindPaths: []string{"my.sif:/mnt:id=2"}, + }, + userbind: true, + wantMounts: &[]specs.Mount{}, + wantErr: true, + }, + { + name: "UnsupportedBindImgSrc", + cfg: launcher.Options{ + BindPaths: []string{"my.sif:/mnt:img-src=/test"}, + }, + userbind: true, + wantMounts: &[]specs.Mount{}, + wantErr: true, + }, + { + name: "ValidMount", + cfg: launcher.Options{ + Mounts: []string{"type=bind,source=/tmp,destination=/mnt"}, + }, + userbind: true, + wantMounts: &[]specs.Mount{ + { + Source: "/tmp", + Destination: "/mnt", + Type: "none", + Options: []string{"rbind", "nosuid", "nodev"}, + }, + }, + wantErr: false, + }, + { + name: "ValidMountRO", + cfg: launcher.Options{ + Mounts: []string{"type=bind,source=/tmp,destination=/mnt,ro"}, + }, + userbind: true, + wantMounts: &[]specs.Mount{ + { + Source: "/tmp", + Destination: "/mnt", + Type: "none", + Options: []string{"rbind", "nosuid", "nodev", "ro"}, + }, + }, + wantErr: false, + }, + { + name: "UnsupportedMountID", + cfg: launcher.Options{ + Mounts: []string{"type=bind,source=my.sif,destination=/mnt,id=2"}, + }, + userbind: true, + wantMounts: &[]specs.Mount{}, + wantErr: true, + }, + { + name: "UnsupportedMountImgSrc", + cfg: launcher.Options{ + Mounts: []string{"type=bind,source=my.sif,destination=/mnt,image-src=/test"}, + }, + userbind: true, + wantMounts: &[]specs.Mount{}, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + l := &Launcher{ + cfg: tt.cfg, + apptainerConf: &apptainerconf.File{}, + } + if tt.userbind { + l.apptainerConf.UserBindControl = true + } + mounts := &[]specs.Mount{} + err := l.addBindMounts(mounts) + if (err != nil) != tt.wantErr { + t.Errorf("addBindMount() error = %v, wantErr %v", err, tt.wantErr) + } + if !reflect.DeepEqual(mounts, tt.wantMounts) { + t.Errorf("addBindMount() want %v, got %v", tt.wantMounts, mounts) + } + }) + } +} From 4052b175678e65cfcd031fadc26b8d17d0baab6e Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Fri, 2 Dec 2022 14:12:58 +0000 Subject: [PATCH 042/114] e2e: port relevant --bind/mount action tests for --oci Signed-off-by: Edita Kizinevic --- e2e/actions/actions.go | 1 + e2e/actions/oci.go | 208 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 209 insertions(+) diff --git a/e2e/actions/actions.go b/e2e/actions/actions.go index d61d00ac44..8a29e1fee8 100644 --- a/e2e/actions/actions.go +++ b/e2e/actions/actions.go @@ -2916,5 +2916,6 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { "ociExec": c.actionOciExec, // apptainer exec --oci "ociShell": c.actionOciShell, // apptainer shell --oci "ociNetwork": c.actionOciNetwork, // apptainer exec --oci --net + "ociBinds": c.actionOciBinds, // apptainer exec --oci --bind / --mount } } diff --git a/e2e/actions/oci.go b/e2e/actions/oci.go index a65b7b7c65..98e9e6c219 100644 --- a/e2e/actions/oci.go +++ b/e2e/actions/oci.go @@ -16,6 +16,7 @@ import ( "testing" "github.com/apptainer/apptainer/e2e/internal/e2e" + "github.com/apptainer/apptainer/internal/pkg/util/fs" ) const ( @@ -283,3 +284,210 @@ func (c actionTests) actionOciNetwork(t *testing.T) { ) } } + +//nolint:maintidx +func (c actionTests) actionOciBinds(t *testing.T) { + e2e.EnsureOCIImage(t, c.env) + imageRef := "oci-archive:" + c.env.OCIImagePath + + workspace, cleanup := e2e.MakeTempDir(t, c.env.TestDir, "bind-workspace-", "") + defer e2e.Privileged(cleanup) + + contCanaryDir := "/canary" + hostCanaryDir := filepath.Join(workspace, "canary") + + contCanaryFile := "/canary/file" + hostCanaryFile := filepath.Join(hostCanaryDir, "file") + + canaryFileBind := hostCanaryFile + ":" + contCanaryFile + canaryFileMount := "type=bind,source=" + hostCanaryFile + ",destination=" + contCanaryFile + canaryDirBind := hostCanaryDir + ":" + contCanaryDir + canaryDirMount := "type=bind,source=" + hostCanaryDir + ",destination=" + contCanaryDir + + createWorkspaceDirs := func(t *testing.T) { + e2e.Privileged(func(t *testing.T) { + if err := os.RemoveAll(hostCanaryDir); err != nil && !os.IsNotExist(err) { + t.Fatalf("failed to delete canary_dir: %s", err) + } + })(t) + + if err := fs.Mkdir(hostCanaryDir, 0o777); err != nil { + t.Fatalf("failed to create canary_dir: %s", err) + } + if err := fs.Touch(hostCanaryFile); err != nil { + t.Fatalf("failed to create canary_file: %s", err) + } + if err := os.Chmod(hostCanaryFile, 0o777); err != nil { + t.Fatalf("failed to apply permissions on canary_file: %s", err) + } + } + + checkHostFn := func(path string, fn func(string) bool) func(*testing.T) { + return func(t *testing.T) { + if t.Failed() { + return + } + if !fn(path) { + t.Errorf("%s not found on host", path) + } + if err := os.RemoveAll(path); err != nil { + t.Errorf("failed to delete %s: %s", path, err) + } + } + } + checkHostFile := func(path string) func(*testing.T) { + return checkHostFn(path, fs.IsFile) + } + checkHostDir := func(path string) func(*testing.T) { + return checkHostFn(path, fs.IsDir) + } + + tests := []struct { + name string + args []string + postRun func(*testing.T) + exit int + }{ + { + name: "NonExistentSource", + args: []string{ + "--bind", "/non/existent/source/path", + imageRef, + "true", + }, + exit: 255, + }, + { + name: "RelativeBindDestination", + args: []string{ + "--bind", hostCanaryFile + ":relative", + imageRef, + "true", + }, + exit: 255, + }, + { + name: "SimpleFile", + args: []string{ + "--bind", canaryFileBind, + imageRef, + "test", "-f", contCanaryFile, + }, + exit: 0, + }, + { + name: "SimpleDir", + args: []string{ + "--bind", canaryDirBind, + imageRef, + "test", "-f", contCanaryFile, + }, + exit: 0, + }, + { + name: "HomeOverride", + args: []string{ + "--bind", hostCanaryDir + ":/home", + imageRef, + "test", "-f", "/home/file", + }, + exit: 0, + }, + { + name: "TmpOverride", + args: []string{ + "--bind", hostCanaryDir + ":/tmp", + imageRef, + "test", "-f", "/tmp/file", + }, + exit: 0, + }, + { + name: "VarTmpOverride", + args: []string{ + "--bind", hostCanaryDir + ":/var/tmp", + imageRef, + "test", "-f", "/var/tmp/file", + }, + exit: 0, + }, + { + name: "NestedBindFile", + args: []string{ + "--bind", canaryDirBind, + "--bind", hostCanaryFile + ":" + filepath.Join(contCanaryDir, "file2"), + imageRef, + "test", "-f", "/canary/file2", + }, + postRun: checkHostFile(filepath.Join(hostCanaryDir, "file2")), + exit: 0, + }, + { + name: "NestedBindDir", + args: []string{ + "--bind", canaryDirBind, + "--bind", hostCanaryDir + ":" + filepath.Join(contCanaryDir, "dir2"), + imageRef, + "test", "-d", "/canary/dir2", + }, + postRun: checkHostDir(filepath.Join(hostCanaryDir, "dir2")), + exit: 0, + }, + { + name: "MultipleNestedBindDir", + args: []string{ + "--bind", canaryDirBind, + "--bind", hostCanaryDir + ":" + filepath.Join(contCanaryDir, "dir2"), + "--bind", hostCanaryFile + ":" + filepath.Join(filepath.Join(contCanaryDir, "dir2"), "nested"), + imageRef, + "test", "-f", "/canary/dir2/nested", + }, + postRun: checkHostFile(filepath.Join(hostCanaryDir, "nested")), + exit: 0, + }, + // For the --mount variants we are really just verifying the CLI + // acceptance of one or more --mount flags. Translation from --mount + // strings to BindPath structs is checked in unit tests. The + // functionality of bind mounts of various kinds is already checked + // above, with --bind flags. No need to duplicate all of these. + { + name: "MountSingle", + args: []string{ + "--mount", canaryFileMount, + imageRef, + "test", "-f", contCanaryFile, + }, + exit: 0, + }, + { + name: "MountNested", + args: []string{ + "--mount", canaryDirMount, + "--mount", "source=" + hostCanaryFile + ",destination=" + filepath.Join(contCanaryDir, "file3"), + imageRef, + "test", "-f", "/canary/file3", + }, + postRun: checkHostFile(filepath.Join(hostCanaryDir, "file3")), + exit: 0, + }, + } + + for _, profile := range e2e.OCIProfiles { + profile := profile + createWorkspaceDirs(t) + + t.Run(profile.String(), func(t *testing.T) { + for _, tt := range tests { + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(profile), + e2e.WithCommand("exec"), + e2e.WithArgs(tt.args...), + e2e.PostRun(tt.postRun), + e2e.ExpectExit(tt.exit), + ) + } + }) + } +} From a63047a9162d1320ec32cb7bd4544be6c21c0d8f Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Fri, 2 Dec 2022 14:23:03 +0000 Subject: [PATCH 043/114] fix: oci: disallow relative bind destinations Signed-off-by: Edita Kizinevic --- .../pkg/runtime/launcher/oci/mounts_linux.go | 4 ++++ .../runtime/launcher/oci/mounts_linux_test.go | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/internal/pkg/runtime/launcher/oci/mounts_linux.go b/internal/pkg/runtime/launcher/oci/mounts_linux.go index f2278247c7..a15df8be70 100644 --- a/internal/pkg/runtime/launcher/oci/mounts_linux.go +++ b/internal/pkg/runtime/launcher/oci/mounts_linux.go @@ -239,6 +239,10 @@ func addBindMount(mounts *[]specs.Mount, b bind.BindPath) error { return fmt.Errorf("cannot stat bind source %s: %w", b.Source, err) } + if !filepath.IsAbs(b.Destination) { + return fmt.Errorf("bind destination %s must be an absolute path", b.Destination) + } + sylog.Debugf("Adding bind of %s to %s, with options %v", absSource, b.Destination, opts) *mounts = append(*mounts, diff --git a/internal/pkg/runtime/launcher/oci/mounts_linux_test.go b/internal/pkg/runtime/launcher/oci/mounts_linux_test.go index 2ee3b58f7d..31c57fb6eb 100644 --- a/internal/pkg/runtime/launcher/oci/mounts_linux_test.go +++ b/internal/pkg/runtime/launcher/oci/mounts_linux_test.go @@ -69,6 +69,15 @@ func Test_addBindMount(t *testing.T) { wantMounts: &[]specs.Mount{}, wantErr: true, }, + { + name: "RelDest", + b: bind.BindPath{ + Source: "/tmp", + Destination: "relative", + }, + wantMounts: &[]specs.Mount{}, + wantErr: true, + }, { name: "ImageID", b: bind.BindPath{ @@ -177,6 +186,15 @@ func TestLauncher_addBindMounts(t *testing.T) { wantMounts: &[]specs.Mount{}, wantErr: true, }, + { + name: "RelBindDst", + cfg: launcher.Options{ + BindPaths: []string{"/tmp:relative"}, + }, + userbind: true, + wantMounts: &[]specs.Mount{}, + wantErr: true, + }, { name: "UnsupportedBindID", cfg: launcher.Options{ From 3578fe660ddf79cb6f72744fbf6e798acc0406d9 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Fri, 2 Dec 2022 14:34:45 +0000 Subject: [PATCH 044/114] chore: fix stutter bind.Bindxxxx lint Signed-off-by: Edita Kizinevic --- internal/pkg/checkpoint/dmtcp/checkpoint.go | 6 +- .../pkg/runtime/launcher/oci/mounts_linux.go | 2 +- .../runtime/launcher/oci/mounts_linux_test.go | 20 +++--- pkg/runtime/engine/apptainer/config/config.go | 6 +- pkg/util/bind/bind.go | 34 ++++----- pkg/util/bind/bind_test.go | 44 ++++++------ pkg/util/bind/mount.go | 30 ++++---- pkg/util/bind/mount_test.go | 70 +++++++++---------- 8 files changed, 106 insertions(+), 106 deletions(-) diff --git a/internal/pkg/checkpoint/dmtcp/checkpoint.go b/internal/pkg/checkpoint/dmtcp/checkpoint.go index 4556acb142..bbe186fadf 100644 --- a/internal/pkg/checkpoint/dmtcp/checkpoint.go +++ b/internal/pkg/checkpoint/dmtcp/checkpoint.go @@ -38,11 +38,11 @@ func (e *Entry) CoordinatorPort() (string, error) { return s.Text(), nil } -func (e *Entry) BindPath() bind.BindPath { - return bind.BindPath{ +func (e *Entry) BindPath() bind.Path { + return bind.Path{ Source: e.path, Destination: containerStatepath, - Options: map[string]*bind.BindOption{ + Options: map[string]*bind.Option{ "rw": {}, }, } diff --git a/internal/pkg/runtime/launcher/oci/mounts_linux.go b/internal/pkg/runtime/launcher/oci/mounts_linux.go index a15df8be70..8f77d71f81 100644 --- a/internal/pkg/runtime/launcher/oci/mounts_linux.go +++ b/internal/pkg/runtime/launcher/oci/mounts_linux.go @@ -221,7 +221,7 @@ func (l *Launcher) addBindMounts(mounts *[]specs.Mount) error { return nil } -func addBindMount(mounts *[]specs.Mount, b bind.BindPath) error { +func addBindMount(mounts *[]specs.Mount, b bind.Path) error { if b.ID() != "" || b.ImageSrc() != "" { return fmt.Errorf("image binds are not yet supported by the OCI runtime") } diff --git a/internal/pkg/runtime/launcher/oci/mounts_linux_test.go b/internal/pkg/runtime/launcher/oci/mounts_linux_test.go index 31c57fb6eb..b4f4b3f54d 100644 --- a/internal/pkg/runtime/launcher/oci/mounts_linux_test.go +++ b/internal/pkg/runtime/launcher/oci/mounts_linux_test.go @@ -25,13 +25,13 @@ import ( func Test_addBindMount(t *testing.T) { tests := []struct { name string - b bind.BindPath + b bind.Path wantMounts *[]specs.Mount wantErr bool }{ { name: "Valid", - b: bind.BindPath{ + b: bind.Path{ Source: "/tmp", Destination: "/tmp", }, @@ -46,10 +46,10 @@ func Test_addBindMount(t *testing.T) { }, { name: "ValidRO", - b: bind.BindPath{ + b: bind.Path{ Source: "/tmp", Destination: "/tmp", - Options: map[string]*bind.BindOption{"ro": {}}, + Options: map[string]*bind.Option{"ro": {}}, }, wantMounts: &[]specs.Mount{ { @@ -62,7 +62,7 @@ func Test_addBindMount(t *testing.T) { }, { name: "BadSource", - b: bind.BindPath{ + b: bind.Path{ Source: "doesnotexist!", Destination: "/mnt", }, @@ -71,7 +71,7 @@ func Test_addBindMount(t *testing.T) { }, { name: "RelDest", - b: bind.BindPath{ + b: bind.Path{ Source: "/tmp", Destination: "relative", }, @@ -80,20 +80,20 @@ func Test_addBindMount(t *testing.T) { }, { name: "ImageID", - b: bind.BindPath{ + b: bind.Path{ Source: "/myimage.sif", Destination: "/mnt", - Options: map[string]*bind.BindOption{"id": {Value: "4"}}, + Options: map[string]*bind.Option{"id": {Value: "4"}}, }, wantMounts: &[]specs.Mount{}, wantErr: true, }, { name: "ImageSrc", - b: bind.BindPath{ + b: bind.Path{ Source: "/myimage.sif", Destination: "/mnt", - Options: map[string]*bind.BindOption{"img-src": {Value: "/test"}}, + Options: map[string]*bind.Option{"img-src": {Value: "/test"}}, }, wantMounts: &[]specs.Mount{}, wantErr: true, diff --git a/pkg/runtime/engine/apptainer/config/config.go b/pkg/runtime/engine/apptainer/config/config.go index e91c943b41..924db912f7 100644 --- a/pkg/runtime/engine/apptainer/config/config.go +++ b/pkg/runtime/engine/apptainer/config/config.go @@ -90,7 +90,7 @@ type JSONConfig struct { LibrariesPath []string `json:"librariesPath,omitempty"` FuseMount []FuseMount `json:"fuseMount,omitempty"` ImageList []image.Image `json:"imageList,omitempty"` - BindPath []bind.BindPath `json:"bindpath,omitempty"` + BindPath []bind.Path `json:"bindpath,omitempty"` ApptainerEnv map[string]string `json:"apptainerEnv,omitempty"` UnixSocketPair [2]int `json:"unixSocketPair,omitempty"` OpenFd []int `json:"openFd,omitempty"` @@ -305,12 +305,12 @@ func (e *EngineConfig) GetCustomHome() bool { } // SetBindPath sets the paths to bind into container. -func (e *EngineConfig) SetBindPath(bindpath []bind.BindPath) { +func (e *EngineConfig) SetBindPath(bindpath []bind.Path) { e.JSON.BindPath = bindpath } // GetBindPath retrieves the bind paths. -func (e *EngineConfig) GetBindPath() []bind.BindPath { +func (e *EngineConfig) GetBindPath() []bind.Path { return e.JSON.BindPath } diff --git a/pkg/util/bind/bind.go b/pkg/util/bind/bind.go index 86f1f37d47..d205057a19 100644 --- a/pkg/util/bind/bind.go +++ b/pkg/util/bind/bind.go @@ -11,9 +11,9 @@ import ( "strings" ) -// BindOption represents a bind option with its associated +// Option represents a bind option with its associated // value if any. -type BindOption struct { +type Option struct { Value string `json:"value,omitempty"` } @@ -31,17 +31,17 @@ var bindOptions = map[string]bool{ "id": valueOption, } -// BindPath stores a parsed bind path specification. Source and Destination +// Path stores a parsed bind path specification. Source and Destination // paths are required. -type BindPath struct { - Source string `json:"source"` - Destination string `json:"destination"` - Options map[string]*BindOption `json:"options"` +type Path struct { + Source string `json:"source"` + Destination string `json:"destination"` + Options map[string]*Option `json:"options"` } // ImageSrc returns the value of the option image-src for a BindPath, or an // empty string if the option wasn't set. -func (b *BindPath) ImageSrc() string { +func (b *Path) ImageSrc() string { if b.Options != nil && b.Options["image-src"] != nil { src := b.Options["image-src"].Value if src == "" { @@ -54,7 +54,7 @@ func (b *BindPath) ImageSrc() string { // ID returns the value of the option id for a BindPath, or an empty string if // the option wasn't set. -func (b *BindPath) ID() string { +func (b *Path) ID() string { if b.Options != nil && b.Options["id"] != nil { return b.Options["id"].Value } @@ -62,7 +62,7 @@ func (b *BindPath) ID() string { } // Readonly returns true if the ro option was set for a BindPath. -func (b *BindPath) Readonly() bool { +func (b *Path) Readonly() bool { return b.Options != nil && b.Options["ro"] != nil } @@ -70,8 +70,8 @@ func (b *BindPath) Readonly() bool { // more (comma separated) bind paths in src[:dst[:options]] format, and // returns all encountered bind paths as a slice. Options may be simple // flags, e.g. 'rw', or take a value, e.g. 'id=2'. -func ParseBindPath(paths []string) ([]BindPath, error) { - var binds []BindPath +func ParseBindPath(paths []string) ([]Path, error) { + var binds []Path // there is a better regular expression to handle // that directly without all the logic below ... @@ -216,8 +216,8 @@ func splitBy(str string, sep byte) []string { // newBindPath returns BindPath record based on the provided bind // string argument and ensures that the options are valid. -func newBindPath(bind string) (BindPath, error) { - var bp BindPath +func newBindPath(bind string) (Path, error) { + var bp Path splitted := splitBy(bind, ':') @@ -233,17 +233,17 @@ func newBindPath(bind string) (BindPath, error) { } if len(splitted) > 2 { - bp.Options = make(map[string]*BindOption) + bp.Options = make(map[string]*Option) for _, value := range strings.Split(splitted[2], ",") { valid := false for optName, isFlag := range bindOptions { if isFlag && optName == value { - bp.Options[optName] = &BindOption{} + bp.Options[optName] = &Option{} valid = true break } else if strings.HasPrefix(value, optName+"=") { - bp.Options[optName] = &BindOption{Value: value[len(optName+"="):]} + bp.Options[optName] = &Option{Value: value[len(optName+"="):]} valid = true break } diff --git a/pkg/util/bind/bind_test.go b/pkg/util/bind/bind_test.go index a1cc61f20a..07e576b404 100644 --- a/pkg/util/bind/bind_test.go +++ b/pkg/util/bind/bind_test.go @@ -14,13 +14,13 @@ func TestParseBindPath(t *testing.T) { tests := []struct { name string bindpaths []string - want []BindPath + want []Path wantErr bool }{ { name: "srcOnly", bindpaths: []string{"/opt"}, - want: []BindPath{ + want: []Path{ { Source: "/opt", Destination: "/opt", @@ -30,7 +30,7 @@ func TestParseBindPath(t *testing.T) { { name: "srcOnlyMultiple", bindpaths: []string{"/opt,/tmp"}, - want: []BindPath{ + want: []Path{ { Source: "/opt", Destination: "/opt", @@ -44,7 +44,7 @@ func TestParseBindPath(t *testing.T) { { name: "srcDst", bindpaths: []string{"/opt:/other"}, - want: []BindPath{ + want: []Path{ { Source: "/opt", Destination: "/other", @@ -54,7 +54,7 @@ func TestParseBindPath(t *testing.T) { { name: "srcDstMultiple", bindpaths: []string{"/opt:/other,/tmp:/other2,"}, - want: []BindPath{ + want: []Path{ { Source: "/opt", Destination: "/other", @@ -68,11 +68,11 @@ func TestParseBindPath(t *testing.T) { { name: "srcDstRO", bindpaths: []string{"/opt:/other:ro"}, - want: []BindPath{ + want: []Path{ { Source: "/opt", Destination: "/other", - Options: map[string]*BindOption{ + Options: map[string]*Option{ "ro": {}, }, }, @@ -81,18 +81,18 @@ func TestParseBindPath(t *testing.T) { { name: "srcDstROMultiple", bindpaths: []string{"/opt:/other:ro,/tmp:/other2:ro"}, - want: []BindPath{ + want: []Path{ { Source: "/opt", Destination: "/other", - Options: map[string]*BindOption{ + Options: map[string]*Option{ "ro": {}, }, }, { Source: "/tmp", Destination: "/other2", - Options: map[string]*BindOption{ + Options: map[string]*Option{ "ro": {}, }, }, @@ -103,11 +103,11 @@ func TestParseBindPath(t *testing.T) { // parsing multiple simple options. name: "srcDstRORW", bindpaths: []string{"/opt:/other:ro,rw"}, - want: []BindPath{ + want: []Path{ { Source: "/opt", Destination: "/other", - Options: map[string]*BindOption{ + Options: map[string]*Option{ "ro": {}, "rw": {}, }, @@ -121,11 +121,11 @@ func TestParseBindPath(t *testing.T) { // delimiting an additional option, vs an additional bind. name: "srcDstRORWMultiple", bindpaths: []string{"/opt:/other:ro,rw,/tmp:/other2:ro,rw"}, - want: []BindPath{ + want: []Path{ { Source: "/opt", Destination: "/other", - Options: map[string]*BindOption{ + Options: map[string]*Option{ "ro": {}, "rw": {}, }, @@ -133,7 +133,7 @@ func TestParseBindPath(t *testing.T) { { Source: "/tmp", Destination: "/other2", - Options: map[string]*BindOption{ + Options: map[string]*Option{ "ro": {}, "rw": {}, }, @@ -143,11 +143,11 @@ func TestParseBindPath(t *testing.T) { { name: "srcDstImageSrc", bindpaths: []string{"test.sif:/other:image-src=/opt"}, - want: []BindPath{ + want: []Path{ { Source: "test.sif", Destination: "/other", - Options: map[string]*BindOption{ + Options: map[string]*Option{ "image-src": {"/opt"}, }, }, @@ -157,17 +157,17 @@ func TestParseBindPath(t *testing.T) { // Can't use image-src without a value name: "srcDstImageSrcNoVal", bindpaths: []string{"test.sif:/other:image-src"}, - want: []BindPath{}, + want: []Path{}, wantErr: true, }, { name: "srcDstId", bindpaths: []string{"test.sif:/other:image-src=/opt,id=2"}, - want: []BindPath{ + want: []Path{ { Source: "test.sif", Destination: "/other", - Options: map[string]*BindOption{ + Options: map[string]*Option{ "image-src": {"/opt"}, "id": {"2"}, }, @@ -177,13 +177,13 @@ func TestParseBindPath(t *testing.T) { { name: "invalidOption", bindpaths: []string{"/opt:/other:invalid"}, - want: []BindPath{}, + want: []Path{}, wantErr: true, }, { name: "invalidSpec", bindpaths: []string{"/opt:/other:rw:invalid"}, - want: []BindPath{}, + want: []Path{}, wantErr: true, }, } diff --git a/pkg/util/bind/mount.go b/pkg/util/bind/mount.go index 959e40706f..6312bf3e73 100644 --- a/pkg/util/bind/mount.go +++ b/pkg/util/bind/mount.go @@ -29,17 +29,17 @@ import ( // // We only support type=bind at present, so assume this if type is missing and // error for other types. -func ParseMountString(mount string) (bindPaths []BindPath, err error) { +func ParseMountString(mount string) (bindPaths []Path, err error) { r := strings.NewReader(mount) c := csv.NewReader(r) records, err := c.ReadAll() if err != nil { - return []BindPath{}, fmt.Errorf("error parsing mount: %v", err) + return []Path{}, fmt.Errorf("error parsing mount: %v", err) } for _, r := range records { - bp := BindPath{ - Options: map[string]*BindOption{}, + bp := Path{ + Options: map[string]*Option{}, } for _, f := range r { @@ -54,41 +54,41 @@ func ParseMountString(mount string) (bindPaths []BindPath, err error) { // TODO - Eventually support volume and tmpfs? Requires structural changes to engine mount functionality. case "type": if val != "bind" { - return []BindPath{}, fmt.Errorf("unsupported mount type %q, only 'bind' is supported", val) + return []Path{}, fmt.Errorf("unsupported mount type %q, only 'bind' is supported", val) } case "source", "src": if val == "" { - return []BindPath{}, fmt.Errorf("mount source cannot be empty") + return []Path{}, fmt.Errorf("mount source cannot be empty") } bp.Source = val case "destination", "dst", "target": if val == "" { - return []BindPath{}, fmt.Errorf("mount destination cannot be empty") + return []Path{}, fmt.Errorf("mount destination cannot be empty") } bp.Destination = val case "ro", "readonly": - bp.Options["ro"] = &BindOption{} + bp.Options["ro"] = &Option{} // Apptainer only - directory inside an image file source to mount from case "image-src": if val == "" { - return []BindPath{}, fmt.Errorf("img-src cannot be empty") + return []Path{}, fmt.Errorf("img-src cannot be empty") } - bp.Options["image-src"] = &BindOption{Value: val} + bp.Options["image-src"] = &Option{Value: val} // Apptainer only - id of the descriptor in a SIF image source to mount from case "id": if val == "" { - return []BindPath{}, fmt.Errorf("id cannot be empty") + return []Path{}, fmt.Errorf("id cannot be empty") } - bp.Options["id"] = &BindOption{Value: val} + bp.Options["id"] = &Option{Value: val} case "bind-propagation": - return []BindPath{}, fmt.Errorf("bind-propagation not supported for individual mounts, check apptainer.conf for global setting") + return []Path{}, fmt.Errorf("bind-propagation not supported for individual mounts, check apptainer.conf for global setting") default: - return []BindPath{}, fmt.Errorf("invalid key %q in mount specification", key) + return []Path{}, fmt.Errorf("invalid key %q in mount specification", key) } } if bp.Source == "" || bp.Destination == "" { - return []BindPath{}, fmt.Errorf("mounts must specify a source and a destination") + return []Path{}, fmt.Errorf("mounts must specify a source and a destination") } bindPaths = append(bindPaths, bp) } diff --git a/pkg/util/bind/mount_test.go b/pkg/util/bind/mount_test.go index a0ffce702a..169c0850db 100644 --- a/pkg/util/bind/mount_test.go +++ b/pkg/util/bind/mount_test.go @@ -14,53 +14,53 @@ func TestParseMountString(t *testing.T) { tests := []struct { name string mountString string - want []BindPath + want []Path wantErr bool }{ { name: "sourceOnly", mountString: "type=bind,source=/opt", - want: []BindPath{}, + want: []Path{}, wantErr: true, }, { name: "destinationOnly", mountString: "type=bind,destination=/opt", - want: []BindPath{}, + want: []Path{}, wantErr: true, }, { name: "emptySource", mountString: "type=bind,source=,destination=/opt", - want: []BindPath{}, + want: []Path{}, wantErr: true, }, { name: "emptyDestination", mountString: "type=bind,source=/opt,destination=", - want: []BindPath{}, + want: []Path{}, wantErr: true, }, { name: "invalidType", mountString: "type=potato,source=/opt,destination=/opt", - want: []BindPath{}, + want: []Path{}, wantErr: true, }, { name: "invalidField", mountString: "type=bind,source=/opt,destination=/opt,color=turquoise", - want: []BindPath{}, + want: []Path{}, wantErr: true, }, { name: "simple", mountString: "type=bind,source=/opt,destination=/opt", - want: []BindPath{ + want: []Path{ { Source: "/opt", Destination: "/opt", - Options: map[string]*BindOption{}, + Options: map[string]*Option{}, }, }, wantErr: false, @@ -68,11 +68,11 @@ func TestParseMountString(t *testing.T) { { name: "simpleSrc", mountString: "type=bind,src=/opt,destination=/opt", - want: []BindPath{ + want: []Path{ { Source: "/opt", Destination: "/opt", - Options: map[string]*BindOption{}, + Options: map[string]*Option{}, }, }, wantErr: false, @@ -80,11 +80,11 @@ func TestParseMountString(t *testing.T) { { name: "simpleDst", mountString: "type=bind,source=/opt,dst=/opt", - want: []BindPath{ + want: []Path{ { Source: "/opt", Destination: "/opt", - Options: map[string]*BindOption{}, + Options: map[string]*Option{}, }, }, wantErr: false, @@ -92,11 +92,11 @@ func TestParseMountString(t *testing.T) { { name: "simpleTarget", mountString: "type=bind,source=/opt,target=/opt", - want: []BindPath{ + want: []Path{ { Source: "/opt", Destination: "/opt", - Options: map[string]*BindOption{}, + Options: map[string]*Option{}, }, }, wantErr: false, @@ -104,11 +104,11 @@ func TestParseMountString(t *testing.T) { { name: "noType", mountString: "source=/opt,destination=/opt", - want: []BindPath{ + want: []Path{ { Source: "/opt", Destination: "/opt", - Options: map[string]*BindOption{}, + Options: map[string]*Option{}, }, }, wantErr: false, @@ -116,11 +116,11 @@ func TestParseMountString(t *testing.T) { { name: "ro", mountString: "type=bind,source=/opt,destination=/opt,ro", - want: []BindPath{ + want: []Path{ { Source: "/opt", Destination: "/opt", - Options: map[string]*BindOption{ + Options: map[string]*Option{ "ro": {}, }, }, @@ -130,11 +130,11 @@ func TestParseMountString(t *testing.T) { { name: "readonly", mountString: "type=bind,source=/opt,destination=/opt,readonly", - want: []BindPath{ + want: []Path{ { Source: "/opt", Destination: "/opt", - Options: map[string]*BindOption{ + Options: map[string]*Option{ "ro": {}, }, }, @@ -144,11 +144,11 @@ func TestParseMountString(t *testing.T) { { name: "imagesrc", mountString: "type=bind,source=test.sif,destination=/opt,image-src=/opt", - want: []BindPath{ + want: []Path{ { Source: "test.sif", Destination: "/opt", - Options: map[string]*BindOption{ + Options: map[string]*Option{ "image-src": {Value: "/opt"}, }, }, @@ -158,23 +158,23 @@ func TestParseMountString(t *testing.T) { { name: "imagesrcNoValue", mountString: "type=bind,source=test.sif,destination=/opt,image-src", - want: []BindPath{}, + want: []Path{}, wantErr: true, }, { name: "imagesrcEmpty", mountString: "type=bind,source=test.sif,destination=/opt,image-src=", - want: []BindPath{}, + want: []Path{}, wantErr: true, }, { name: "id", mountString: "type=bind,source=test.sif,destination=/opt,image-src=/opt,id=2", - want: []BindPath{ + want: []Path{ { Source: "test.sif", Destination: "/opt", - Options: map[string]*BindOption{ + Options: map[string]*Option{ "image-src": {Value: "/opt"}, "id": {Value: "2"}, }, @@ -185,29 +185,29 @@ func TestParseMountString(t *testing.T) { { name: "idNoValue", mountString: "type=bind,source=test.sif,destination=/opt,image-src=/opt,id", - want: []BindPath{}, + want: []Path{}, wantErr: true, }, { name: "idEmpty", mountString: "type=bind,source=test.sif,destination=/opt,image-src=/opt,id=", - want: []BindPath{}, + want: []Path{}, wantErr: true, }, { name: "bindpropagation", mountString: "type=bind,source=/opt,destination=/opt,bind-propagation=shared", - want: []BindPath{}, + want: []Path{}, wantErr: true, }, { name: "csvEscaped", mountString: `type=bind,"source=/comma,dir","destination=/quote""dir"`, - want: []BindPath{ + want: []Path{ { Source: "/comma,dir", Destination: "/quote\"dir", - Options: map[string]*BindOption{}, + Options: map[string]*Option{}, }, }, wantErr: false, @@ -215,16 +215,16 @@ func TestParseMountString(t *testing.T) { { name: "multiple", mountString: "type=bind,source=/opt,destination=/opt\ntype=bind,source=/srv,destination=/srv", - want: []BindPath{ + want: []Path{ { Source: "/opt", Destination: "/opt", - Options: map[string]*BindOption{}, + Options: map[string]*Option{}, }, { Source: "/srv", Destination: "/srv", - Options: map[string]*BindOption{}, + Options: map[string]*Option{}, }, }, wantErr: false, From 8cf8779360f8638cf42a6bb712ff86953c90bd8c Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Mon, 5 Dec 2022 17:03:05 +0000 Subject: [PATCH 045/114] feat: oci: support --env option in --oci mode * Merge image config ENV and env vars requested by user with the --env CLI option. * Set default SINGULARITY_CONTAINER and SINGULARITY_NAME env variables. * Set default LD_LIBRARY_PATH to be used later for library injection (this is a singularity default). Fixes sylabs/singularity#1029 Signed-off-by: Edita Kizinevic --- .../runtime/launcher/oci/launcher_linux.go | 23 +++- .../pkg/runtime/launcher/oci/process_linux.go | 8 ++ pkg/ocibundle/native/bundle_linux.go | 79 +++++++++++ pkg/ocibundle/native/bundle_linux_test.go | 123 ++++++++++++++++++ 4 files changed, 228 insertions(+), 5 deletions(-) diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index 67a00bd536..7c9f2db864 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -128,9 +128,6 @@ func checkOpts(lo launcher.Options) error { badOpt = append(badOpt, "ContainLibs") } - if len(lo.Env) > 0 { - badOpt = append(badOpt, "Env") - } if lo.EnvFile != "" { badOpt = append(badOpt, "EnvFile") } @@ -231,8 +228,9 @@ func checkOpts(lo launcher.Options) error { } // createSpec produces an OCI runtime specification, suitable to launch a -// container. This spec excludes ProcessArgs, as these have to be computed where -// the image config is available, to account for the image's CMD / ENTRYPOINT. +// container. This spec excludes ProcessArgs and Env, as these have to be +// computed where the image config is available, to account for the image's CMD +// / ENTRYPOINT / ENV. func (l *Launcher) createSpec() (*specs.Spec, error) { spec := minimalSpec() @@ -319,12 +317,20 @@ func (l *Launcher) Exec(ctx context.Context, image string, process string, args return fmt.Errorf("while creating OCI spec: %w", err) } + // Assemble the runtime & user-requested environment, which will be merged + // with the image ENV and set in the container at runtime. + rtEnv := defaultEnv(image, bundleDir) + // --env flag + rtEnv = mergeMap(rtEnv, l.cfg.Env) + // TODO - --env-file, APPTAINERENV_ + b, err := native.New( native.OptBundlePath(bundleDir), native.OptImageRef(image), native.OptSysCtx(sysCtx), native.OptImgCache(imgCache), native.OptProcessArgs(process, args), + native.OptProcessEnv(rtEnv), ) if err != nil { return err @@ -352,3 +358,10 @@ func (l *Launcher) Exec(ctx context.Context, image string, process string, args } return err } + +func mergeMap(a map[string]string, b map[string]string) map[string]string { + for k, v := range b { + a[k] = v + } + return a +} diff --git a/internal/pkg/runtime/launcher/oci/process_linux.go b/internal/pkg/runtime/launcher/oci/process_linux.go index 21dd98fdc5..cf6780d686 100644 --- a/internal/pkg/runtime/launcher/oci/process_linux.go +++ b/internal/pkg/runtime/launcher/oci/process_linux.go @@ -141,3 +141,11 @@ func (l *Launcher) getReverseUserMaps() (uidMap, gidMap []specs.LinuxIDMapping, return uidMap, gidMap, nil } + +// defaultEnv returns default environment variables set in the container. +func defaultEnv(image, bundle string) map[string]string { + return map[string]string{ + "APPTAINER_CONTAINER": bundle, + "APPTAINER_NAME": image, + } +} diff --git a/pkg/ocibundle/native/bundle_linux.go b/pkg/ocibundle/native/bundle_linux.go index 6573396a43..203eb89b65 100644 --- a/pkg/ocibundle/native/bundle_linux.go +++ b/pkg/ocibundle/native/bundle_linux.go @@ -39,6 +39,8 @@ import ( "github.com/opencontainers/umoci/pkg/idtools" ) +const apptainerLibs = "/.singularity.d/libs" + // Bundle is a native OCI bundle, created from imageRef. type Bundle struct { // imageRef is the reference to the OCI image source, e.g. docker://ubuntu:latest. @@ -57,6 +59,8 @@ type Bundle struct { process string // args are the command arguments, which may override the image's CMD. args []string + // env is the container environment to set, which will be merged with the image's env. + env map[string]string // Generic bundle properties ocibundle.Bundle } @@ -108,6 +112,14 @@ func OptProcessArgs(process string, args []string) Option { } } +// OptEnv sets the environment to be set, merged with the image ENV. +func OptProcessEnv(env map[string]string) Option { + return func(b *Bundle) error { + b.env = env + return nil + } +} + // New returns a bundle interface to create/delete an OCI bundle from an OCI image ref. func New(opts ...Option) (ocibundle.Bundle, error) { b := Bundle{ @@ -160,6 +172,8 @@ func (b *Bundle) Create(ctx context.Context, ociConfig *specs.Spec) error { // consult the image Config to handle combining ENTRYPOINT/CMD with user // provided args. b.setProcessArgs(g) + // Ditto for environment handling (merge image and user/rt requested). + b.setProcessEnv(g) return b.writeConfig(g) } @@ -189,6 +203,71 @@ func (b *Bundle) setProcessArgs(g *generate.Generator) { g.SetProcessArgs(processArgs) } +// setProcessEnv combines the image config ENV with the ENV requested in the runtime provided spec. +// APPEND_PATH and PREPEND_PATH are honored as with the native apptainer runtime. +// LD_LIBRARY_PATH is modified to always include the apptainer lib bind directory. +func (b *Bundle) setProcessEnv(g *generate.Generator) { + if g.Config == nil { + g.Config = &specs.Spec{} + } + if g.Config.Process == nil { + g.Config.Process = &specs.Process{} + } + g.Config.Process.Env = b.imageSpec.Config.Env + + path := "" + appendPath := "" + prependPath := "" + ldLibraryPath := "" + + // Obtain PATH, and LD_LIBRARY_PATH if set in the image config. + for _, env := range b.imageSpec.Config.Env { + e := strings.SplitN(env, "=", 2) + if len(e) < 2 { + continue + } + if e[0] == "PATH" { + path = e[1] + } + if e[0] == "LD_LIBRARY_PATH" { + ldLibraryPath = e[1] + } + } + + // Apply env vars from spec, except PATH and LD_LIBRARY_PATH releated. + for k, v := range b.env { + switch k { + case "PATH": + path = v + case "APPEND_PATH": + appendPath = v + case "PREPEND_PATH": + prependPath = v + case "LD_LIBRARY_PATH": + ldLibraryPath = v + default: + g.SetProcessEnv(k, v) + } + } + + // Compute and set optionally APPEND-ed / PREPEND-ed PATH. + if appendPath != "" { + path = path + ":" + appendPath + } + if prependPath != "" { + path = prependPath + ":" + path + } + if path != "" { + g.SetProcessEnv("PATH", path) + } + + // Ensure LD_LIBRARY_PATH always contains apptainer lib binding dir. + if !strings.Contains(ldLibraryPath, apptainerLibs) { + ldLibraryPath = strings.TrimPrefix(ldLibraryPath+":"+apptainerLibs, ":") + } + g.SetProcessEnv("LD_LIBRARY_PATH", ldLibraryPath) +} + func (b *Bundle) writeConfig(g *generate.Generator) error { return tools.SaveBundleConfig(b.bundlePath, g) } diff --git a/pkg/ocibundle/native/bundle_linux_test.go b/pkg/ocibundle/native/bundle_linux_test.go index 77e6b03a2f..ee29e9f7b9 100644 --- a/pkg/ocibundle/native/bundle_linux_test.go +++ b/pkg/ocibundle/native/bundle_linux_test.go @@ -21,6 +21,7 @@ import ( "github.com/apptainer/apptainer/internal/pkg/cache" "github.com/apptainer/apptainer/internal/pkg/runtime/engine/config/oci" + "github.com/apptainer/apptainer/internal/pkg/runtime/engine/config/oci/generate" v1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/opencontainers/runtime-tools/validate" ) @@ -268,3 +269,125 @@ func TestSetProcessArgs(t *testing.T) { }) } } + +func TestSetProcessEnv(t *testing.T) { + tests := []struct { + name string + imageEnv []string + bundleEnv map[string]string + wantEnv []string + }{ + { + name: "Default", + imageEnv: []string{}, + bundleEnv: map[string]string{}, + wantEnv: []string{"LD_LIBRARY_PATH=/.singularity.d/libs"}, + }, + { + name: "ImagePath", + imageEnv: []string{"PATH=/foo"}, + bundleEnv: map[string]string{}, + wantEnv: []string{ + "PATH=/foo", + "LD_LIBRARY_PATH=/.singularity.d/libs", + }, + }, + { + name: "OverridePath", + imageEnv: []string{"PATH=/foo"}, + bundleEnv: map[string]string{"PATH": "/bar"}, + wantEnv: []string{ + "PATH=/bar", + "LD_LIBRARY_PATH=/.singularity.d/libs", + }, + }, + { + name: "AppendPath", + imageEnv: []string{"PATH=/foo"}, + bundleEnv: map[string]string{"APPEND_PATH": "/bar"}, + wantEnv: []string{ + "PATH=/foo:/bar", + "LD_LIBRARY_PATH=/.singularity.d/libs", + }, + }, + { + name: "PrependPath", + imageEnv: []string{"PATH=/foo"}, + bundleEnv: map[string]string{"PREPEND_PATH": "/bar"}, + wantEnv: []string{ + "PATH=/bar:/foo", + "LD_LIBRARY_PATH=/.singularity.d/libs", + }, + }, + { + name: "ImageLdLibraryPath", + imageEnv: []string{"LD_LIBRARY_PATH=/foo"}, + bundleEnv: map[string]string{}, + wantEnv: []string{ + "LD_LIBRARY_PATH=/foo:/.singularity.d/libs", + }, + }, + { + name: "BundleLdLibraryPath", + imageEnv: []string{}, + bundleEnv: map[string]string{"LD_LIBRARY_PATH": "/foo"}, + wantEnv: []string{ + "LD_LIBRARY_PATH=/foo:/.singularity.d/libs", + }, + }, + { + name: "OverrideLdLibraryPath", + imageEnv: []string{"LD_LIBRARY_PATH=/foo"}, + bundleEnv: map[string]string{"LD_LIBRARY_PATH": "/bar"}, + wantEnv: []string{ + "LD_LIBRARY_PATH=/bar:/.singularity.d/libs", + }, + }, + { + name: "ImageVar", + imageEnv: []string{"FOO=bar"}, + bundleEnv: map[string]string{}, + wantEnv: []string{ + "FOO=bar", + "LD_LIBRARY_PATH=/.singularity.d/libs", + }, + }, + { + name: "ImageOverride", + imageEnv: []string{"FOO=bar"}, + bundleEnv: map[string]string{"FOO": "baz"}, + wantEnv: []string{ + "FOO=baz", + "LD_LIBRARY_PATH=/.singularity.d/libs", + }, + }, + { + name: "ImageAdditional", + imageEnv: []string{"FOO=bar"}, + bundleEnv: map[string]string{"ABC": "123"}, + wantEnv: []string{ + "FOO=bar", + "ABC=123", + "LD_LIBRARY_PATH=/.singularity.d/libs", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + imgSpec := &v1.Image{ + Config: v1.ImageConfig{Env: tt.imageEnv}, + } + + b := &Bundle{ + imageSpec: imgSpec, + env: tt.bundleEnv, + } + g := &generate.Generator{} + b.setProcessEnv(g) + + if !reflect.DeepEqual(g.Config.Process.Env, tt.wantEnv) { + t.Errorf("want: %v, got: %v", tt.wantEnv, g.Config.Process.Env) + } + }) + } +} From ed191d11a218ccebe62119a8f34e77c5c4ed21b6 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Tue, 6 Dec 2022 10:47:12 +0000 Subject: [PATCH 046/114] e2e: add --oci --env tests Signed-off-by: Edita Kizinevic --- e2e/env/env.go | 4 ++ e2e/env/oci.go | 126 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 130 insertions(+) create mode 100644 e2e/env/oci.go diff --git a/e2e/env/env.go b/e2e/env/env.go index 0c928bc747..56f7198798 100644 --- a/e2e/env/env.go +++ b/e2e/env/env.go @@ -645,5 +645,9 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { "issue 5426": c.issue5426, // https://github.com/apptainer/singularity/issues/5426 "issue 43": c.issue43, // https://github.com/sylabs/singularity/issues/43 "issue 1263": c.issue1263, // https://github.com/sylabs/singularity/issues/1263 + // + // --oci mode + // + "oci environment option": c.ociEnvOption, } } diff --git a/e2e/env/oci.go b/e2e/env/oci.go new file mode 100644 index 0000000000..e0a24c954e --- /dev/null +++ b/e2e/env/oci.go @@ -0,0 +1,126 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2022, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package apptainerenv + +import ( + "strings" + "testing" + + "github.com/apptainer/apptainer/e2e/internal/e2e" +) + +func (c ctx) ociEnvOption(t *testing.T) { + e2e.EnsureOCIImage(t, c.env) + defaultImage := "oci-archive:" + c.env.OCIImagePath + + tests := []struct { + name string + image string + envOpt []string + hostEnv []string + matchEnv string + matchVal string + }{ + { + name: "DefaultPath", + image: defaultImage, + matchEnv: "PATH", + matchVal: defaultPath, + }, + { + name: "DefaultPathOverride", + image: defaultImage, + envOpt: []string{"PATH=/"}, + matchEnv: "PATH", + matchVal: "/", + }, + { + name: "AppendDefaultPath", + image: defaultImage, + envOpt: []string{"APPEND_PATH=/foo"}, + matchEnv: "PATH", + matchVal: defaultPath + ":/foo", + }, + { + name: "PrependDefaultPath", + image: defaultImage, + envOpt: []string{"PREPEND_PATH=/foo"}, + matchEnv: "PATH", + matchVal: "/foo:" + defaultPath, + }, + { + name: "TestMultiLine", + image: defaultImage, + envOpt: []string{"MULTI=Hello\nWorld"}, + matchEnv: "MULTI", + matchVal: "Hello\nWorld", + }, + { + name: "TestEscapedNewline", + image: defaultImage, + envOpt: []string{"ESCAPED=Hello\\nWorld"}, + matchEnv: "ESCAPED", + matchVal: "Hello\\nWorld", + }, + { + name: "TestInvalidKey", + image: defaultImage, + envOpt: []string{"BASH_FUNC_ml%%=TEST"}, + matchEnv: "BASH_FUNC_ml%%", + matchVal: "", + }, + { + name: "TestDefaultLdLibraryPath", + image: defaultImage, + matchEnv: "LD_LIBRARY_PATH", + matchVal: apptainerLibs, + }, + { + name: "TestCustomTrailingCommaPath", + image: defaultImage, + envOpt: []string{"LD_LIBRARY_PATH=/foo,"}, + matchEnv: "LD_LIBRARY_PATH", + matchVal: "/foo,:" + apptainerLibs, + }, + { + name: "TestCustomLdLibraryPath", + image: defaultImage, + envOpt: []string{"LD_LIBRARY_PATH=/foo"}, + matchEnv: "LD_LIBRARY_PATH", + matchVal: "/foo:" + apptainerLibs, + }, + { + name: "APPTAINER_NAME", + image: defaultImage, + matchEnv: "APPTAINER_NAME", + matchVal: defaultImage, + }, + } + + for _, tt := range tests { + args := make([]string, 0) + if tt.envOpt != nil { + args = append(args, "--env", strings.Join(tt.envOpt, ",")) + } + args = append(args, tt.image, "/bin/sh", "-c", "echo \"${"+tt.matchEnv+"}\"") + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(e2e.OCIUserProfile), + e2e.WithCommand("exec"), + e2e.WithEnv(tt.hostEnv), + e2e.WithArgs(args...), + e2e.ExpectExit( + 0, + e2e.ExpectOutput(e2e.ExactMatch, tt.matchVal), + ), + ) + } +} From 75ec7b4a7e85bb95600beae6129c934fcce8e6b5 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Tue, 6 Dec 2022 11:13:13 +0000 Subject: [PATCH 047/114] feat: oci: add SINGULARITYENV_ handling for --oci mode Pass SINGULARITYENV_ prefixed environment variables into container in --oci mode. Fixes sylabs/singularity#1031 Signed-off-by: Edita Kizinevic --- .../runtime/launcher/oci/launcher_linux.go | 4 +- .../pkg/runtime/launcher/oci/process_linux.go | 25 +++++++- .../launcher/oci/process_linux_test.go | 64 +++++++++++++++++++ 3 files changed, 90 insertions(+), 3 deletions(-) create mode 100644 internal/pkg/runtime/launcher/oci/process_linux_test.go diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index 7c9f2db864..2a55551758 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -320,9 +320,11 @@ func (l *Launcher) Exec(ctx context.Context, image string, process string, args // Assemble the runtime & user-requested environment, which will be merged // with the image ENV and set in the container at runtime. rtEnv := defaultEnv(image, bundleDir) + // APPTAINERENV_ + rtEnv = mergeMap(rtEnv, apptainerEnvMap()) // --env flag rtEnv = mergeMap(rtEnv, l.cfg.Env) - // TODO - --env-file, APPTAINERENV_ + // TODO - --env-file b, err := native.New( native.OptBundlePath(bundleDir), diff --git a/internal/pkg/runtime/launcher/oci/process_linux.go b/internal/pkg/runtime/launcher/oci/process_linux.go index cf6780d686..ad15049c08 100644 --- a/internal/pkg/runtime/launcher/oci/process_linux.go +++ b/internal/pkg/runtime/launcher/oci/process_linux.go @@ -12,8 +12,10 @@ package oci import ( "fmt" "os" + "strings" "github.com/apptainer/apptainer/internal/pkg/fakeroot" + "github.com/apptainer/apptainer/internal/pkg/util/env" "github.com/apptainer/apptainer/internal/pkg/util/user" "github.com/opencontainers/runtime-spec/specs-go" ) @@ -145,7 +147,26 @@ func (l *Launcher) getReverseUserMaps() (uidMap, gidMap []specs.LinuxIDMapping, // defaultEnv returns default environment variables set in the container. func defaultEnv(image, bundle string) map[string]string { return map[string]string{ - "APPTAINER_CONTAINER": bundle, - "APPTAINER_NAME": image, + env.ApptainerPrefix + "CONTAINER": bundle, + env.ApptainerPrefix + "NAME": image, } } + +// apptainerEnvMap returns a map of APPTAINERENV_ prefixed env vars to their values. +func apptainerEnvMap() map[string]string { + apptainerEnv := map[string]string{} + + for _, envVar := range os.Environ() { + if !strings.HasPrefix(envVar, env.ApptainerEnvPrefix) { + continue + } + parts := strings.SplitN(envVar, "=", 2) + if len(parts) < 2 { + continue + } + key := strings.TrimPrefix(parts[0], env.ApptainerEnvPrefix) + apptainerEnv[key] = parts[1] + } + + return apptainerEnv +} diff --git a/internal/pkg/runtime/launcher/oci/process_linux_test.go b/internal/pkg/runtime/launcher/oci/process_linux_test.go new file mode 100644 index 0000000000..c144325b0d --- /dev/null +++ b/internal/pkg/runtime/launcher/oci/process_linux_test.go @@ -0,0 +1,64 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2022, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package oci + +import ( + "os" + "reflect" + "testing" +) + +func TestApptainerEnvMap(t *testing.T) { + tests := []struct { + name string + setEnv map[string]string + want map[string]string + }{ + { + name: "None", + setEnv: map[string]string{}, + want: map[string]string{}, + }, + { + name: "NonPrefixed", + setEnv: map[string]string{"FOO": "bar"}, + want: map[string]string{}, + }, + { + name: "PrefixedSingle", + setEnv: map[string]string{"APPTAINERENV_FOO": "bar"}, + want: map[string]string{"FOO": "bar"}, + }, + { + name: "PrefixedMultiple", + setEnv: map[string]string{ + "APPTAINERENV_FOO": "bar", + "APPTAINERENV_ABC": "123", + }, + want: map[string]string{ + "FOO": "bar", + "ABC": "123", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for k, v := range tt.setEnv { + os.Setenv(k, v) + t.Cleanup(func() { + os.Unsetenv(k) + }) + } + if got := apptainerEnvMap(); !reflect.DeepEqual(got, tt.want) { + t.Errorf("apptainerEnvMap() = %v, want %v", got, tt.want) + } + }) + } +} From 1da3b41225a7597099d5b7178e98b2b69882cf1e Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Tue, 6 Dec 2022 11:20:29 +0000 Subject: [PATCH 048/114] e2e: add SINGULARITYENV_ tests for --oci mode Signed-off-by: Edita Kizinevic --- e2e/env/env.go | 3 ++- e2e/env/oci.go | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 69 insertions(+), 1 deletion(-) diff --git a/e2e/env/env.go b/e2e/env/env.go index 56f7198798..26ffa3b5c5 100644 --- a/e2e/env/env.go +++ b/e2e/env/env.go @@ -648,6 +648,7 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { // // --oci mode // - "oci environment option": c.ociEnvOption, + "oci environment apptainerenv": c.ociApptainerEnv, + "oci environment option": c.ociEnvOption, } } diff --git a/e2e/env/oci.go b/e2e/env/oci.go index e0a24c954e..9ef6777225 100644 --- a/e2e/env/oci.go +++ b/e2e/env/oci.go @@ -16,6 +16,73 @@ import ( "github.com/apptainer/apptainer/e2e/internal/e2e" ) +func (c ctx) ociApptainerEnv(t *testing.T) { + e2e.EnsureOCIImage(t, c.env) + defaultImage := "oci-archive:" + c.env.OCIImagePath + + // Append or prepend this path. + partialPath := "/foo" + + // Overwrite the path with this one. + overwrittenPath := "/usr/bin:/bin" + + // A path with a trailing comma + trailingCommaPath := "/usr/bin:/bin," + + tests := []struct { + name string + image string + path string + env []string + }{ + { + name: "DefaultPath", + image: defaultImage, + path: defaultPath, + env: []string{}, + }, + { + name: "AppendToDefaultPath", + image: defaultImage, + path: defaultPath + ":" + partialPath, + env: []string{"APPTAINERENV_APPEND_PATH=/foo"}, + }, + { + name: "PrependToDefaultPath", + image: defaultImage, + path: partialPath + ":" + defaultPath, + env: []string{"APPTAINERENV_PREPEND_PATH=/foo"}, + }, + { + name: "OverwriteDefaultPath", + image: defaultImage, + path: overwrittenPath, + env: []string{"APPTAINERENV_PATH=" + overwrittenPath}, + }, + { + name: "OverwriteTrailingCommaPath", + image: defaultImage, + path: trailingCommaPath, + env: []string{"APPTAINERENV_PATH=" + trailingCommaPath}, + }, + } + + for _, tt := range tests { + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(e2e.OCIUserProfile), + e2e.WithCommand("exec"), + e2e.WithEnv(tt.env), + e2e.WithArgs(tt.image, "/bin/sh", "-c", "echo $PATH"), + e2e.ExpectExit( + 0, + e2e.ExpectOutput(e2e.ExactMatch, tt.path), + ), + ) + } +} + func (c ctx) ociEnvOption(t *testing.T) { e2e.EnsureOCIImage(t, c.env) defaultImage := "oci-archive:" + c.env.OCIImagePath From 1c527b4cfcb7358edced0ce8aec64eb2e5dd5532 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Tue, 6 Dec 2022 12:12:17 +0000 Subject: [PATCH 049/114] feat: oci: enable --env-file in --oci mode Allow --env-file to be used to provide environment variables in a file, when running a container in --oci mode. We use the same approach as the native runtime for compatibility. The env file is evaluated in the embedded shell interpreter, but starting with an empty environment. This handles quoting, comments etc. for us, and keeps maximum compatibility with the existing handling. Fixes sylabs/singularity#1030 Signed-off-by: Edita Kizinevic --- .../runtime/launcher/native/launcher_linux.go | 6 +- .../runtime/launcher/oci/launcher_linux.go | 16 ++-- .../pkg/runtime/launcher/oci/process_linux.go | 38 +++++++++ .../launcher/oci/process_linux_test.go | 84 +++++++++++++++++++ 4 files changed, 135 insertions(+), 9 deletions(-) diff --git a/internal/pkg/runtime/launcher/native/launcher_linux.go b/internal/pkg/runtime/launcher/native/launcher_linux.go index 9784996fe1..e7afef09db 100644 --- a/internal/pkg/runtime/launcher/native/launcher_linux.go +++ b/internal/pkg/runtime/launcher/native/launcher_linux.go @@ -985,7 +985,7 @@ func (l *Launcher) setEnvVars(ctx context.Context, args []string) error { content, err := os.ReadFile(l.cfg.EnvFile) if err != nil { - return fmt.Errorf("could not read %q environment file: %w", l.cfg.EnvFile, err) + return fmt.Errorf("could not read environment file %q: %w", l.cfg.EnvFile, err) } envvars, err := interpreter.EvaluateEnv(ctx, content, args, currentEnv) @@ -1000,7 +1000,7 @@ func (l *Launcher) setEnvVars(ctx context.Context, args []string) error { for _, envar := range envvars { e := strings.SplitN(envar, "=", 2) if len(e) != 2 { - sylog.Warningf("Ignore environment variable %q: '=' is missing", envar) + sylog.Warningf("Ignored environment variable %q: '=' is missing", envar) continue } // Don't attempt to overwrite bash builtin readonly vars @@ -1010,7 +1010,7 @@ func (l *Launcher) setEnvVars(ctx context.Context, args []string) error { } // Ensure we don't overwrite --env variables with environment file if _, ok := l.cfg.Env[e[0]]; ok { - sylog.Warningf("Ignore environment variable %s from %s: override from --env", e[0], l.cfg.EnvFile) + sylog.Warningf("Ignored environment variable %s from %s: override from --env", e[0], l.cfg.EnvFile) } else { l.cfg.Env[e[0]] = e[1] } diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index 2a55551758..1cd2300cd0 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -128,9 +128,6 @@ func checkOpts(lo launcher.Options) error { badOpt = append(badOpt, "ContainLibs") } - if lo.EnvFile != "" { - badOpt = append(badOpt, "EnvFile") - } if lo.CleanEnv { badOpt = append(badOpt, "CleanEnv") } @@ -320,11 +317,18 @@ func (l *Launcher) Exec(ctx context.Context, image string, process string, args // Assemble the runtime & user-requested environment, which will be merged // with the image ENV and set in the container at runtime. rtEnv := defaultEnv(image, bundleDir) - // APPTAINERENV_ + // APPTAINERENV_ has lowest priority rtEnv = mergeMap(rtEnv, apptainerEnvMap()) - // --env flag + // --env-file can override APPTAINERENV_ + if l.cfg.EnvFile != "" { + e, err := envFileMap(ctx, l.cfg.EnvFile) + if err != nil { + return err + } + rtEnv = mergeMap(rtEnv, e) + } + // --env flag can override --env-file and APPTAINERENV_ rtEnv = mergeMap(rtEnv, l.cfg.Env) - // TODO - --env-file b, err := native.New( native.OptBundlePath(bundleDir), diff --git a/internal/pkg/runtime/launcher/oci/process_linux.go b/internal/pkg/runtime/launcher/oci/process_linux.go index ad15049c08..9390935f70 100644 --- a/internal/pkg/runtime/launcher/oci/process_linux.go +++ b/internal/pkg/runtime/launcher/oci/process_linux.go @@ -10,12 +10,14 @@ package oci import ( + "context" "fmt" "os" "strings" "github.com/apptainer/apptainer/internal/pkg/fakeroot" "github.com/apptainer/apptainer/internal/pkg/util/env" + "github.com/apptainer/apptainer/internal/pkg/util/shell/interpreter" "github.com/apptainer/apptainer/internal/pkg/util/user" "github.com/opencontainers/runtime-spec/specs-go" ) @@ -170,3 +172,39 @@ func apptainerEnvMap() map[string]string { return apptainerEnv } + +// envFileMap returns a map of KEY=VAL env vars from an environment file +func envFileMap(ctx context.Context, f string) (map[string]string, error) { + envMap := map[string]string{} + + content, err := os.ReadFile(f) + if err != nil { + return envMap, fmt.Errorf("could not read environment file %q: %w", f, err) + } + + // Use the embedded shell interpreter to evaluate the env file, with an empty starting environment. + // Shell takes care of comments, quoting etc. for us and keeps compatibility with native runtime. + env, err := interpreter.EvaluateEnv(ctx, content, []string{}, []string{}) + if err != nil { + return envMap, fmt.Errorf("while processing %s: %w", f, err) + } + + for _, envVar := range env { + parts := strings.SplitN(envVar, "=", 2) + if len(parts) < 2 { + continue + } + // Strip out the runtime env vars set by the shell interpreter + if parts[0] == "GID" || + parts[0] == "HOME" || + parts[0] == "IFS" || + parts[0] == "OPTIND" || + parts[0] == "PWD" || + parts[0] == "UID" { + continue + } + envMap[parts[0]] = parts[1] + } + + return envMap, nil +} diff --git a/internal/pkg/runtime/launcher/oci/process_linux_test.go b/internal/pkg/runtime/launcher/oci/process_linux_test.go index c144325b0d..9fffa596d3 100644 --- a/internal/pkg/runtime/launcher/oci/process_linux_test.go +++ b/internal/pkg/runtime/launcher/oci/process_linux_test.go @@ -10,7 +10,9 @@ package oci import ( + "context" "os" + "path/filepath" "reflect" "testing" ) @@ -62,3 +64,85 @@ func TestApptainerEnvMap(t *testing.T) { }) } } + +func TestEnvFileMap(t *testing.T) { + tests := []struct { + name string + envFile string + want map[string]string + wantErr bool + }{ + { + name: "EmptyFile", + envFile: "", + want: map[string]string{ + "EUID": "0", + }, + wantErr: false, + }, + { + name: "Simple", + envFile: `FOO=BAR + ABC=123`, + want: map[string]string{ + "EUID": "0", + "FOO": "BAR", + "ABC": "123", + }, + wantErr: false, + }, + { + name: "DoubleQuote", + envFile: `FOO="FOO BAR"`, + want: map[string]string{ + "EUID": "0", + "FOO": "FOO BAR", + }, + wantErr: false, + }, + { + name: "SingleQuote", + envFile: `FOO='FOO BAR'`, + want: map[string]string{ + "EUID": "0", + "FOO": "FOO BAR", + }, + wantErr: false, + }, + { + name: "MultiLine", + envFile: "FOO=\"FOO\nBAR\"", + want: map[string]string{ + "EUID": "0", + "FOO": "FOO\nBAR", + }, + wantErr: false, + }, + { + name: "Invalid", + envFile: "!!!@@NOTAVAR", + want: map[string]string{}, + wantErr: true, + }, + } + + tmpDir := t.TempDir() + envFile := filepath.Join(tmpDir, "env-file") + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := os.WriteFile(envFile, []byte(tt.envFile), 0o755); err != nil { + t.Fatalf("Could not write test env-file: %v", err) + } + + got, err := envFileMap(context.Background(), envFile) + if (err != nil) != tt.wantErr { + t.Errorf("envFileMap() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("envFileMap() = %v, want %v", got, tt.want) + } + }) + } +} From 31ea98e1d0a17e36f3125502a569d20c0f80b1bf Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Tue, 6 Dec 2022 12:21:08 +0000 Subject: [PATCH 050/114] e2e: add --env-file tests for --oci mode Signed-off-by: Edita Kizinevic --- e2e/env/env.go | 1 + e2e/env/oci.go | 104 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 105 insertions(+) diff --git a/e2e/env/env.go b/e2e/env/env.go index 26ffa3b5c5..b4509dc523 100644 --- a/e2e/env/env.go +++ b/e2e/env/env.go @@ -650,5 +650,6 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { // "oci environment apptainerenv": c.ociApptainerEnv, "oci environment option": c.ociEnvOption, + "oci environment file": c.ociEnvFile, } } diff --git a/e2e/env/oci.go b/e2e/env/oci.go index 9ef6777225..444d8128fb 100644 --- a/e2e/env/oci.go +++ b/e2e/env/oci.go @@ -10,6 +10,8 @@ package apptainerenv import ( + "os" + "path/filepath" "strings" "testing" @@ -191,3 +193,105 @@ func (c ctx) ociEnvOption(t *testing.T) { ) } } + +func (c ctx) ociEnvFile(t *testing.T) { + e2e.EnsureOCIImage(t, c.env) + defaultImage := "oci-archive:" + c.env.OCIImagePath + + dir, cleanup := e2e.MakeTempDir(t, c.env.TestDir, "envfile-", "") + defer cleanup(t) + p := filepath.Join(dir, "env.file") + + tests := []struct { + name string + image string + envFile string + envOpt []string + hostEnv []string + matchEnv string + matchVal string + }{ + { + name: "DefaultPathOverride", + image: defaultImage, + envFile: "PATH=/", + matchEnv: "PATH", + matchVal: "/", + }, + { + name: "DefaultPathOverrideEnvOptionPrecedence", + image: defaultImage, + envOpt: []string{"PATH=/etc"}, + envFile: "PATH=/", + matchEnv: "PATH", + matchVal: "/etc", + }, + { + name: "DefaultPathOverrideEnvOptionPrecedence", + image: defaultImage, + envOpt: []string{"PATH=/etc"}, + envFile: "PATH=/", + matchEnv: "PATH", + matchVal: "/etc", + }, + { + name: "AppendDefaultPath", + image: defaultImage, + envFile: "APPEND_PATH=/", + matchEnv: "PATH", + matchVal: defaultPath + ":/", + }, + { + name: "PrependDefaultPath", + image: defaultImage, + envFile: "PREPEND_PATH=/", + matchEnv: "PATH", + matchVal: "/:" + defaultPath, + }, + { + name: "DefaultLdLibraryPath", + image: defaultImage, + matchEnv: "LD_LIBRARY_PATH", + matchVal: apptainerLibs, + }, + { + name: "CustomLdLibraryPath", + image: defaultImage, + envFile: "LD_LIBRARY_PATH=/foo", + matchEnv: "LD_LIBRARY_PATH", + matchVal: "/foo:" + apptainerLibs, + }, + { + name: "CustomTrailingCommaPath", + image: defaultImage, + envFile: "LD_LIBRARY_PATH=/foo,", + matchEnv: "LD_LIBRARY_PATH", + matchVal: "/foo,:" + apptainerLibs, + }, + } + + for _, tt := range tests { + args := make([]string, 0) + if tt.envOpt != nil { + args = append(args, "--env", strings.Join(tt.envOpt, ",")) + } + if tt.envFile != "" { + os.WriteFile(p, []byte(tt.envFile), 0o644) + args = append(args, "--env-file", p) + } + args = append(args, tt.image, "/bin/sh", "-c", "echo $"+tt.matchEnv) + + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(e2e.OCIUserProfile), + e2e.WithCommand("exec"), + e2e.WithEnv(tt.hostEnv), + e2e.WithArgs(args...), + e2e.ExpectExit( + 0, + e2e.ExpectOutput(e2e.ExactMatch, tt.matchVal), + ), + ) + } +} From d896ab60148fe4e3b54ed600e670278f6b90baad Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Wed, 7 Dec 2022 10:24:55 +0000 Subject: [PATCH 051/114] doc: Add CHANGELOG entry for `--oci` Now that sufficient functionality is in place for `--oci` mode, add a CHANGELOG entry for the current state. Signed-off-by: Edita Kizinevic --- CHANGELOG.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e5674c888..4afaeece20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,21 @@ For older changes see the [archived Singularity change log](https://github.com/a of the logged-in user, if available. - New option `--warn-unused-build-args` is provided to output warnings rather than fatal errors for any unused variables given in --build-arg or --build-arg-file. +- A new `--oci` flag for `run/exec/shell` enables the experimental OCI runtime + mode. This mode: + - Runs OCI container images from an OCI bundle, using `runc` or `crun`. + - Supports `docker://`, `docker-archive:`, `docker-daemon:`, `oci:`, + `oci-archive:` image sources. + - Does not support running Apptainer SIF, SquashFS, or EXT3 images. + - Provides an environment similar to Apptainer's native runtime, running + with `--compat`. + - Supports the following options / flags. Other options are not yet supported: + - `--fakeroot` for effective root in the container. Requires subuid/subgid + mappings. + - Bind mounts via `--bind` or `--mount`. No image mounts. + - Additional namespaces requests with `--net`, `--uts`, `--user`. + - Container environment variables via `--env`, `--env-file`, and + `APPTAINERENV_` host env vars. ### New Features & Functionality From 3ec413394095a6eb7e7e27826538336148b0968f Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Thu, 8 Dec 2022 11:59:30 +0000 Subject: [PATCH 052/114] fix: oci: ensure user/group entries in container Signed-off-by: Edita Kizinevic --- .../runtime/launcher/oci/launcher_linux.go | 51 ++++++++++++++++--- 1 file changed, 45 insertions(+), 6 deletions(-) diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index 1cd2300cd0..ffd9493ce1 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -18,13 +18,17 @@ import ( "fmt" "os" "os/exec" + "path/filepath" "strings" "syscall" "github.com/apptainer/apptainer/internal/pkg/buildcfg" "github.com/apptainer/apptainer/internal/pkg/cache" "github.com/apptainer/apptainer/internal/pkg/runtime/launcher" + "github.com/apptainer/apptainer/internal/pkg/util/fs/files" + "github.com/apptainer/apptainer/internal/pkg/util/user" "github.com/apptainer/apptainer/pkg/ocibundle/native" + "github.com/apptainer/apptainer/pkg/ocibundle/tools" "github.com/apptainer/apptainer/pkg/syfs" "github.com/apptainer/apptainer/pkg/sylog" "github.com/apptainer/apptainer/pkg/util/apptainerconf" @@ -117,12 +121,6 @@ func checkOpts(lo launcher.Options) error { if lo.NoNvidia { badOpt = append(badOpt, "NoNvidia") } - if lo.Rocm { - badOpt = append(badOpt, "Rocm") - } - if lo.NoRocm { - badOpt = append(badOpt, "NoRocm") - } if len(lo.ContainLibs) > 0 { badOpt = append(badOpt, "ContainLibs") @@ -266,6 +264,43 @@ func (l *Launcher) createSpec() (*specs.Spec, error) { return &spec, nil } +func (l *Launcher) updatePasswdGroup(rootfs string) error { + uid := os.Getuid() + gid := os.Getgid() + + if os.Getuid() == 0 || l.cfg.Fakeroot { + return nil + } + + containerPasswd := filepath.Join(rootfs, "etc", "passwd") + containerGroup := filepath.Join(rootfs, "etc", "group") + + pw, err := user.CurrentOriginal() + if err != nil { + return err + } + + sylog.Debugf("Updating passwd file: %s", containerPasswd) + content, err := files.Passwd(containerPasswd, pw.Dir, uidi, nil) + if err != nil { + return fmt.Errorf("while creating passwd file: %w", err) + } + if err := os.WriteFile(containerPasswd, content, 0o755); err != nil { + return fmt.Errorf("while writing passwd file: %w", err) + } + + sylog.Debugf("Updating group file: %s", containerGroup) + content, err = files.Group(containerGroup, uid, []int{gid}, nil) + if err != nil { + return fmt.Errorf("while creating group file: %w", err) + } + if err := os.WriteFile(containerGroup, content, 0o755); err != nil { + return fmt.Errorf("while writing passwd file: %w", err) + } + + return nil +} + // Exec will interactively execute a container via the runc low-level runtime. // image is a reference to an OCI image, e.g. docker://ubuntu or oci:/tmp/mycontainer func (l *Launcher) Exec(ctx context.Context, image string, process string, args []string, instanceName string) error { @@ -346,6 +381,10 @@ func (l *Launcher) Exec(ctx context.Context, image string, process string, args return err } + if err := l.updatePasswdGroup(tools.RootFs(b.Path()).Path()); err != nil { + return err + } + id, err := uuid.NewRandom() if err != nil { return fmt.Errorf("while generating container id: %w", err) From 5f601bf5ec8e4182a6094d2eecf53b4332f7a9b0 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Thu, 8 Dec 2022 12:12:26 +0000 Subject: [PATCH 053/114] feat: oci: Support --rocm with --oci mode Under --oci mode, allow --rocm, which will bind ROCm devices, libraries, and binaries into the container. Fixes sylabs/singularity#1034 Signed-off-by: Edita Kizinevic --- .../pkg/runtime/launcher/oci/mounts_linux.go | 96 +++++++++++++++++++ 1 file changed, 96 insertions(+) diff --git a/internal/pkg/runtime/launcher/oci/mounts_linux.go b/internal/pkg/runtime/launcher/oci/mounts_linux.go index 8f77d71f81..11143a0c95 100644 --- a/internal/pkg/runtime/launcher/oci/mounts_linux.go +++ b/internal/pkg/runtime/launcher/oci/mounts_linux.go @@ -16,13 +16,18 @@ import ( "fmt" "os" "path/filepath" + "strings" + "github.com/apptainer/apptainer/internal/pkg/buildcfg" + "github.com/apptainer/apptainer/internal/pkg/util/gpu" "github.com/apptainer/apptainer/internal/pkg/util/user" "github.com/apptainer/apptainer/pkg/sylog" "github.com/apptainer/apptainer/pkg/util/bind" "github.com/opencontainers/runtime-spec/specs-go" ) +const containerLibDir = "/.singularity.d/libs" + // getMounts returns a mount list for the container's OCI runtime spec. func (l *Launcher) getMounts() ([]specs.Mount, error) { mounts := &[]specs.Mount{} @@ -38,6 +43,12 @@ func (l *Launcher) getMounts() ([]specs.Mount, error) { if err := l.addBindMounts(mounts); err != nil { return nil, fmt.Errorf("while configuring bind mount(s): %w", err) } + if (l.cfg.Rocm || l.apptainerConf.AlwaysUseRocm) && !l.cfg.NoRocm { + if err := l.addRocmMounts(mounts); err != nil { + return nil, fmt.Errorf("while configuring ROCm mount(s): %w", err) + } + } + return *mounts, nil } @@ -254,3 +265,88 @@ func addBindMount(mounts *[]specs.Mount, b bind.Path) error { }) return nil } + +func addDevBindMount(mounts *[]specs.Mount, b bind.Path) error { + opts := []string{"bind", "nosuid"} + if b.Readonly() { + opts = append(opts, "ro") + } + + b.Source = filepath.Clean(b.Source) + if !strings.HasPrefix(b.Source, "/dev") { + return fmt.Errorf("device bind source must be an absolute path under /dev: %s", b.Source) + } + if b.Source != b.Destination { + return fmt.Errorf("device bind source %s must be the same as destination %s", b.Source, b.Destination) + } + if _, err := os.Stat(b.Source); err != nil { + return fmt.Errorf("cannot stat bind source %s: %w", b.Source, err) + } + + sylog.Debugf("Adding device bind of %s to %s, with options %v", b.Source, b.Destination, opts) + + *mounts = append(*mounts, + specs.Mount{ + Source: b.Source, + Destination: b.Destination, + Type: "none", + Options: opts, + }) + return nil +} + +func (l *Launcher) addRocmMounts(mounts *[]specs.Mount) error { + gpuConfFile := filepath.Join(buildcfg.APPTAINER_CONFDIR, "rocmliblist.conf") + + libs, bins, err := gpu.RocmPaths(gpuConfFile) + if err != nil { + sylog.Warningf("While finding ROCm bind points: %v", err) + } + if len(libs) == 0 { + sylog.Warningf("Could not find any ROCm libraries on this host!") + } + + devs, err := gpu.RocmDevices() + if err != nil { + sylog.Warningf("While finding ROCm devices: %v", err) + } + if len(devs) == 0 { + sylog.Warningf("Could not find any ROCm devices on this host!") + } + + for _, binary := range bins { + containerBinary := filepath.Join("/usr/bin", filepath.Base(binary)) + bind := bind.Path{ + Source: binary, + Destination: containerBinary, + Options: map[string]*bind.Option{"ro": {}}, + } + if err := addBindMount(mounts, bind); err != nil { + return err + } + } + + for _, lib := range libs { + containerLib := filepath.Join(containerLibDir, filepath.Base(lib)) + bind := bind.Path{ + Source: lib, + Destination: containerLib, + Options: map[string]*bind.Option{"ro": {}}, + } + if err := addBindMount(mounts, bind); err != nil { + return err + } + } + + for _, dev := range devs { + bind := bind.Path{ + Source: dev, + Destination: dev, + } + if err := addDevBindMount(mounts, bind); err != nil { + return err + } + } + + return nil +} From 1c653b9e55d9d4fddf04d3de0f556422689fc19b Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Thu, 8 Dec 2022 12:20:30 +0000 Subject: [PATCH 054/114] e2e: minimal --rocm --oci test Signed-off-by: Edita Kizinevic --- e2e/gpu/gpu.go | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/e2e/gpu/gpu.go b/e2e/gpu/gpu.go index 02f5fd3cc5..2a0f3742f0 100644 --- a/e2e/gpu/gpu.go +++ b/e2e/gpu/gpu.go @@ -289,6 +289,43 @@ func (c ctx) testRocm(t *testing.T) { } } +func (c ctx) ociTestRocm(t *testing.T) { + require.Rocm(t) + // Use Ubuntu 20.04 as this is the most recent distro officially supported by ROCm. + // We can't use our test image as it's alpine based and we need a compatible glibc. + imageURL := "docker://ubuntu:20.04" + + // Basic test that we can run the bound in `rocminfo` which *should* be on the PATH + tests := []struct { + profile e2e.Profile + args []string + }{ + { + profile: e2e.OCIUserProfile, + args: []string{"--rocm", imageURL, "rocminfo"}, + }, + { + profile: e2e.OCIFakerootProfile, + args: []string{"--rocm", imageURL, "rocminfo"}, + }, + { + profile: e2e.OCIRootProfile, + args: []string{"--rocm", imageURL, "rocminfo"}, + }, + } + + for _, tt := range tests { + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.profile.String()), + e2e.WithProfile(tt.profile), + e2e.WithCommand("exec"), + e2e.WithArgs(tt.args...), + e2e.ExpectExit(0), + ) + } +} + //nolint:dupl func (c ctx) testBuildNvidiaLegacy(t *testing.T) { require.Nvidia(t) @@ -555,5 +592,7 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { "build nvidia": c.testBuildNvidiaLegacy, "build nvccli": c.testBuildNvCCLI, "build rocm": c.testBuildRocm, + // oci mode + "oci rocm": c.ociTestRocm, } } From 33b061bda61dcc2ed621636db7b42bfebf793a5e Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Thu, 8 Dec 2022 12:45:05 +0000 Subject: [PATCH 055/114] fix: rocm: update rocmliblist and fix e2e tests `rocminfo` now needs `lsmod` and libdrm libdrm_amdgpu. Bind the former in tests, the libraries from rocmliblist.conf. We can now use Ubuntu 22.04 for ROCm tests. Signed-off-by: Edita Kizinevic --- e2e/gpu/gpu.go | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/e2e/gpu/gpu.go b/e2e/gpu/gpu.go index 2a0f3742f0..641a25398b 100644 --- a/e2e/gpu/gpu.go +++ b/e2e/gpu/gpu.go @@ -291,9 +291,18 @@ func (c ctx) testRocm(t *testing.T) { func (c ctx) ociTestRocm(t *testing.T) { require.Rocm(t) - // Use Ubuntu 20.04 as this is the most recent distro officially supported by ROCm. + + require.Command(t, "lsmod") + + // rocminfo now needs lsmod - do a brittle bind in for simplicity. + lsmod, err := exec.LookPath("lsmod") + if err != nil { + t.Fatalf("while finding lsmod: %v", err) + } + + // Use Ubuntu 22.04 as this is the most recent distro officially supported by ROCm. // We can't use our test image as it's alpine based and we need a compatible glibc. - imageURL := "docker://ubuntu:20.04" + imageURL := "docker://ubuntu:22.04" // Basic test that we can run the bound in `rocminfo` which *should* be on the PATH tests := []struct { @@ -302,15 +311,15 @@ func (c ctx) ociTestRocm(t *testing.T) { }{ { profile: e2e.OCIUserProfile, - args: []string{"--rocm", imageURL, "rocminfo"}, + args: []string{"-B", lsmod, "--rocm", imageURL, "rocminfo"}, }, { profile: e2e.OCIFakerootProfile, - args: []string{"--rocm", imageURL, "rocminfo"}, + args: []string{"-B", lsmod, "--rocm", imageURL, "rocminfo"}, }, { profile: e2e.OCIRootProfile, - args: []string{"--rocm", imageURL, "rocminfo"}, + args: []string{"-B", lsmod, "--rocm", imageURL, "rocminfo"}, }, } From 8313a4d64c730fcdee2a7d9edbfb2a260c58e212 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Fri, 9 Dec 2022 10:12:51 +0000 Subject: [PATCH 056/114] doc: --rocm --oci CHANGELOG entry Signed-off-by: Edita Kizinevic --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4afaeece20..a6d6dcb87a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -41,6 +41,7 @@ For older changes see the [archived Singularity change log](https://github.com/a - Additional namespaces requests with `--net`, `--uts`, `--user`. - Container environment variables via `--env`, `--env-file`, and `APPTAINERENV_` host env vars. + - `--rocm` to bind ROCm GPU libraries and devices into the container. ### New Features & Functionality From 17fcf131e7c4c3df544baca1f77dab04c38666f4 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Tue, 20 Dec 2022 12:27:10 +0000 Subject: [PATCH 057/114] oci: support legacy --nv bind mode Closes sylabs/singularity#1033 Signed-off-by: Edita Kizinevic --- CHANGELOG.md | 2 + e2e/gpu/gpu.go | 41 +++++++++- .../runtime/launcher/oci/launcher_linux.go | 6 -- .../pkg/runtime/launcher/oci/mounts_linux.go | 79 +++++++++++++++++++ 4 files changed, 121 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a6d6dcb87a..a1aab02582 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,8 @@ For older changes see the [archived Singularity change log](https://github.com/a - Container environment variables via `--env`, `--env-file`, and `APPTAINERENV_` host env vars. - `--rocm` to bind ROCm GPU libraries and devices into the container. + - `--nv` to bind Nvidia driver / basic CUDA libraries and devices into + the container. ### New Features & Functionality diff --git a/e2e/gpu/gpu.go b/e2e/gpu/gpu.go index 641a25398b..d9f73374aa 100644 --- a/e2e/gpu/gpu.go +++ b/e2e/gpu/gpu.go @@ -107,6 +107,44 @@ func (c ctx) testNvidiaLegacy(t *testing.T) { } } +func (c ctx) ociTestNvidiaLegacy(t *testing.T) { + require.Nvidia(t) + + imageURL := "docker://ubuntu:20.04" + + // Basic test that we can run the bound in `nvidia-smi` which *should* be on the PATH + tests := []struct { + profile e2e.Profile + args []string + env []string + }{ + { + profile: e2e.OCIUserProfile, + args: []string{"--nv", imageURL, "nvidia-smi"}, + }, + { + profile: e2e.OCIFakerootProfile, + args: []string{"--nv", imageURL, "nvidia-smi"}, + }, + { + profile: e2e.OCIRootProfile, + args: []string{"--nv", imageURL, "nvidia-smi"}, + }, + } + + for _, tt := range tests { + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.profile.String()), + e2e.WithProfile(tt.profile), + e2e.WithCommand("exec"), + e2e.WithArgs(tt.args...), + e2e.WithEnv(tt.env), + e2e.ExpectExit(0), + ) + } +} + func (c ctx) testNvCCLI(t *testing.T) { require.Nvidia(t) require.NvCCLI(t) @@ -602,6 +640,7 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { "build nvccli": c.testBuildNvCCLI, "build rocm": c.testBuildRocm, // oci mode - "oci rocm": c.ociTestRocm, + "oci nvidia": c.ociTestNvidiaLegacy, + "oci rocm": c.ociTestRocm, } } diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index ffd9493ce1..2b7dbe538a 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -112,15 +112,9 @@ func checkOpts(lo launcher.Options) error { badOpt = append(badOpt, "NoMount") } - if lo.Nvidia { - badOpt = append(badOpt, "Nvidia") - } if lo.NvCCLI { badOpt = append(badOpt, "NvCCLI") } - if lo.NoNvidia { - badOpt = append(badOpt, "NoNvidia") - } if len(lo.ContainLibs) > 0 { badOpt = append(badOpt, "ContainLibs") diff --git a/internal/pkg/runtime/launcher/oci/mounts_linux.go b/internal/pkg/runtime/launcher/oci/mounts_linux.go index 11143a0c95..51c6fddbe7 100644 --- a/internal/pkg/runtime/launcher/oci/mounts_linux.go +++ b/internal/pkg/runtime/launcher/oci/mounts_linux.go @@ -48,6 +48,11 @@ func (l *Launcher) getMounts() ([]specs.Mount, error) { return nil, fmt.Errorf("while configuring ROCm mount(s): %w", err) } } + if (l.cfg.Nvidia || l.apptainerConf.AlwaysUseNv) && !l.cfg.NoNvidia { + if err := l.addNvidiaMounts(mounts); err != nil { + return nil, fmt.Errorf("while configuring Nvidia mount(s): %w", err) + } + } return *mounts, nil } @@ -350,3 +355,77 @@ func (l *Launcher) addRocmMounts(mounts *[]specs.Mount) error { return nil } + +func (l *Launcher) addNvidiaMounts(mounts *[]specs.Mount) error { + if l.apptainerConf.UseNvCCLI { + sylog.Warningf("--nvccli not yet supported with --oci. Falling back to legacy --nv support.") + } + + gpuConfFile := filepath.Join(buildcfg.APPTAINER_CONFDIR, "nvliblist.conf") + libs, bins, err := gpu.NvidiaPaths(gpuConfFile) + if err != nil { + sylog.Warningf("While finding Nvidia bind points: %v", err) + } + if len(libs) == 0 { + sylog.Warningf("Could not find any Nvidia libraries on this host!") + } + + ipcs, err := gpu.NvidiaIpcsPath() + if err != nil { + sylog.Warningf("While finding Nvidia IPCs: %v", err) + } + + devs, err := gpu.NvidiaDevices(true) + if err != nil { + sylog.Warningf("While finding Nvidia devices: %v", err) + } + if len(devs) == 0 { + sylog.Warningf("Could not find any ROCm devices on this host!") + } + + for _, binary := range bins { + containerBinary := filepath.Join("/usr/bin", filepath.Base(binary)) + bind := bind.Path{ + Source: binary, + Destination: containerBinary, + Options: map[string]*bind.Option{"ro": {}}, + } + if err := addBindMount(mounts, bind); err != nil { + return err + } + } + + for _, lib := range libs { + containerLib := filepath.Join(containerLibDir, filepath.Base(lib)) + bind := bind.Path{ + Source: lib, + Destination: containerLib, + Options: map[string]*bind.Option{"ro": {}}, + } + if err := addBindMount(mounts, bind); err != nil { + return err + } + } + + for _, ipc := range ipcs { + bind := bind.Path{ + Source: ipc, + Destination: ipc, + } + if err := addBindMount(mounts, bind); err != nil { + return err + } + } + + for _, dev := range devs { + bind := bind.Path{ + Source: dev, + Destination: dev, + } + if err := addDevBindMount(mounts, bind); err != nil { + return err + } + } + + return nil +} From cdcf38e2e9fdcf84bb0410bb48001f1aa5d46a55 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Thu, 22 Dec 2022 10:41:18 +0000 Subject: [PATCH 058/114] chore: refactor runtime spec handling out of bundle Bring all handling of the runtime spec out of the native oci bundle package, up into the oci launcher. We obtain the bundle's image spec after it is downloaded / extracted. The launcher then computes the correct Process config and updates the bundle with it. This is required so that we can handle an image USER in the launcher. Signed-off-by: Edita Kizinevic --- LICENSE_THIRD_PARTY.md | 1 + .../runtime/launcher/oci/launcher_linux.go | 44 ++- .../pkg/runtime/launcher/oci/process_linux.go | 133 ++++++++++ .../launcher/oci/process_linux_test.go | 234 ++++++++++++++++ pkg/ocibundle/bundle.go | 3 + pkg/ocibundle/native/bundle_linux.go | 129 +-------- pkg/ocibundle/native/bundle_linux_test.go | 250 ------------------ pkg/ocibundle/sif/bundle_linux.go | 10 + 8 files changed, 410 insertions(+), 394 deletions(-) diff --git a/LICENSE_THIRD_PARTY.md b/LICENSE_THIRD_PARTY.md index 052d722b85..3f827622e9 100644 --- a/LICENSE_THIRD_PARTY.md +++ b/LICENSE_THIRD_PARTY.md @@ -238,6 +238,7 @@ The source files: * `pkg/sypgp/testdata_test.go` * `internal/pkg/util/user/cgo_lookup_unix.go` +* `internal/pkg/util/passwdfile/passwdfile_unix.go` Contain code from the Go project. diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index 2b7dbe538a..c3d5286e47 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -20,7 +20,6 @@ import ( "os/exec" "path/filepath" "strings" - "syscall" "github.com/apptainer/apptainer/internal/pkg/buildcfg" "github.com/apptainer/apptainer/internal/pkg/cache" @@ -36,7 +35,6 @@ import ( "github.com/containers/image/v5/types" "github.com/google/uuid" "github.com/opencontainers/runtime-spec/specs-go" - "golang.org/x/term" ) var ( @@ -217,19 +215,12 @@ func checkOpts(lo launcher.Options) error { } // createSpec produces an OCI runtime specification, suitable to launch a -// container. This spec excludes ProcessArgs and Env, as these have to be +// container. This spec excludes the Process config, as this have to be // computed where the image config is available, to account for the image's CMD // / ENTRYPOINT / ENV. func (l *Launcher) createSpec() (*specs.Spec, error) { spec := minimalSpec() - // Override the default Process.Terminal to false if our stdin is not a terminal. - if !term.IsTerminal(syscall.Stdin) { - spec.Process.Terminal = false - } - - spec.Process.User = l.getProcessUser() - // If we are *not* requesting fakeroot, then we need to map the container // uid back to host uid, through the initial fakeroot userns. if !l.cfg.Fakeroot && os.Getuid() != 0 { @@ -338,43 +329,38 @@ func (l *Launcher) Exec(ctx context.Context, image string, process string, args } } + // Create OCI runtime spec, excluding the Process settings which must consider the image spec. spec, err := l.createSpec() if err != nil { return fmt.Errorf("while creating OCI spec: %w", err) } - // Assemble the runtime & user-requested environment, which will be merged - // with the image ENV and set in the container at runtime. - rtEnv := defaultEnv(image, bundleDir) - // APPTAINERENV_ has lowest priority - rtEnv = mergeMap(rtEnv, apptainerEnvMap()) - // --env-file can override APPTAINERENV_ - if l.cfg.EnvFile != "" { - e, err := envFileMap(ctx, l.cfg.EnvFile) - if err != nil { - return err - } - rtEnv = mergeMap(rtEnv, e) - } - // --env flag can override --env-file and APPTAINERENV_ - rtEnv = mergeMap(rtEnv, l.cfg.Env) - + // Create a bundle - obtain and extract the image. b, err := native.New( native.OptBundlePath(bundleDir), native.OptImageRef(image), native.OptSysCtx(sysCtx), native.OptImgCache(imgCache), - native.OptProcessArgs(process, args), - native.OptProcessEnv(rtEnv), ) if err != nil { return err } - if err := b.Create(ctx, spec); err != nil { return err } + // With reference to the bundle's image spec, now set the process configuration. + imgSpec := b.ImageSpec() + if imgSpec == nil { + return fmt.Errorf("bundle has no image spec") + } + specProcess, err := l.getProcess(ctx, *imgSpec, image, b.Path(), process, args) + if err != nil { + return err + } + spec.Process = specProcess + b.Update(ctx, spec) + if err := l.updatePasswdGroup(tools.RootFs(b.Path()).Path()); err != nil { return err } diff --git a/internal/pkg/runtime/launcher/oci/process_linux.go b/internal/pkg/runtime/launcher/oci/process_linux.go index 9390935f70..b864a1bcaa 100644 --- a/internal/pkg/runtime/launcher/oci/process_linux.go +++ b/internal/pkg/runtime/launcher/oci/process_linux.go @@ -14,14 +14,84 @@ import ( "fmt" "os" "strings" + "syscall" "github.com/apptainer/apptainer/internal/pkg/fakeroot" + "github.com/apptainer/apptainer/internal/pkg/runtime/engine/config/oci/generate" "github.com/apptainer/apptainer/internal/pkg/util/env" "github.com/apptainer/apptainer/internal/pkg/util/shell/interpreter" "github.com/apptainer/apptainer/internal/pkg/util/user" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/opencontainers/runtime-spec/specs-go" + "golang.org/x/term" ) +const apptainerLibs = "/.singularity.d/libs" + +func (l *Launcher) getProcess(ctx context.Context, imgSpec imgspecv1.Image, image, bundle, process string, args []string) (*specs.Process, error) { + // Assemble the runtime & user-requested environment, which will be merged + // with the image ENV and set in the container at runtime. + rtEnv := defaultEnv(image, bundle) + // APPTAINERENV_ has lowest priority + rtEnv = mergeMap(rtEnv, apptainerEnvMap()) + // --env-file can override APPTAINERENV_ + if l.cfg.EnvFile != "" { + e, err := envFileMap(ctx, l.cfg.EnvFile) + if err != nil { + return nil, err + } + rtEnv = mergeMap(rtEnv, e) + } + // --env flag can override --env-file and APPTAINERENV_ + rtEnv = mergeMap(rtEnv, l.cfg.Env) + + cwd, err := l.getProcessCwd() + if err != nil { + return nil, err + } + + p := specs.Process{ + Args: getProcessArgs(imgSpec, process, args), + Cwd: cwd, + Env: getProcessEnv(imgSpec, rtEnv), + User: l.getProcessUser(), + Terminal: getProcessTerminal(), + } + + return &p, nil +} + +// getProcessTerminal determines whether the container process should run with a terminal. +func getProcessTerminal() bool { + // Override the default Process.Terminal to false if our stdin is not a terminal. + if term.IsTerminal(syscall.Stdin) { + return true + } + return false +} + +// getProcessArgs returns the process args for a container, with reference to the OCI Image Spec. +// The process and image parameters may override the image CMD and/or ENTRYPOINT. +func getProcessArgs(imageSpec imgspecv1.Image, process string, args []string) []string { + var processArgs []string + + if process != "" { + processArgs = []string{process} + } else { + processArgs = imageSpec.Config.Entrypoint + } + + if len(args) > 0 { + processArgs = append(processArgs, args...) + } else { + if process == "" { + processArgs = append(processArgs, imageSpec.Config.Cmd...) + } + } + + return processArgs +} + // getProcessUser computes the uid/gid(s) to be set on process execution. // Currently this only supports the same uid / primary gid as on the host. // TODO - expand for fakeroot, and arbitrary mapped user. @@ -146,6 +216,69 @@ func (l *Launcher) getReverseUserMaps() (uidMap, gidMap []specs.LinuxIDMapping, return uidMap, gidMap, nil } +// getProcessEnv combines the image config ENV with the ENV requested at runtime. +// APPEND_PATH and PREPEND_PATH are honored as with the native apptainer runtime. +// LD_LIBRARY_PATH is modified to always include the apptainer lib bind directory. +func getProcessEnv(imageSpec imgspecv1.Image, runtimeEnv map[string]string) []string { + path := "" + appendPath := "" + prependPath := "" + ldLibraryPath := "" + + // Start with the environment from the image config. + g := generate.New(nil) + g.Config.Process = &specs.Process{Env: imageSpec.Config.Env} + + // Obtain PATH, and LD_LIBRARY_PATH if set in the image config, for special handling. + for _, env := range imageSpec.Config.Env { + e := strings.SplitN(env, "=", 2) + if len(e) < 2 { + continue + } + if e[0] == "PATH" { + path = e[1] + } + if e[0] == "LD_LIBRARY_PATH" { + ldLibraryPath = e[1] + } + } + + // Apply env vars from runtime, except PATH and LD_LIBRARY_PATH releated. + for k, v := range runtimeEnv { + switch k { + case "PATH": + path = v + case "APPEND_PATH": + appendPath = v + case "PREPEND_PATH": + prependPath = v + case "LD_LIBRARY_PATH": + ldLibraryPath = v + default: + g.SetProcessEnv(k, v) + } + } + + // Compute and set optionally APPEND-ed / PREPEND-ed PATH. + if appendPath != "" { + path = path + ":" + appendPath + } + if prependPath != "" { + path = prependPath + ":" + path + } + if path != "" { + g.SetProcessEnv("PATH", path) + } + + // Ensure LD_LIBRARY_PATH always contains apptainer lib binding dir. + if !strings.Contains(ldLibraryPath, apptainerLibs) { + ldLibraryPath = strings.TrimPrefix(ldLibraryPath+":"+apptainerLibs, ":") + } + g.SetProcessEnv("LD_LIBRARY_PATH", ldLibraryPath) + + return g.Config.Process.Env +} + // defaultEnv returns default environment variables set in the container. func defaultEnv(image, bundle string) map[string]string { return map[string]string{ diff --git a/internal/pkg/runtime/launcher/oci/process_linux_test.go b/internal/pkg/runtime/launcher/oci/process_linux_test.go index 9fffa596d3..8163158ea2 100644 --- a/internal/pkg/runtime/launcher/oci/process_linux_test.go +++ b/internal/pkg/runtime/launcher/oci/process_linux_test.go @@ -15,6 +15,8 @@ import ( "path/filepath" "reflect" "testing" + + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" ) func TestApptainerEnvMap(t *testing.T) { @@ -146,3 +148,235 @@ func TestEnvFileMap(t *testing.T) { }) } } + +func TestGetProcessArgs(t *testing.T) { + tests := []struct { + name string + imgEntrypoint []string + imgCmd []string + bundleProcess string + bundleArgs []string + expectProcessArgs []string + }{ + { + name: "imageEntrypointOnly", + imgEntrypoint: []string{"ENTRYPOINT"}, + imgCmd: []string{}, + bundleProcess: "", + bundleArgs: []string{}, + expectProcessArgs: []string{"ENTRYPOINT"}, + }, + { + name: "imageCmdOnly", + imgEntrypoint: []string{}, + imgCmd: []string{"CMD"}, + bundleProcess: "", + bundleArgs: []string{}, + expectProcessArgs: []string{"CMD"}, + }, + { + name: "imageEntrypointCMD", + imgEntrypoint: []string{"ENTRYPOINT"}, + imgCmd: []string{"CMD"}, + bundleProcess: "", + bundleArgs: []string{}, + expectProcessArgs: []string{"ENTRYPOINT", "CMD"}, + }, + { + name: "ProcessOnly", + imgEntrypoint: []string{}, + imgCmd: []string{}, + bundleProcess: "PROCESS", + bundleArgs: []string{}, + expectProcessArgs: []string{"PROCESS"}, + }, + { + name: "ArgsOnly", + imgEntrypoint: []string{}, + imgCmd: []string{}, + bundleProcess: "", + bundleArgs: []string{"ARGS"}, + expectProcessArgs: []string{"ARGS"}, + }, + { + name: "ProcessArgs", + imgEntrypoint: []string{}, + imgCmd: []string{}, + bundleProcess: "PROCESS", + bundleArgs: []string{"ARGS"}, + expectProcessArgs: []string{"PROCESS", "ARGS"}, + }, + { + name: "overrideEntrypointOnlyProcess", + imgEntrypoint: []string{"ENTRYPOINT"}, + imgCmd: []string{}, + bundleProcess: "PROCESS", + bundleArgs: []string{}, + expectProcessArgs: []string{"PROCESS"}, + }, + { + name: "overrideCmdOnlyArgs", + imgEntrypoint: []string{}, + imgCmd: []string{"CMD"}, + bundleProcess: "", + bundleArgs: []string{"ARGS"}, + expectProcessArgs: []string{"ARGS"}, + }, + { + name: "overrideBothProcess", + imgEntrypoint: []string{"ENTRYPOINT"}, + imgCmd: []string{"CMD"}, + bundleProcess: "PROCESS", + bundleArgs: []string{}, + expectProcessArgs: []string{"PROCESS"}, + }, + { + name: "overrideBothArgs", + imgEntrypoint: []string{"ENTRYPOINT"}, + imgCmd: []string{"CMD"}, + bundleProcess: "", + bundleArgs: []string{"ARGS"}, + expectProcessArgs: []string{"ENTRYPOINT", "ARGS"}, + }, + { + name: "overrideBothProcessArgs", + imgEntrypoint: []string{"ENTRYPOINT"}, + imgCmd: []string{"CMD"}, + bundleProcess: "PROCESS", + bundleArgs: []string{"ARGS"}, + expectProcessArgs: []string{"PROCESS", "ARGS"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + i := imgspecv1.Image{ + Config: imgspecv1.ImageConfig{ + Entrypoint: tt.imgEntrypoint, + Cmd: tt.imgCmd, + }, + } + args := getProcessArgs(i, tt.bundleProcess, tt.bundleArgs) + if !reflect.DeepEqual(args, tt.expectProcessArgs) { + t.Errorf("Expected: %v, Got: %v", tt.expectProcessArgs, args) + } + }) + } +} + +func TestGetProcessEnv(t *testing.T) { + tests := []struct { + name string + imageEnv []string + bundleEnv map[string]string + wantEnv []string + }{ + { + name: "Default", + imageEnv: []string{}, + bundleEnv: map[string]string{}, + wantEnv: []string{"LD_LIBRARY_PATH=/.singularity.d/libs"}, + }, + { + name: "ImagePath", + imageEnv: []string{"PATH=/foo"}, + bundleEnv: map[string]string{}, + wantEnv: []string{ + "PATH=/foo", + "LD_LIBRARY_PATH=/.singularity.d/libs", + }, + }, + { + name: "OverridePath", + imageEnv: []string{"PATH=/foo"}, + bundleEnv: map[string]string{"PATH": "/bar"}, + wantEnv: []string{ + "PATH=/bar", + "LD_LIBRARY_PATH=/.singularity.d/libs", + }, + }, + { + name: "AppendPath", + imageEnv: []string{"PATH=/foo"}, + bundleEnv: map[string]string{"APPEND_PATH": "/bar"}, + wantEnv: []string{ + "PATH=/foo:/bar", + "LD_LIBRARY_PATH=/.singularity.d/libs", + }, + }, + { + name: "PrependPath", + imageEnv: []string{"PATH=/foo"}, + bundleEnv: map[string]string{"PREPEND_PATH": "/bar"}, + wantEnv: []string{ + "PATH=/bar:/foo", + "LD_LIBRARY_PATH=/.singularity.d/libs", + }, + }, + { + name: "ImageLdLibraryPath", + imageEnv: []string{"LD_LIBRARY_PATH=/foo"}, + bundleEnv: map[string]string{}, + wantEnv: []string{ + "LD_LIBRARY_PATH=/foo:/.singularity.d/libs", + }, + }, + { + name: "BundleLdLibraryPath", + imageEnv: []string{}, + bundleEnv: map[string]string{"LD_LIBRARY_PATH": "/foo"}, + wantEnv: []string{ + "LD_LIBRARY_PATH=/foo:/.singularity.d/libs", + }, + }, + { + name: "OverrideLdLibraryPath", + imageEnv: []string{"LD_LIBRARY_PATH=/foo"}, + bundleEnv: map[string]string{"LD_LIBRARY_PATH": "/bar"}, + wantEnv: []string{ + "LD_LIBRARY_PATH=/bar:/.singularity.d/libs", + }, + }, + { + name: "ImageVar", + imageEnv: []string{"FOO=bar"}, + bundleEnv: map[string]string{}, + wantEnv: []string{ + "FOO=bar", + "LD_LIBRARY_PATH=/.singularity.d/libs", + }, + }, + { + name: "ImageOverride", + imageEnv: []string{"FOO=bar"}, + bundleEnv: map[string]string{"FOO": "baz"}, + wantEnv: []string{ + "FOO=baz", + "LD_LIBRARY_PATH=/.singularity.d/libs", + }, + }, + { + name: "ImageAdditional", + imageEnv: []string{"FOO=bar"}, + bundleEnv: map[string]string{"ABC": "123"}, + wantEnv: []string{ + "FOO=bar", + "ABC=123", + "LD_LIBRARY_PATH=/.singularity.d/libs", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + imgSpec := imgspecv1.Image{ + Config: imgspecv1.ImageConfig{Env: tt.imageEnv}, + } + + env := getProcessEnv(imgSpec, tt.bundleEnv) + + if !reflect.DeepEqual(env, tt.wantEnv) { + t.Errorf("want: %v, got: %v", tt.wantEnv, env) + } + }) + } +} diff --git a/pkg/ocibundle/bundle.go b/pkg/ocibundle/bundle.go index a1e7dc9631..f96695f3a3 100644 --- a/pkg/ocibundle/bundle.go +++ b/pkg/ocibundle/bundle.go @@ -12,12 +12,15 @@ package ocibundle import ( "context" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/opencontainers/runtime-spec/specs-go" ) // Bundle defines an OCI bundle interface to create/delete OCI bundles type Bundle interface { Create(context.Context, *specs.Spec) error + Update(context.Context, *specs.Spec) error + ImageSpec() *imgspecv1.Image Delete() error Path() string } diff --git a/pkg/ocibundle/native/bundle_linux.go b/pkg/ocibundle/native/bundle_linux.go index 203eb89b65..f848efab18 100644 --- a/pkg/ocibundle/native/bundle_linux.go +++ b/pkg/ocibundle/native/bundle_linux.go @@ -39,8 +39,6 @@ import ( "github.com/opencontainers/umoci/pkg/idtools" ) -const apptainerLibs = "/.singularity.d/libs" - // Bundle is a native OCI bundle, created from imageRef. type Bundle struct { // imageRef is the reference to the OCI image source, e.g. docker://ubuntu:latest. @@ -56,11 +54,6 @@ type Bundle struct { // OCI->SIF conversions, which are not used here. imgCache *cache.Handle // process is the command to execute, which may override the image's ENTRYPOINT / CMD. - process string - // args are the command arguments, which may override the image's CMD. - args []string - // env is the container environment to set, which will be merged with the image's env. - env map[string]string // Generic bundle properties ocibundle.Bundle } @@ -103,23 +96,6 @@ func OptImgCache(ic *cache.Handle) Option { } } -// OptProcessArgs sets the command and arguments to run in the container. -func OptProcessArgs(process string, args []string) Option { - return func(b *Bundle) error { - b.process = process - b.args = args - return nil - } -} - -// OptEnv sets the environment to be set, merged with the image ENV. -func OptProcessEnv(env map[string]string) Option { - return func(b *Bundle) error { - b.env = env - return nil - } -} - // New returns a bundle interface to create/delete an OCI bundle from an OCI image ref. func New(opts ...Option) (ocibundle.Bundle, error) { b := Bundle{ @@ -168,104 +144,27 @@ func (b *Bundle) Create(ctx context.Context, ociConfig *specs.Spec) error { if err := os.RemoveAll(tmpDir); err != nil { return err } - // ProcessArgs are set here, rather than in the launcher spec generation, as we need to - // consult the image Config to handle combining ENTRYPOINT/CMD with user - // provided args. - b.setProcessArgs(g) - // Ditto for environment handling (merge image and user/rt requested). - b.setProcessEnv(g) - return b.writeConfig(g) } -// Path returns the bundle's path on disk. -func (b *Bundle) Path() string { - return b.bundlePath -} - -func (b *Bundle) setProcessArgs(g *generate.Generator) { - var processArgs []string - - if b.process != "" { - processArgs = []string{b.process} - } else { - processArgs = b.imageSpec.Config.Entrypoint - } - - if len(b.args) > 0 { - processArgs = append(processArgs, b.args...) - } else { - if b.process == "" { - processArgs = append(processArgs, b.imageSpec.Config.Cmd...) - } +// Update will update the OCI config for the OCI bundle, so that it is ready for execution. +func (b *Bundle) Update(ctx context.Context, ociConfig *specs.Spec) error { + // generate OCI bundle directory and config + g, err := tools.GenerateBundleConfig(b.bundlePath, ociConfig) + if err != nil { + return fmt.Errorf("failed to generate OCI bundle/config: %s", err) } - - g.SetProcessArgs(processArgs) + return b.writeConfig(g) } -// setProcessEnv combines the image config ENV with the ENV requested in the runtime provided spec. -// APPEND_PATH and PREPEND_PATH are honored as with the native apptainer runtime. -// LD_LIBRARY_PATH is modified to always include the apptainer lib bind directory. -func (b *Bundle) setProcessEnv(g *generate.Generator) { - if g.Config == nil { - g.Config = &specs.Spec{} - } - if g.Config.Process == nil { - g.Config.Process = &specs.Process{} - } - g.Config.Process.Env = b.imageSpec.Config.Env - - path := "" - appendPath := "" - prependPath := "" - ldLibraryPath := "" - - // Obtain PATH, and LD_LIBRARY_PATH if set in the image config. - for _, env := range b.imageSpec.Config.Env { - e := strings.SplitN(env, "=", 2) - if len(e) < 2 { - continue - } - if e[0] == "PATH" { - path = e[1] - } - if e[0] == "LD_LIBRARY_PATH" { - ldLibraryPath = e[1] - } - } - - // Apply env vars from spec, except PATH and LD_LIBRARY_PATH releated. - for k, v := range b.env { - switch k { - case "PATH": - path = v - case "APPEND_PATH": - appendPath = v - case "PREPEND_PATH": - prependPath = v - case "LD_LIBRARY_PATH": - ldLibraryPath = v - default: - g.SetProcessEnv(k, v) - } - } - - // Compute and set optionally APPEND-ed / PREPEND-ed PATH. - if appendPath != "" { - path = path + ":" + appendPath - } - if prependPath != "" { - path = prependPath + ":" + path - } - if path != "" { - g.SetProcessEnv("PATH", path) - } +// ImageSpec returns the OCI image spec associated with the bundle. +func (b *Bundle) ImageSpec() (imgSpec *imgspecv1.Image) { + return b.imageSpec +} - // Ensure LD_LIBRARY_PATH always contains apptainer lib binding dir. - if !strings.Contains(ldLibraryPath, apptainerLibs) { - ldLibraryPath = strings.TrimPrefix(ldLibraryPath+":"+apptainerLibs, ":") - } - g.SetProcessEnv("LD_LIBRARY_PATH", ldLibraryPath) +// Path returns the bundle's path on disk. +func (b *Bundle) Path() string { + return b.bundlePath } func (b *Bundle) writeConfig(g *generate.Generator) error { diff --git a/pkg/ocibundle/native/bundle_linux_test.go b/pkg/ocibundle/native/bundle_linux_test.go index ee29e9f7b9..91e195d407 100644 --- a/pkg/ocibundle/native/bundle_linux_test.go +++ b/pkg/ocibundle/native/bundle_linux_test.go @@ -16,13 +16,9 @@ import ( "net/http" "os" "os/exec" - "reflect" "testing" "github.com/apptainer/apptainer/internal/pkg/cache" - "github.com/apptainer/apptainer/internal/pkg/runtime/engine/config/oci" - "github.com/apptainer/apptainer/internal/pkg/runtime/engine/config/oci/generate" - v1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/opencontainers/runtime-tools/validate" ) @@ -145,249 +141,3 @@ func TestFromImageRef(t *testing.T) { }) } } - -func TestSetProcessArgs(t *testing.T) { - tests := []struct { - name string - imgEntrypoint []string - imgCmd []string - bundleProcess string - bundleArgs []string - expectProcessArgs []string - }{ - { - name: "imageEntrypointOnly", - imgEntrypoint: []string{"ENTRYPOINT"}, - imgCmd: []string{}, - bundleProcess: "", - bundleArgs: []string{}, - expectProcessArgs: []string{"ENTRYPOINT"}, - }, - { - name: "imageCmdOnly", - imgEntrypoint: []string{}, - imgCmd: []string{"CMD"}, - bundleProcess: "", - bundleArgs: []string{}, - expectProcessArgs: []string{"CMD"}, - }, - { - name: "imageEntrypointCMD", - imgEntrypoint: []string{"ENTRYPOINT"}, - imgCmd: []string{"CMD"}, - bundleProcess: "", - bundleArgs: []string{}, - expectProcessArgs: []string{"ENTRYPOINT", "CMD"}, - }, - { - name: "ProcessOnly", - imgEntrypoint: []string{}, - imgCmd: []string{}, - bundleProcess: "PROCESS", - bundleArgs: []string{}, - expectProcessArgs: []string{"PROCESS"}, - }, - { - name: "ArgsOnly", - imgEntrypoint: []string{}, - imgCmd: []string{}, - bundleProcess: "", - bundleArgs: []string{"ARGS"}, - expectProcessArgs: []string{"ARGS"}, - }, - { - name: "ProcessArgs", - imgEntrypoint: []string{}, - imgCmd: []string{}, - bundleProcess: "PROCESS", - bundleArgs: []string{"ARGS"}, - expectProcessArgs: []string{"PROCESS", "ARGS"}, - }, - { - name: "overrideEntrypointOnlyProcess", - imgEntrypoint: []string{"ENTRYPOINT"}, - imgCmd: []string{}, - bundleProcess: "PROCESS", - bundleArgs: []string{}, - expectProcessArgs: []string{"PROCESS"}, - }, - { - name: "overrideCmdOnlyArgs", - imgEntrypoint: []string{}, - imgCmd: []string{"CMD"}, - bundleProcess: "", - bundleArgs: []string{"ARGS"}, - expectProcessArgs: []string{"ARGS"}, - }, - { - name: "overrideBothProcess", - imgEntrypoint: []string{"ENTRYPOINT"}, - imgCmd: []string{"CMD"}, - bundleProcess: "PROCESS", - bundleArgs: []string{}, - expectProcessArgs: []string{"PROCESS"}, - }, - { - name: "overrideBothArgs", - imgEntrypoint: []string{"ENTRYPOINT"}, - imgCmd: []string{"CMD"}, - bundleProcess: "", - bundleArgs: []string{"ARGS"}, - expectProcessArgs: []string{"ENTRYPOINT", "ARGS"}, - }, - { - name: "overrideBothProcessArgs", - imgEntrypoint: []string{"ENTRYPOINT"}, - imgCmd: []string{"CMD"}, - bundleProcess: "PROCESS", - bundleArgs: []string{"ARGS"}, - expectProcessArgs: []string{"PROCESS", "ARGS"}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - b := Bundle{ - imageSpec: &v1.Image{ - Config: v1.ImageConfig{ - Entrypoint: tt.imgEntrypoint, - Cmd: tt.imgCmd, - }, - }, - process: tt.bundleProcess, - args: tt.bundleArgs, - } - - g, err := oci.DefaultConfig() - if err != nil { - t.Fatal(err) - } - b.setProcessArgs(g) - if !reflect.DeepEqual(g.Config.Process.Args, tt.expectProcessArgs) { - t.Errorf("Expected: %v, Got: %v", tt.expectProcessArgs, g.Config.Process.Args) - } - }) - } -} - -func TestSetProcessEnv(t *testing.T) { - tests := []struct { - name string - imageEnv []string - bundleEnv map[string]string - wantEnv []string - }{ - { - name: "Default", - imageEnv: []string{}, - bundleEnv: map[string]string{}, - wantEnv: []string{"LD_LIBRARY_PATH=/.singularity.d/libs"}, - }, - { - name: "ImagePath", - imageEnv: []string{"PATH=/foo"}, - bundleEnv: map[string]string{}, - wantEnv: []string{ - "PATH=/foo", - "LD_LIBRARY_PATH=/.singularity.d/libs", - }, - }, - { - name: "OverridePath", - imageEnv: []string{"PATH=/foo"}, - bundleEnv: map[string]string{"PATH": "/bar"}, - wantEnv: []string{ - "PATH=/bar", - "LD_LIBRARY_PATH=/.singularity.d/libs", - }, - }, - { - name: "AppendPath", - imageEnv: []string{"PATH=/foo"}, - bundleEnv: map[string]string{"APPEND_PATH": "/bar"}, - wantEnv: []string{ - "PATH=/foo:/bar", - "LD_LIBRARY_PATH=/.singularity.d/libs", - }, - }, - { - name: "PrependPath", - imageEnv: []string{"PATH=/foo"}, - bundleEnv: map[string]string{"PREPEND_PATH": "/bar"}, - wantEnv: []string{ - "PATH=/bar:/foo", - "LD_LIBRARY_PATH=/.singularity.d/libs", - }, - }, - { - name: "ImageLdLibraryPath", - imageEnv: []string{"LD_LIBRARY_PATH=/foo"}, - bundleEnv: map[string]string{}, - wantEnv: []string{ - "LD_LIBRARY_PATH=/foo:/.singularity.d/libs", - }, - }, - { - name: "BundleLdLibraryPath", - imageEnv: []string{}, - bundleEnv: map[string]string{"LD_LIBRARY_PATH": "/foo"}, - wantEnv: []string{ - "LD_LIBRARY_PATH=/foo:/.singularity.d/libs", - }, - }, - { - name: "OverrideLdLibraryPath", - imageEnv: []string{"LD_LIBRARY_PATH=/foo"}, - bundleEnv: map[string]string{"LD_LIBRARY_PATH": "/bar"}, - wantEnv: []string{ - "LD_LIBRARY_PATH=/bar:/.singularity.d/libs", - }, - }, - { - name: "ImageVar", - imageEnv: []string{"FOO=bar"}, - bundleEnv: map[string]string{}, - wantEnv: []string{ - "FOO=bar", - "LD_LIBRARY_PATH=/.singularity.d/libs", - }, - }, - { - name: "ImageOverride", - imageEnv: []string{"FOO=bar"}, - bundleEnv: map[string]string{"FOO": "baz"}, - wantEnv: []string{ - "FOO=baz", - "LD_LIBRARY_PATH=/.singularity.d/libs", - }, - }, - { - name: "ImageAdditional", - imageEnv: []string{"FOO=bar"}, - bundleEnv: map[string]string{"ABC": "123"}, - wantEnv: []string{ - "FOO=bar", - "ABC=123", - "LD_LIBRARY_PATH=/.singularity.d/libs", - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - imgSpec := &v1.Image{ - Config: v1.ImageConfig{Env: tt.imageEnv}, - } - - b := &Bundle{ - imageSpec: imgSpec, - env: tt.bundleEnv, - } - g := &generate.Generator{} - b.setProcessEnv(g) - - if !reflect.DeepEqual(g.Config.Process.Env, tt.wantEnv) { - t.Errorf("want: %v, got: %v", tt.wantEnv, g.Config.Process.Env) - } - }) - } -} diff --git a/pkg/ocibundle/sif/bundle_linux.go b/pkg/ocibundle/sif/bundle_linux.go index 5b46feb208..abaa7f45f5 100644 --- a/pkg/ocibundle/sif/bundle_linux.go +++ b/pkg/ocibundle/sif/bundle_linux.go @@ -157,6 +157,11 @@ func (s *sifBundle) Create(ctx context.Context, ociConfig *specs.Spec) error { return nil } +// Update will update the OCI config for the OCI bundle, so that it is ready for execution. +func (s *sifBundle) Update(ctx context.Context, ociConfig *specs.Spec) error { + return fmt.Errorf("cannot update config of a SIF OCI bundle: not implemented") +} + // Delete erases OCI bundle create from SIF image func (s *sifBundle) Delete() error { if s.writable { @@ -173,6 +178,11 @@ func (s *sifBundle) Delete() error { return tools.DeleteBundle(s.bundlePath) } +// ImageSpec returns nil for SIF bundles, as they currently do not carry an OCI image spec. +func (s *sifBundle) ImageSpec() (imgSpec *imageSpecs.Image) { + return nil +} + func (s *sifBundle) Path() string { return s.bundlePath } From 92beb960dc6687901d2e288f3e15bf843668df86 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Thu, 29 Dec 2022 14:07:33 +0000 Subject: [PATCH 059/114] oci: inspection of image user in bundle Signed-off-by: Edita Kizinevic --- .../runtime/launcher/oci/launcher_linux.go | 35 ++-- .../pkg/util/passwdfile/passwdfile_unix.go | 180 ++++++++++++++++++ pkg/ocibundle/tools/oci.go | 18 ++ 3 files changed, 221 insertions(+), 12 deletions(-) create mode 100644 internal/pkg/util/passwdfile/passwdfile_unix.go diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index c3d5286e47..cd54ec366c 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -26,6 +26,7 @@ import ( "github.com/apptainer/apptainer/internal/pkg/runtime/launcher" "github.com/apptainer/apptainer/internal/pkg/util/fs/files" "github.com/apptainer/apptainer/internal/pkg/util/user" + "github.com/apptainer/apptainer/pkg/ocibundle" "github.com/apptainer/apptainer/pkg/ocibundle/native" "github.com/apptainer/apptainer/pkg/ocibundle/tools" "github.com/apptainer/apptainer/pkg/syfs" @@ -249,6 +250,27 @@ func (l *Launcher) createSpec() (*specs.Spec, error) { return &spec, nil } +func (l *Launcher) updateSpecFromImage(ctx context.Context, b ocibundle.Bundle, spec *specs.Spec, image string, process string, args []string) error { + imgSpec := b.ImageSpec() + if imgSpec == nil { + return fmt.Errorf("bundle has no image spec") + } + + specProcess, err := l.getProcess(ctx, *imgSpec, image, b.Path(), process, args) + if err != nil { + return err + } + spec.Process = specProcess + if err := b.Update(ctx, spec); err != nil { + return err + } + + if err := l.updatePasswdGroup(tools.RootFs(b.Path()).Path()); err != nil { + return err + } + return nil +} + func (l *Launcher) updatePasswdGroup(rootfs string) error { uid := os.Getuid() gid := os.Getgid() @@ -350,18 +372,7 @@ func (l *Launcher) Exec(ctx context.Context, image string, process string, args } // With reference to the bundle's image spec, now set the process configuration. - imgSpec := b.ImageSpec() - if imgSpec == nil { - return fmt.Errorf("bundle has no image spec") - } - specProcess, err := l.getProcess(ctx, *imgSpec, image, b.Path(), process, args) - if err != nil { - return err - } - spec.Process = specProcess - b.Update(ctx, spec) - - if err := l.updatePasswdGroup(tools.RootFs(b.Path()).Path()); err != nil { + if err := l.updateSpecFromImage(ctx, b, spec, image, process, args); err != nil { return err } diff --git a/internal/pkg/util/passwdfile/passwdfile_unix.go b/internal/pkg/util/passwdfile/passwdfile_unix.go new file mode 100644 index 0000000000..5f70e2c2d3 --- /dev/null +++ b/internal/pkg/util/passwdfile/passwdfile_unix.go @@ -0,0 +1,180 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. +// +// This source code is an adaptation of: +// https://go.dev/src/os/user/lookup_unix.go +// to provide user lookup functionality against an arbitrary password file. + +package passwdfile + +import ( + "bufio" + "bytes" + "errors" + "io" + "os" + "os/user" + "strconv" + "strings" +) + +var colon = []byte(":") + +// lineFunc returns a value, an error, or (nil, nil) to skip the row. +type lineFunc func(line []byte) (v any, err error) + +// readColonFile parses r as an /etc/group or /etc/passwd style file, running +// fn for each row. readColonFile returns a value, an error, or (nil, nil) if +// the end of the file is reached without a match. +// +// readCols is the minimum number of colon-separated fields that will be passed +// to fn; in a long line additional fields may be silently discarded. +func readColonFile(r io.Reader, fn lineFunc, readCols int) (v any, err error) { + rd := bufio.NewReader(r) + + // Read the file line-by-line. + for { + var isPrefix bool + var wholeLine []byte + + // Read the next line. We do so in chunks (as much as reader's + // buffer is able to keep), check if we read enough columns + // already on each step and store final result in wholeLine. + for { + var line []byte + line, isPrefix, err = rd.ReadLine() + + if err != nil { + // We should return (nil, nil) if EOF is reached + // without a match. + if err == io.EOF { + err = nil + } + return nil, err + } + + // Simple common case: line is short enough to fit in a + // single reader's buffer. + if !isPrefix && len(wholeLine) == 0 { + wholeLine = line + break + } + + wholeLine = append(wholeLine, line...) + + // Check if we read the whole line (or enough columns) + // already. + if !isPrefix || bytes.Count(wholeLine, []byte{':'}) >= readCols { + break + } + } + + // There's no spec for /etc/passwd or /etc/group, but we try to follow + // the same rules as the glibc parser, which allows comments and blank + // space at the beginning of a line. + wholeLine = bytes.TrimSpace(wholeLine) + if len(wholeLine) == 0 || wholeLine[0] == '#' { + continue + } + v, err = fn(wholeLine) + if v != nil || err != nil { + return + } + + // If necessary, skip the rest of the line + for ; isPrefix; _, isPrefix, err = rd.ReadLine() { + if err != nil { + // We should return (nil, nil) if EOF is reached without a match. + if err == io.EOF { + err = nil + } + return nil, err + } + } + } +} + +// returns a *User for a row if that row's has the given value at the +// given index. +func matchUserIndexValue(value string, idx int) lineFunc { + var leadColon string + if idx > 0 { + leadColon = ":" + } + substr := []byte(leadColon + value + ":") + return func(line []byte) (v any, err error) { + if !bytes.Contains(line, substr) || bytes.Count(line, colon) < 6 { + return + } + // kevin:x:1005:1006::/home/kevin:/usr/bin/zsh + parts := strings.SplitN(string(line), ":", 7) + if len(parts) < 6 || parts[idx] != value || parts[0] == "" || + parts[0][0] == '+' || parts[0][0] == '-' { + return + } + if _, err := strconv.Atoi(parts[2]); err != nil { + return nil, nil + } + if _, err := strconv.Atoi(parts[3]); err != nil { + return nil, nil + } + u := &user.User{ + Username: parts[0], + Uid: parts[2], + Gid: parts[3], + Name: parts[4], + HomeDir: parts[5], + } + // The pw_gecos field isn't quite standardized. Some docs + // say: "It is expected to be a comma separated list of + // personal data where the first item is the full name of the + // user." + u.Name, _, _ = strings.Cut(u.Name, ",") + return u, nil + } +} + +func findUserID(uid string, r io.Reader) (*user.User, error) { + i, e := strconv.Atoi(uid) + if e != nil { + return nil, errors.New("user: invalid userid " + uid) + } + if v, err := readColonFile(r, matchUserIndexValue(uid, 2), 6); err != nil { + return nil, err + } else if v != nil { + return v.(*user.User), nil + } + return nil, user.UnknownUserIdError(i) +} + +func findUsername(name string, r io.Reader) (*user.User, error) { + if v, err := readColonFile(r, matchUserIndexValue(name, 0), 6); err != nil { + return nil, err + } else if v != nil { + return v.(*user.User), nil + } + return nil, user.UnknownUserError(name) +} + +func LookupUserInFile(userFile, username string) (*user.User, error) { + f, err := os.Open(userFile) + if err != nil { + return nil, err + } + defer f.Close() + return findUsername(username, f) +} + +func LookupUserIDInFile(userFile, uid string) (*user.User, error) { + f, err := os.Open(userFile) + if err != nil { + return nil, err + } + defer f.Close() + return findUserID(uid, f) +} diff --git a/pkg/ocibundle/tools/oci.go b/pkg/ocibundle/tools/oci.go index 5989c4f041..c1cad97ae4 100644 --- a/pkg/ocibundle/tools/oci.go +++ b/pkg/ocibundle/tools/oci.go @@ -12,11 +12,14 @@ package tools import ( "fmt" "os" + "os/user" "path/filepath" + "strconv" "syscall" "github.com/apptainer/apptainer/internal/pkg/runtime/engine/config/oci" "github.com/apptainer/apptainer/internal/pkg/runtime/engine/config/oci/generate" + "github.com/apptainer/apptainer/internal/pkg/util/passwdfile" "github.com/opencontainers/runtime-spec/specs-go" ) @@ -107,3 +110,18 @@ func DeleteBundle(bundlePath string) error { } return nil } + +// BundleUser returns a user struct for the specified user, from the bundle passwd file. +func Bundle(bundlePath, user string) (u *user.User, err error) { + passwd := filepath.Join(RootFs(bundlePath).Path(), "etc", "passwd") + if _, err := os.Stat(passwd); err != nil { + return nil, fmt.Errorf("cannot access container passwd file: %w", err) + } + + // We have a numeric container uid + if _, err := strconv.Atoi(user); err == nil { + return passwdfile.LookupUserIDInFile(passwd, user) + } + // We have a container username + return passwdfile.LookupUserInFile(passwd, user) +} From 073da5de9b689b419abc9c034ff50824ebc9a005 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Thu, 29 Dec 2022 14:53:18 +0000 Subject: [PATCH 060/114] feat: oci: honor USER in image config When a USER is specified in the image config: * If running unprivileged, ensure the inner uid / gid mapping results in the container process running as the USER, by default. * If running privileged, run as the USER, by default. Fixes sylabs/singularity#77 Signed-off-by: Edita Kizinevic --- e2e/docker/docker.go | 51 ++++++++- .../runtime/launcher/oci/launcher_linux.go | 102 ++++++++++++------ .../pkg/runtime/launcher/oci/process_linux.go | 35 ++---- .../launcher/oci/process_linux_test.go | 2 +- pkg/ocibundle/native/bundle_linux.go | 2 +- pkg/ocibundle/native/bundle_linux_test.go | 2 +- pkg/ocibundle/sif/bundle_linux.go | 2 +- pkg/ocibundle/tools/oci.go | 4 +- 8 files changed, 134 insertions(+), 66 deletions(-) diff --git a/e2e/docker/docker.go b/e2e/docker/docker.go index b4f27e7c8d..8bbbc429a0 100644 --- a/e2e/docker/docker.go +++ b/e2e/docker/docker.go @@ -2,7 +2,7 @@ // Apptainer a Series of LF Projects LLC. // For website terms of use, trademark policy, privacy policy and other // project policies see https://lfprojects.org/policies -// Copyright (c) 2019-2022 Sylabs Inc. All rights reserved. +// Copyright (c) 2019-2023 Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -880,6 +880,54 @@ func (c ctx) testDockerCMDENTRYPOINT(t *testing.T) { } } +// Check that the USER in a docker container is honored under --oci mode +func (c ctx) testDockerUSER(t *testing.T) { + tests := []struct { + name string + expectOutput string + profile e2e.Profile + }{ + // Sanity check apptainer native engine... no support for USER + { + name: "default", + profile: e2e.UserProfile, + expectOutput: fmt.Sprintf("uid=%d(%s) gid=%d", + e2e.UserProfile.ContainerUser(t).UID, + e2e.UserProfile.ContainerUser(t).Name, + e2e.UserProfile.ContainerUser(t).GID), + }, + // `--oci` modes (USER honored by default) + { + name: "OCIUser", + profile: e2e.OCIUserProfile, + expectOutput: `uid=2000(testuser) gid=2000(testgroup)`, + }, + { + name: "OCIFakeroot", + profile: e2e.OCIFakerootProfile, + expectOutput: `uid=0(root) gid=0(root)`, + }, + { + name: "OCIRoot", + profile: e2e.OCIRootProfile, + expectOutput: `uid=2000(testuser) gid=2000(testgroup)`, + }, + } + + for _, tt := range tests { + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(tt.profile), + e2e.WithCommand("run"), + e2e.WithArgs("docker://ghcr.io/apptainer/docker-user"), + e2e.ExpectExit(0, + e2e.ExpectOutput(e2e.ContainMatch, tt.expectOutput), + ), + ) + } +} + // E2ETests is the main func to trigger the test suite func E2ETests(env e2e.TestEnv) testhelper.Tests { c := ctx{ @@ -901,6 +949,7 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { t.Run("entrypoint", c.testDockerENTRYPOINT) t.Run("cmdentrypoint", c.testDockerCMDENTRYPOINT) t.Run("cmd quotes", c.testDockerCMDQuotes) + t.Run("user", c.testDockerUSER) // Regressions t.Run("issue 4524", c.issue4524) }, diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index cd54ec366c..008ac7fa6f 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -2,7 +2,7 @@ // Apptainer a Series of LF Projects LLC. // For website terms of use, trademark policy, privacy policy and other // project policies see https://lfprojects.org/policies -// Copyright (c) 2022, Sylabs Inc. All rights reserved. +// Copyright (c) 2022-2023, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -19,6 +19,7 @@ import ( "os" "os/exec" "path/filepath" + "strconv" "strings" "github.com/apptainer/apptainer/internal/pkg/buildcfg" @@ -215,32 +216,15 @@ func checkOpts(lo launcher.Options) error { return nil } -// createSpec produces an OCI runtime specification, suitable to launch a -// container. This spec excludes the Process config, as this have to be -// computed where the image config is available, to account for the image's CMD -// / ENTRYPOINT / ENV. +// createSpec creates an initial OCI runtime specification, suitable to launch a +// container. This spec excludes the Process config, as this has to be computed +// where the image config is available, to account for the image's CMD / +// ENTRYPOINT / ENV / USER. func (l *Launcher) createSpec() (*specs.Spec, error) { spec := minimalSpec() - // If we are *not* requesting fakeroot, then we need to map the container - // uid back to host uid, through the initial fakeroot userns. - if !l.cfg.Fakeroot && os.Getuid() != 0 { - uidMap, gidMap, err := l.getReverseUserMaps() - if err != nil { - return nil, err - } - spec.Linux.UIDMappings = uidMap - spec.Linux.GIDMappings = gidMap - } - spec = addNamespaces(spec, l.cfg.Namespaces) - cwd, err := l.getProcessCwd() - if err != nil { - return nil, err - } - spec.Process.Cwd = cwd - mounts, err := l.getMounts() if err != nil { return nil, err @@ -250,13 +234,64 @@ func (l *Launcher) createSpec() (*specs.Spec, error) { return &spec, nil } -func (l *Launcher) updateSpecFromImage(ctx context.Context, b ocibundle.Bundle, spec *specs.Spec, image string, process string, args []string) error { +// finalizeSpec updates the bundle config, filling in Process config that depends on the image spec. +func (l *Launcher) finalizeSpec(ctx context.Context, b ocibundle.Bundle, spec *specs.Spec, image string, process string, args []string) (err error) { imgSpec := b.ImageSpec() if imgSpec == nil { return fmt.Errorf("bundle has no image spec") } - specProcess, err := l.getProcess(ctx, *imgSpec, image, b.Path(), process, args) + // In the absence of a USER in the OCI image config, we will run the + // container process as our current user / group. + currentUID := uint32(os.Getuid()) + currentGID := uint32(os.Getgid()) + targetUID := currentUID + targetGID := currentGID + containerUser := false + + // If the OCI image config specifies a USER we will: + // * When unprivileged - run as that user, via nested subuid/gid mappings (host user -> userns root -> OCI USER) + // * When privileged - directly run as that user, as a host uid/gid. + if imgSpec.Config.User != "" { + imgUser, err := tools.BundleUser(b.Path(), imgSpec.Config.User) + if err != nil { + return err + } + imgUID, err := strconv.ParseUint(imgUser.Uid, 10, 32) + if err != nil { + return err + } + imgGID, err := strconv.ParseUint(imgUser.Gid, 10, 32) + if err != nil { + return err + } + targetUID = uint32(imgUID) + targetGID = uint32(imgGID) + containerUser = true + sylog.Debugf("Running as USER specified in OCI image config %d:%d", targetUID, targetGID) + } + + // Fakeroot always overrides to give us root in the container (via userns & idmap if unprivileged). + if l.cfg.Fakeroot { + targetUID = 0 + targetGID = 0 + } + + if targetUID != 0 && currentUID != 0 { + uidMap, gidMap, err := l.getReverseUserMaps(targetUID, targetGID) + if err != nil { + return err + } + spec.Linux.UIDMappings = uidMap + spec.Linux.GIDMappings = gidMap + } + + u := specs.User{ + UID: targetUID, + GID: targetGID, + } + + specProcess, err := l.getProcess(ctx, *imgSpec, image, b.Path(), process, args, u) if err != nil { return err } @@ -265,16 +300,19 @@ func (l *Launcher) updateSpecFromImage(ctx context.Context, b ocibundle.Bundle, return err } - if err := l.updatePasswdGroup(tools.RootFs(b.Path()).Path()); err != nil { + // If we are entering as root, or a USER defined in the container, then passwd/group + // information should be present already. + if targetUID == 0 || containerUser { + return nil + } + // Otherewise, add to the passwd and group files in the container. + if err := l.updatePasswdGroup(tools.RootFs(b.Path()).Path(), targetUID, targetGID); err != nil { return err } return nil } -func (l *Launcher) updatePasswdGroup(rootfs string) error { - uid := os.Getuid() - gid := os.Getgid() - +func (l *Launcher) updatePasswdGroup(rootfs string, uid, gid uint32) error { if os.Getuid() == 0 || l.cfg.Fakeroot { return nil } @@ -288,7 +326,7 @@ func (l *Launcher) updatePasswdGroup(rootfs string) error { } sylog.Debugf("Updating passwd file: %s", containerPasswd) - content, err := files.Passwd(containerPasswd, pw.Dir, uidi, nil) + content, err := files.Passwd(containerPasswd, pw.Dir, int(uid), nil) if err != nil { return fmt.Errorf("while creating passwd file: %w", err) } @@ -297,7 +335,7 @@ func (l *Launcher) updatePasswdGroup(rootfs string) error { } sylog.Debugf("Updating group file: %s", containerGroup) - content, err = files.Group(containerGroup, uid, []int{gid}, nil) + content, err = files.Group(containerGroup, int(uid), []int{int(gid)}, nil) if err != nil { return fmt.Errorf("while creating group file: %w", err) } @@ -372,7 +410,7 @@ func (l *Launcher) Exec(ctx context.Context, image string, process string, args } // With reference to the bundle's image spec, now set the process configuration. - if err := l.updateSpecFromImage(ctx, b, spec, image, process, args); err != nil { + if err := l.finalizeSpec(ctx, b, spec, image, process, args); err != nil { return err } diff --git a/internal/pkg/runtime/launcher/oci/process_linux.go b/internal/pkg/runtime/launcher/oci/process_linux.go index b864a1bcaa..59b268eb9b 100644 --- a/internal/pkg/runtime/launcher/oci/process_linux.go +++ b/internal/pkg/runtime/launcher/oci/process_linux.go @@ -2,7 +2,7 @@ // Apptainer a Series of LF Projects LLC. // For website terms of use, trademark policy, privacy policy and other // project policies see https://lfprojects.org/policies -// Copyright (c) 2022, Sylabs Inc. All rights reserved. +// Copyright (c) 2022-2023, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -28,7 +28,7 @@ import ( const apptainerLibs = "/.singularity.d/libs" -func (l *Launcher) getProcess(ctx context.Context, imgSpec imgspecv1.Image, image, bundle, process string, args []string) (*specs.Process, error) { +func (l *Launcher) getProcess(ctx context.Context, imgSpec imgspecv1.Image, image, bundle, process string, args []string, u specs.User) (*specs.Process, error) { // Assemble the runtime & user-requested environment, which will be merged // with the image ENV and set in the container at runtime. rtEnv := defaultEnv(image, bundle) @@ -54,7 +54,7 @@ func (l *Launcher) getProcess(ctx context.Context, imgSpec imgspecv1.Image, imag Args: getProcessArgs(imgSpec, process, args), Cwd: cwd, Env: getProcessEnv(imgSpec, rtEnv), - User: l.getProcessUser(), + User: u, Terminal: getProcessTerminal(), } @@ -63,11 +63,8 @@ func (l *Launcher) getProcess(ctx context.Context, imgSpec imgspecv1.Image, imag // getProcessTerminal determines whether the container process should run with a terminal. func getProcessTerminal() bool { - // Override the default Process.Terminal to false if our stdin is not a terminal. - if term.IsTerminal(syscall.Stdin) { - return true - } - return false + // Sets the default Process.Terminal to false if our stdin is not a terminal. + return term.IsTerminal(syscall.Stdin) } // getProcessArgs returns the process args for a container, with reference to the OCI Image Spec. @@ -92,22 +89,6 @@ func getProcessArgs(imageSpec imgspecv1.Image, process string, args []string) [] return processArgs } -// getProcessUser computes the uid/gid(s) to be set on process execution. -// Currently this only supports the same uid / primary gid as on the host. -// TODO - expand for fakeroot, and arbitrary mapped user. -func (l *Launcher) getProcessUser() specs.User { - if l.cfg.Fakeroot { - return specs.User{ - UID: 0, - GID: 0, - } - } - return specs.User{ - UID: uint32(os.Getuid()), - GID: uint32(os.Getgid()), - } -} - // getProcessCwd computes the Cwd that the container process should start in. // Currently this is the user's tmpfs home directory (see --containall). func (l *Launcher) getProcessCwd() (dir string, err error) { @@ -122,12 +103,12 @@ func (l *Launcher) getProcessCwd() (dir string, err error) { return pw.Dir, nil } -// getReverseUserMaps returns uid and gid mappings that re-map container uid to host +// getReverseUserMaps returns uid and gid mappings that re-map container uid to target // uid. This 'reverses' the host user to container root mapping in the initial // userns from which the OCI runtime is launched. // -// host 1001 -> fakeroot userns 0 -> container 1001 -func (l *Launcher) getReverseUserMaps() (uidMap, gidMap []specs.LinuxIDMapping, err error) { +// e.g. host 1001 -> fakeroot userns 0 -> container targetUID +func (l *Launcher) getReverseUserMaps(targetUID, targetGID uint32) (uidMap, gidMap []specs.LinuxIDMapping, err error) { uid := uint32(os.Getuid()) gid := uint32(os.Getgid()) // Get user's configured subuid & subgid ranges diff --git a/internal/pkg/runtime/launcher/oci/process_linux_test.go b/internal/pkg/runtime/launcher/oci/process_linux_test.go index 8163158ea2..6c4050e159 100644 --- a/internal/pkg/runtime/launcher/oci/process_linux_test.go +++ b/internal/pkg/runtime/launcher/oci/process_linux_test.go @@ -2,7 +2,7 @@ // Apptainer a Series of LF Projects LLC. // For website terms of use, trademark policy, privacy policy and other // project policies see https://lfprojects.org/policies -// Copyright (c) 2022, Sylabs Inc. All rights reserved. +// Copyright (c) 2022-2023, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. diff --git a/pkg/ocibundle/native/bundle_linux.go b/pkg/ocibundle/native/bundle_linux.go index f848efab18..f3cb82390a 100644 --- a/pkg/ocibundle/native/bundle_linux.go +++ b/pkg/ocibundle/native/bundle_linux.go @@ -2,7 +2,7 @@ // Apptainer a Series of LF Projects LLC. // For website terms of use, trademark policy, privacy policy and other // project policies see https://lfprojects.org/policies -// Copyright (c) 2022, Sylabs Inc. All rights reserved. +// Copyright (c) 2022-2023, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. diff --git a/pkg/ocibundle/native/bundle_linux_test.go b/pkg/ocibundle/native/bundle_linux_test.go index 91e195d407..e52c438c79 100644 --- a/pkg/ocibundle/native/bundle_linux_test.go +++ b/pkg/ocibundle/native/bundle_linux_test.go @@ -2,7 +2,7 @@ // Apptainer a Series of LF Projects LLC. // For website terms of use, trademark policy, privacy policy and other // project policies see https://lfprojects.org/policies -// Copyright (c) 2022, Sylabs Inc. All rights reserved. +// Copyright (c) 2022-2023, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. diff --git a/pkg/ocibundle/sif/bundle_linux.go b/pkg/ocibundle/sif/bundle_linux.go index abaa7f45f5..63e3f8caa6 100644 --- a/pkg/ocibundle/sif/bundle_linux.go +++ b/pkg/ocibundle/sif/bundle_linux.go @@ -2,7 +2,7 @@ // Apptainer a Series of LF Projects LLC. // For website terms of use, trademark policy, privacy policy and other // project policies see https://lfprojects.org/policies -// Copyright (c) 2019-2020, Sylabs Inc. All rights reserved. +// Copyright (c) 2019-2023, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. diff --git a/pkg/ocibundle/tools/oci.go b/pkg/ocibundle/tools/oci.go index c1cad97ae4..74a7b7e247 100644 --- a/pkg/ocibundle/tools/oci.go +++ b/pkg/ocibundle/tools/oci.go @@ -2,7 +2,7 @@ // Apptainer a Series of LF Projects LLC. // For website terms of use, trademark policy, privacy policy and other // project policies see https://lfprojects.org/policies -// Copyright (c) 2019-2020, Sylabs Inc. All rights reserved. +// Copyright (c) 2019-2023, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -112,7 +112,7 @@ func DeleteBundle(bundlePath string) error { } // BundleUser returns a user struct for the specified user, from the bundle passwd file. -func Bundle(bundlePath, user string) (u *user.User, err error) { +func BundleUser(bundlePath, user string) (u *user.User, err error) { passwd := filepath.Join(RootFs(bundlePath).Path(), "etc", "passwd") if _, err := os.Stat(passwd); err != nil { return nil, fmt.Errorf("cannot access container passwd file: %w", err) From 4a03673acddb8482314184a69bc192196fe10b04 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Mon, 6 Feb 2023 15:36:50 +0000 Subject: [PATCH 061/114] fix: lack of passwd / group should be warning in --oci mode In the --oci launcher, When a minimal container doesn't have an `/etc/passwd` or `/etc/groups` file then don't fail with a fatal error. Instead warn like the native runtime does. Fixes sylabs/singularity#1286 Signed-off-by: Edita Kizinevic --- e2e/docker/docker.go | 1 + e2e/docker/regressions.go | 17 +++++++++++++++++ e2e/internal/e2e/profile.go | 12 ++++++++++++ .../pkg/runtime/launcher/oci/launcher_linux.go | 10 ++++------ 4 files changed, 34 insertions(+), 6 deletions(-) diff --git a/e2e/docker/docker.go b/e2e/docker/docker.go index 8bbbc429a0..a262685b10 100644 --- a/e2e/docker/docker.go +++ b/e2e/docker/docker.go @@ -952,6 +952,7 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { t.Run("user", c.testDockerUSER) // Regressions t.Run("issue 4524", c.issue4524) + t.Run("issue 1286", c.issue1286) }, // Tests that are especially slow, or run against a local docker // registry, can be run in parallel, with `--disable-cache` used within diff --git a/e2e/docker/regressions.go b/e2e/docker/regressions.go index ccf410d1d6..4bdee283c7 100644 --- a/e2e/docker/regressions.go +++ b/e2e/docker/regressions.go @@ -210,3 +210,20 @@ func (c ctx) issue1704(t *testing.T) { e2e.ExpectExit(0, e2e.ExpectOutput(e2e.ContainMatch, strings.TrimSpace(defFileContents))), ) } + +// https://github.com/sylabs/singularity/issues/1286 +// Ensure the bare docker://hello-world image runs in all modes +func (c ctx) issue1286(t *testing.T) { + for _, profile := range e2e.AllProfiles() { + c.env.RunApptainer( + t, + e2e.AsSubtest(profile.String()), + e2e.WithProfile(profile), + e2e.WithCommand("run"), + e2e.WithArgs("docker://hello-world"), + e2e.ExpectExit(0, + e2e.ExpectOutput(e2e.ContainMatch, "Hello from Docker!"), + ), + ) + } +} diff --git a/e2e/internal/e2e/profile.go b/e2e/internal/e2e/profile.go index ce1cbce315..61d45fad76 100644 --- a/e2e/internal/e2e/profile.go +++ b/e2e/internal/e2e/profile.go @@ -158,6 +158,18 @@ var OCIProfiles = map[string]Profile{ }, } +// AllProfiles is initialized to the union of NativeProfiles and OCIProfiles +func AllProfiles() map[string]Profile { + ap := map[string]Profile{} + for k, p := range NativeProfiles { + ap[k] = p + } + for k, p := range OCIProfiles { + ap[k] = p + } + return ap +} + // Privileged returns whether the test should be executed with // elevated privileges or not. func (p Profile) Privileged() bool { diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index 008ac7fa6f..2d36105fca 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -328,18 +328,16 @@ func (l *Launcher) updatePasswdGroup(rootfs string, uid, gid uint32) error { sylog.Debugf("Updating passwd file: %s", containerPasswd) content, err := files.Passwd(containerPasswd, pw.Dir, int(uid), nil) if err != nil { - return fmt.Errorf("while creating passwd file: %w", err) - } - if err := os.WriteFile(containerPasswd, content, 0o755); err != nil { + sylog.Warningf("%s", err) + } else if err := os.WriteFile(containerPasswd, content, 0o755); err != nil { return fmt.Errorf("while writing passwd file: %w", err) } sylog.Debugf("Updating group file: %s", containerGroup) content, err = files.Group(containerGroup, int(uid), []int{int(gid)}, nil) if err != nil { - return fmt.Errorf("while creating group file: %w", err) - } - if err := os.WriteFile(containerGroup, content, 0o755); err != nil { + sylog.Warningf("%s", err) + } else if err := os.WriteFile(containerGroup, content, 0o755); err != nil { return fmt.Errorf("while writing passwd file: %w", err) } From a489ef6bfc3033da16cbe597e5fcd7bcf4ea6375 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Fri, 6 Jan 2023 11:21:37 +0000 Subject: [PATCH 062/114] oci: pass in SystemContext for image handling Honour the DOCKER_HOST, `--no-https`, and auth configuration. Fixes sylabs/singularity#1220 Signed-off-by: Edita Kizinevic --- cmd/internal/cli/actions.go | 17 ++++ e2e/docker/docker.go | 90 ++++++++++--------- .../runtime/launcher/oci/launcher_linux.go | 22 ++--- internal/pkg/runtime/launcher/options.go | 15 +++- 4 files changed, 84 insertions(+), 60 deletions(-) diff --git a/cmd/internal/cli/actions.go b/cmd/internal/cli/actions.go index 5fd30b3dc5..814875b3e5 100644 --- a/cmd/internal/cli/actions.go +++ b/cmd/internal/cli/actions.go @@ -29,7 +29,10 @@ import ( ocilauncher "github.com/apptainer/apptainer/internal/pkg/runtime/launcher/oci" "github.com/apptainer/apptainer/internal/pkg/util/env" "github.com/apptainer/apptainer/internal/pkg/util/uri" + "github.com/apptainer/apptainer/pkg/syfs" "github.com/apptainer/apptainer/pkg/sylog" + useragent "github.com/apptainer/apptainer/pkg/util/user-agent" + "github.com/containers/image/v5/types" "github.com/spf13/cobra" ) @@ -410,6 +413,20 @@ func launchContainer(cmd *cobra.Command, image string, containerCmd string, cont if ociRuntime { sylog.Debugf("Using OCI runtime launcher.") + + sysCtx := &types.SystemContext{ + OCIInsecureSkipTLSVerify: noHTTPS, + DockerAuthConfig: &dockerAuthConfig, + DockerDaemonHost: dockerHost, + OSChoice: "linux", + AuthFilePath: syfs.DockerConf(), + DockerRegistryUserAgent: useragent.Value(), + } + if noHTTPS { + sysCtx.DockerInsecureSkipTLSVerify = types.NewOptionalBool(true) + } + opts = append(opts, launcher.OptSysContext(sysCtx)) + l, err = ocilauncher.NewLauncher(opts...) if err != nil { return fmt.Errorf("while configuring container: %s", err) diff --git a/e2e/docker/docker.go b/e2e/docker/docker.go index a262685b10..e75beca7a7 100644 --- a/e2e/docker/docker.go +++ b/e2e/docker/docker.go @@ -208,49 +208,55 @@ func (c ctx) testDockerHost(t *testing.T) { }, } - t.Run("exec", func(t *testing.T) { - for _, tt := range tests { - cmdOps := []e2e.ApptainerCmdOp{ - e2e.WithProfile(e2e.RootProfile), - e2e.AsSubtest(tt.name), - e2e.WithCommand("exec"), - e2e.WithArgs("--disable-cache", dockerURI, "/bin/true"), - e2e.WithEnv(append(os.Environ(), tt.envarName+"="+tt.envarValue)), - e2e.ExpectExit(tt.exit), - } - c.env.RunApptainer(t, cmdOps...) - } - }) - - t.Run("pull", func(t *testing.T) { - for _, tt := range tests { - cmdOps := []e2e.ApptainerCmdOp{ - e2e.WithProfile(e2e.RootProfile), - e2e.AsSubtest(tt.name), - e2e.WithCommand("pull"), - e2e.WithArgs("--force", "--disable-cache", dockerURI), - e2e.WithEnv(append(os.Environ(), tt.envarName+"="+tt.envarValue)), - e2e.WithDir(tmpPath), - e2e.ExpectExit(tt.exit), - } - c.env.RunApptainer(t, cmdOps...) - } - }) + for _, profile := range []e2e.Profile{e2e.RootProfile, e2e.OCIRootProfile} { + t.Run(profile.String(), func(t *testing.T) { + + t.Run("exec", func(t *testing.T) { + for _, tt := range tests { + cmdOps := []e2e.ApptainerCmdOp{ + e2e.WithProfile(profile), + e2e.AsSubtest(profile.String() + "/" + tt.name), + e2e.WithCommand("exec"), + e2e.WithArgs("--disable-cache", dockerURI, "/bin/true"), + e2e.WithEnv(append(os.Environ(), tt.envarName+"="+tt.envarValue)), + e2e.ExpectExit(tt.exit), + } + c.env.RunApptainer(t, cmdOps...) + } + }) + + t.Run("pull", func(t *testing.T) { + for _, tt := range tests { + cmdOps := []e2e.ApptainerCmdOp{ + e2e.WithProfile(profile), + e2e.AsSubtest(tt.name), + e2e.WithCommand("pull"), + e2e.WithArgs("--force", "--disable-cache", dockerURI), + e2e.WithEnv(append(os.Environ(), tt.envarName+"="+tt.envarValue)), + e2e.WithDir(tmpPath), + e2e.ExpectExit(tt.exit), + } + c.env.RunApptainer(t, cmdOps...) + } + }) + + t.Run("build", func(t *testing.T) { + for _, tt := range tests { + cmdOps := []e2e.ApptainerCmdOp{ + e2e.WithProfile(profile), + e2e.AsSubtest(tt.name), + e2e.WithCommand("build"), + e2e.WithArgs("--force", "--disable-cache", "test.sif", dockerURI), + e2e.WithEnv(append(os.Environ(), tt.envarName+"="+tt.envarValue)), + e2e.WithDir(tmpPath), + e2e.ExpectExit(tt.exit), + } + c.env.RunApptainer(t, cmdOps...) + } + }) - t.Run("build", func(t *testing.T) { - for _, tt := range tests { - cmdOps := []e2e.ApptainerCmdOp{ - e2e.WithProfile(e2e.RootProfile), - e2e.AsSubtest(tt.name), - e2e.WithCommand("build"), - e2e.WithArgs("--force", "--disable-cache", "test.sif", dockerURI), - e2e.WithEnv(append(os.Environ(), tt.envarName+"="+tt.envarValue)), - e2e.WithDir(tmpPath), - e2e.ExpectExit(tt.exit), - } - c.env.RunApptainer(t, cmdOps...) - } - }) + }) + } // Clean up docker image e2e.Privileged(func(t *testing.T) { diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index 2d36105fca..b389ae67f9 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -30,11 +30,8 @@ import ( "github.com/apptainer/apptainer/pkg/ocibundle" "github.com/apptainer/apptainer/pkg/ocibundle/native" "github.com/apptainer/apptainer/pkg/ocibundle/tools" - "github.com/apptainer/apptainer/pkg/syfs" "github.com/apptainer/apptainer/pkg/sylog" "github.com/apptainer/apptainer/pkg/util/apptainerconf" - useragent "github.com/apptainer/apptainer/pkg/util/user-agent" - "github.com/containers/image/v5/types" "github.com/google/uuid" "github.com/opencontainers/runtime-spec/specs-go" ) @@ -351,6 +348,10 @@ func (l *Launcher) Exec(ctx context.Context, image string, process string, args return fmt.Errorf("%w: instanceName", ErrNotImplemented) } + if l.cfg.SysContext == nil { + return fmt.Errorf("launcher SysContext must be set for OCI image handling") + } + bundleDir, err := os.MkdirTemp("", "oci-bundle") if err != nil { return nil @@ -364,19 +365,6 @@ func (l *Launcher) Exec(ctx context.Context, image string, process string, args sylog.Debugf("Creating OCI bundle at: %s", bundleDir) - // TODO - propagate auth config - sysCtx := &types.SystemContext{ - // OCIInsecureSkipTLSVerify: cp.b.Opts.NoHTTPS, - // DockerAuthConfig: cp.b.Opts.DockerAuthConfig, - // DockerDaemonHost: cp.b.Opts.DockerDaemonHost, - OSChoice: "linux", - AuthFilePath: syfs.DockerConf(), - DockerRegistryUserAgent: useragent.Value(), - } - // if cp.b.Opts.NoHTTPS { - // cp.sysCtx.DockerInsecureSkipTLSVerify = types.NewOptionalBool(true) - // } - var imgCache *cache.Handle if !l.cfg.CacheDisabled { imgCache, err = cache.New(cache.Config{ @@ -397,7 +385,7 @@ func (l *Launcher) Exec(ctx context.Context, image string, process string, args b, err := native.New( native.OptBundlePath(bundleDir), native.OptImageRef(image), - native.OptSysCtx(sysCtx), + native.OptSysCtx(l.cfg.SysContext), native.OptImgCache(imgCache), ) if err != nil { diff --git a/internal/pkg/runtime/launcher/options.go b/internal/pkg/runtime/launcher/options.go index 1af7785bbc..ef7d2462ae 100644 --- a/internal/pkg/runtime/launcher/options.go +++ b/internal/pkg/runtime/launcher/options.go @@ -11,6 +11,7 @@ package launcher import ( "github.com/apptainer/apptainer/pkg/util/cryptkey" + "github.com/containers/image/v5/types" ) // Namespaces holds flags for the optional (non-mount) namespaces that can be @@ -148,6 +149,10 @@ type Options struct { UseBuildConfig bool TmpDir string Underlay bool // whether prefer underlay over overlay + + // SysContext holds Docker/OCI image handling configuration. + // This will be used by a launcher handling OCI images directly. + SysContext *types.SystemContext } type Option func(co *Options) error @@ -466,7 +471,7 @@ func OptKeyInfo(ki *cryptkey.KeyInfo) Option { } } -// CacheDisabled indicates caching of images was disabled in the CLI. +// OptCacheDisabled indicates caching of images was disabled in the CLI. func OptCacheDisabled(b bool) Option { return func(lo *Options) error { lo.CacheDisabled = b @@ -545,3 +550,11 @@ func OptUnderlay(b bool) Option { return nil } } + +// OptSysContext sets Docker/OCI image handling configuration. +func OptSysContext(sc *types.SystemContext) Option { + return func(lo *Options) error { + lo.SysContext = sc + return nil + } +} From 0137ee3a40cf9aee7ea326b21a5a8058caaac8ed Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Fri, 6 Jan 2023 11:03:16 +0000 Subject: [PATCH 063/114] oci: Allow disabled cache in oci launcher Signed-off-by: Edita Kizinevic --- internal/pkg/runtime/launcher/oci/launcher_linux.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index b389ae67f9..f611f3d71d 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -202,9 +202,6 @@ func checkOpts(lo launcher.Options) error { if lo.SIFFUSE { badOpt = append(badOpt, "SIFFUSE") } - if lo.CacheDisabled { - badOpt = append(badOpt, "CacheDisabled") - } if len(badOpt) > 0 { return fmt.Errorf("%w: %s", ErrUnsupportedOption, strings.Join(badOpt, ",")) From 1b3230f88fc0617789922fe0d26b4f16ffd8a138 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Mon, 9 Jan 2023 11:01:41 +0000 Subject: [PATCH 064/114] fix: e2e: docker rmi ref, not uri Signed-off-by: Edita Kizinevic --- e2e/docker/docker.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/e2e/docker/docker.go b/e2e/docker/docker.go index e75beca7a7..0f2014403d 100644 --- a/e2e/docker/docker.go +++ b/e2e/docker/docker.go @@ -210,7 +210,6 @@ func (c ctx) testDockerHost(t *testing.T) { for _, profile := range []e2e.Profile{e2e.RootProfile, e2e.OCIRootProfile} { t.Run(profile.String(), func(t *testing.T) { - t.Run("exec", func(t *testing.T) { for _, tt := range tests { cmdOps := []e2e.ApptainerCmdOp{ @@ -254,7 +253,6 @@ func (c ctx) testDockerHost(t *testing.T) { c.env.RunApptainer(t, cmdOps...) } }) - }) } From c7ec495ec1b76cda4328c885392ffc1c68a140af Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Mon, 9 Jan 2023 12:18:45 +0000 Subject: [PATCH 065/114] fix: change option for oci launcher unsupported option test Signed-off-by: Edita Kizinevic --- internal/pkg/runtime/launcher/oci/launcher_linux_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux_test.go b/internal/pkg/runtime/launcher/oci/launcher_linux_test.go index 25ff92f791..7549b1be6e 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux_test.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux_test.go @@ -49,7 +49,7 @@ func TestNewLauncher(t *testing.T) { { name: "unsupportedOption", opts: []launcher.Option{ - launcher.OptCacheDisabled(true), + launcher.OptSecurity([]string{"seccomp:example.json"}), }, want: nil, wantErr: true, From fb2ec7dde6cb27c9666b4965753f867a120f3ac2 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Tue, 13 Dec 2022 16:06:06 +0000 Subject: [PATCH 066/114] feat: oci: support cgroups requests in --oci mode Honor the --apply-cgroups and individual cgroups resources flags when running in OCI mode. The launcher instructs runc/crun to create a named cgroup with specified LinuxResources in the config.json. runc/crun must be called with the `--systemd-cgroup` flag when using systemd as cgroup manager. Closes sylabs/singularity#1032 Signed-off-by: Edita Kizinevic --- CHANGELOG.md | 2 + e2e/cgroups/cgroups.go | 103 ++++++++++++++---- e2e/env/oci.go | 3 + e2e/internal/e2e/apptainercmd.go | 14 +++ e2e/internal/e2e/profile.go | 14 +++ internal/app/apptainer/oci_linux.go | 74 +++++++++++-- internal/pkg/cgroups/manager_linux.go | 12 +- internal/pkg/cgroups/util.go | 24 ++++ .../runtime/launcher/oci/launcher_linux.go | 30 ++++- .../runtime/launcher/oci/oci_conmon_linux.go | 8 +- .../runtime/launcher/oci/oci_runc_linux.go | 78 ++++++++----- 11 files changed, 285 insertions(+), 77 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a1aab02582..fff053cdc9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,6 +44,8 @@ For older changes see the [archived Singularity change log](https://github.com/a - `--rocm` to bind ROCm GPU libraries and devices into the container. - `--nv` to bind Nvidia driver / basic CUDA libraries and devices into the container. + - `--apply-cgroups`, and the `--cpu*`, `--blkio*`, `--memory*`, + `--pids-limit` flags to apply resource limits. ### New Features & Functionality diff --git a/e2e/cgroups/cgroups.go b/e2e/cgroups/cgroups.go index 2538c3106f..645511159c 100644 --- a/e2e/cgroups/cgroups.go +++ b/e2e/cgroups/cgroups.go @@ -255,9 +255,7 @@ func (c *ctx) instanceStatsRootless(t *testing.T) { c.instanceStats(t, e2e.UserProfile) } -func (c *ctx) actionApply(t *testing.T, profile e2e.Profile) { - e2e.EnsureImage(t, c.env) - +func (c *ctx) actionApply(t *testing.T, profile e2e.Profile, imageRef string) { tests := []struct { name string args []string @@ -265,64 +263,92 @@ func (c *ctx) actionApply(t *testing.T, profile e2e.Profile) { expectErrorOut string rootfull bool rootless bool + skipOCI bool + onlyOCI bool }{ { name: "nonexistent toml", - args: []string{"--apply-cgroups", "testdata/cgroups/doesnotexist.toml", c.env.ImagePath, "/bin/sleep", "5"}, + args: []string{"--apply-cgroups", "testdata/cgroups/doesnotexist.toml", imageRef, "/bin/sleep", "5"}, expectErrorCode: 255, expectErrorOut: "no such file or directory", rootfull: true, rootless: true, + skipOCI: false, + onlyOCI: false, }, { name: "invalid toml", - args: []string{"--apply-cgroups", "testdata/cgroups/invalid.toml", c.env.ImagePath, "/bin/sleep", "5"}, + args: []string{"--apply-cgroups", "testdata/cgroups/invalid.toml", imageRef, "/bin/sleep", "5"}, expectErrorCode: 255, expectErrorOut: "toml: expected character", rootfull: true, rootless: true, + skipOCI: false, + onlyOCI: false, }, { name: "memory limit", - args: []string{"--apply-cgroups", "testdata/cgroups/memory_limit.toml", c.env.ImagePath, "/bin/sleep", "5"}, + args: []string{"--apply-cgroups", "testdata/cgroups/memory_limit.toml", imageRef, "/bin/sleep", "5"}, expectErrorCode: 137, rootfull: true, rootless: true, + skipOCI: true, + onlyOCI: false, + }, + { + name: "memory limit oci", + args: []string{"--apply-cgroups", "testdata/cgroups/memory_limit.toml", imageRef, "/bin/sleep", "5"}, + // crun returns a 1 when the OOM kill happens. + expectErrorCode: 1, + rootfull: true, + rootless: true, + skipOCI: false, + onlyOCI: true, }, { name: "cpu success", - args: []string{"--apply-cgroups", "testdata/cgroups/cpu_success.toml", c.env.ImagePath, "/bin/true"}, + args: []string{"--apply-cgroups", "testdata/cgroups/cpu_success.toml", imageRef, "/bin/true"}, expectErrorCode: 0, rootfull: true, // This currently fails in the e2e scenario due to the way we are using a mount namespace. // It *does* work if you test it, directly calling the apptainer CLI. // Reason is believed to be: https://github.com/opencontainers/runc/issues/3026 rootless: false, + skipOCI: false, + onlyOCI: false, }, // Device access is allowed by default. { name: "device allow default", - args: []string{"--apply-cgroups", "testdata/cgroups/null.toml", c.env.ImagePath, "cat", "/dev/null"}, + args: []string{"--apply-cgroups", "testdata/cgroups/null.toml", imageRef, "cat", "/dev/null"}, expectErrorCode: 0, rootfull: true, rootless: true, + skipOCI: false, + onlyOCI: false, }, // Device limits are properly applied only in rootful mode. Rootless will ignore them with a warning. { name: "device deny", - args: []string{"--apply-cgroups", "testdata/cgroups/deny_device.toml", c.env.ImagePath, "cat", "/dev/null"}, + args: []string{"--apply-cgroups", "testdata/cgroups/deny_device.toml", imageRef, "cat", "/dev/null"}, expectErrorCode: 1, expectErrorOut: "Operation not permitted", rootfull: true, rootless: false, + // runc/crun always allow /dev/null access + skipOCI: true, + onlyOCI: false, }, { name: "device ignored", - args: []string{"--apply-cgroups", "testdata/cgroups/deny_device.toml", c.env.ImagePath, "cat", "/dev/null"}, + args: []string{"--apply-cgroups", "testdata/cgroups/deny_device.toml", imageRef, "cat", "/dev/null"}, expectErrorCode: 0, expectErrorOut: "Device limits will not be applied with rootless cgroups", rootfull: false, rootless: true, + // runc/crun silently ignore in rootless + skipOCI: true, + onlyOCI: false, }, } @@ -334,6 +360,13 @@ func (c *ctx) actionApply(t *testing.T, profile e2e.Profile) { if !profile.Privileged() && !tt.rootless { t.Skip() } + if profile.OCI() && tt.skipOCI { + t.Skip() + } + if !profile.OCI() && tt.onlyOCI { + t.Skip() + } + exitFunc := []e2e.ApptainerCmdResultOp{} if tt.expectErrorOut != "" { exitFunc = []e2e.ApptainerCmdResultOp{e2e.ExpectError(e2e.ContainMatch, tt.expectErrorOut)} @@ -350,13 +383,27 @@ func (c *ctx) actionApply(t *testing.T, profile e2e.Profile) { } func (c *ctx) actionApplyRoot(t *testing.T) { - c.actionApply(t, e2e.RootProfile) + e2e.EnsureImage(t, c.env) + e2e.EnsureOCIImage(t, c.env) + t.Run(e2e.RootProfile.String(), func(t *testing.T) { + c.actionApply(t, e2e.RootProfile, c.env.ImagePath) + }) + t.Run(e2e.OCIRootProfile.String(), func(t *testing.T) { + c.actionApply(t, e2e.OCIRootProfile, "oci-archive:"+c.env.OCIImagePath) + }) } func (c *ctx) actionApplyRootless(t *testing.T) { + e2e.EnsureImage(t, c.env) + e2e.EnsureOCIImage(t, c.env) for _, profile := range []e2e.Profile{e2e.UserProfile, e2e.UserNamespaceProfile, e2e.FakerootProfile} { t.Run(profile.String(), func(t *testing.T) { - c.actionApply(t, profile) + c.actionApply(t, profile, c.env.ImagePath) + }) + } + for _, profile := range []e2e.Profile{e2e.OCIUserProfile, e2e.OCIFakerootProfile} { + t.Run(profile.String(), func(t *testing.T) { + c.actionApply(t, profile, "oci-archive:"+c.env.OCIImagePath) }) } } @@ -499,21 +546,21 @@ var resourceFlagTests = []resourceFlagTest{ }, } -func (c *ctx) actionFlags(t *testing.T, profile e2e.Profile) { +func (c *ctx) actionFlags(t *testing.T, profile e2e.Profile, imageRef string) { e2e.EnsureImage(t, c.env) for _, tt := range resourceFlagTests { t.Run(tt.name, func(t *testing.T) { if cgroups.IsCgroup2UnifiedMode() { - c.actionFlagV2(t, tt, profile) + c.actionFlagV2(t, tt, profile, imageRef) return } - c.actionFlagV1(t, tt, profile) + c.actionFlagV1(t, tt, profile, imageRef) }) } } -func (c *ctx) actionFlagV1(t *testing.T, tt resourceFlagTest, profile e2e.Profile) { +func (c *ctx) actionFlagV1(t *testing.T, tt resourceFlagTest, profile e2e.Profile, imageRef string) { // Don't try to test a resource that doesn't exist in our caller cgroup. // E.g. some systems don't have memory.memswp, and might not have blkio.bfq require.CgroupsResourceExists(t, tt.controllerV1, tt.resourceV1) @@ -530,7 +577,7 @@ func (c *ctx) actionFlagV1(t *testing.T, tt resourceFlagTest, profile e2e.Profil } args := tt.args - args = append(args, "-B", "/sys/fs/cgroup", c.env.ImagePath, "/bin/sh", "-c", shellCmd) + args = append(args, "-B", "/sys/fs/cgroup", imageRef, "/bin/sh", "-c", shellCmd) c.env.RunApptainer( t, @@ -541,7 +588,7 @@ func (c *ctx) actionFlagV1(t *testing.T, tt resourceFlagTest, profile e2e.Profil ) } -func (c *ctx) actionFlagV2(t *testing.T, tt resourceFlagTest, profile e2e.Profile) { +func (c *ctx) actionFlagV2(t *testing.T, tt resourceFlagTest, profile e2e.Profile, imageRef string) { if tt.skipV2 { t.Skip() } @@ -566,7 +613,7 @@ func (c *ctx) actionFlagV2(t *testing.T, tt resourceFlagTest, profile e2e.Profil shellCmd := fmt.Sprintf("cat /sys/fs/cgroup$(cat /proc/self/cgroup | grep '^0::' | cut -d ':' -f 3)/%s", tt.resourceV2) args := tt.args - args = append(args, "-B", "/sys/fs/cgroup", c.env.ImagePath, "/bin/sh", "-c", shellCmd) + args = append(args, "-B", "/sys/fs/cgroup", imageRef, "/bin/sh", "-c", shellCmd) c.env.RunApptainer( t, @@ -578,13 +625,27 @@ func (c *ctx) actionFlagV2(t *testing.T, tt resourceFlagTest, profile e2e.Profil } func (c *ctx) actionFlagsRoot(t *testing.T) { - c.actionFlags(t, e2e.RootProfile) + e2e.EnsureImage(t, c.env) + e2e.EnsureOCIImage(t, c.env) + t.Run(e2e.RootProfile.String(), func(t *testing.T) { + c.actionFlags(t, e2e.RootProfile, c.env.ImagePath) + }) + t.Run(e2e.OCIRootProfile.String(), func(t *testing.T) { + c.actionFlags(t, e2e.OCIRootProfile, "oci-archive:"+c.env.OCIImagePath) + }) } func (c *ctx) actionFlagsRootless(t *testing.T) { + e2e.EnsureImage(t, c.env) + e2e.EnsureOCIImage(t, c.env) for _, profile := range []e2e.Profile{e2e.UserProfile, e2e.UserNamespaceProfile, e2e.FakerootProfile} { t.Run(profile.String(), func(t *testing.T) { - c.actionFlags(t, profile) + c.actionFlags(t, profile, c.env.ImagePath) + }) + } + for _, profile := range []e2e.Profile{e2e.OCIUserProfile, e2e.OCIFakerootProfile} { + t.Run(profile.String(), func(t *testing.T) { + c.actionFlags(t, profile, "oci-archive:"+c.env.OCIImagePath) }) } } diff --git a/e2e/env/oci.go b/e2e/env/oci.go index 444d8128fb..7632b52a85 100644 --- a/e2e/env/oci.go +++ b/e2e/env/oci.go @@ -76,6 +76,7 @@ func (c ctx) ociApptainerEnv(t *testing.T) { e2e.WithProfile(e2e.OCIUserProfile), e2e.WithCommand("exec"), e2e.WithEnv(tt.env), + e2e.WithRootlessEnv(), e2e.WithArgs(tt.image, "/bin/sh", "-c", "echo $PATH"), e2e.ExpectExit( 0, @@ -185,6 +186,7 @@ func (c ctx) ociEnvOption(t *testing.T) { e2e.WithProfile(e2e.OCIUserProfile), e2e.WithCommand("exec"), e2e.WithEnv(tt.hostEnv), + e2e.WithRootlessEnv(), e2e.WithArgs(args...), e2e.ExpectExit( 0, @@ -287,6 +289,7 @@ func (c ctx) ociEnvFile(t *testing.T) { e2e.WithProfile(e2e.OCIUserProfile), e2e.WithCommand("exec"), e2e.WithEnv(tt.hostEnv), + e2e.WithRootlessEnv(), e2e.WithArgs(args...), e2e.ExpectExit( 0, diff --git a/e2e/internal/e2e/apptainercmd.go b/e2e/internal/e2e/apptainercmd.go index 47af2c1dfa..e08bd729c3 100644 --- a/e2e/internal/e2e/apptainercmd.go +++ b/e2e/internal/e2e/apptainercmd.go @@ -535,6 +535,20 @@ func (env TestEnv) RunApptainer(t *testing.T, cmdOps ...ApptainerCmdOp) { cmd.Env = os.Environ() } + // Clear user-specific DBUS / XDG vars when we are using a priv profile, + // as they don't make sense for the root user... and wouldn't be set in a + // real root user session. + if privileged { + i := 0 + for _, e := range cmd.Env { + if !(strings.HasPrefix(e, "DBUS_SESSION_BUS_ADDRESS=") || strings.HasPrefix(e, "XDG_RUNTIME_DIR=")) { + cmd.Env[i] = e + i++ + } + } + cmd.Env = cmd.Env[:i] + } + // By default, each E2E command shares a temporary image cache // directory. If a test is directly testing the cache, or depends on // specific ordered cache behavior then diff --git a/e2e/internal/e2e/profile.go b/e2e/internal/e2e/profile.go index 61d45fad76..930c86e782 100644 --- a/e2e/internal/e2e/profile.go +++ b/e2e/internal/e2e/profile.go @@ -68,6 +68,7 @@ type Profile struct { requirementsFn func(*testing.T) // function checking requirements for the profile apptainerOption string // option added to apptainer command for the profile optionForCommands []string // apptainer commands concerned by the option to be added + oci bool // whether the profile uses the OCI low-level runtime } // NativeProfiles defines all available profiles for the native apptainer runtime @@ -81,6 +82,7 @@ var NativeProfiles = map[string]Profile{ requirementsFn: nil, apptainerOption: "", optionForCommands: []string{}, + oci: false, }, rootProfile: { name: "Root", @@ -91,6 +93,7 @@ var NativeProfiles = map[string]Profile{ requirementsFn: nil, apptainerOption: "", optionForCommands: []string{}, + oci: false, }, fakerootProfile: { name: "Fakeroot", @@ -101,6 +104,7 @@ var NativeProfiles = map[string]Profile{ requirementsFn: fakerootRequirements, apptainerOption: "--fakeroot", optionForCommands: []string{"shell", "exec", "run", "test", "instance start", "build"}, + oci: false, }, userNamespaceProfile: { name: "UserNamespace", @@ -111,6 +115,7 @@ var NativeProfiles = map[string]Profile{ requirementsFn: require.UserNamespace, apptainerOption: "--userns", optionForCommands: []string{"shell", "exec", "run", "test", "instance start"}, + oci: false, }, rootUserNamespaceProfile: { name: "RootUserNamespace", @@ -121,6 +126,7 @@ var NativeProfiles = map[string]Profile{ requirementsFn: require.UserNamespace, apptainerOption: "--userns", optionForCommands: []string{"shell", "exec", "run", "test", "instance start"}, + oci: false, }, } @@ -135,6 +141,7 @@ var OCIProfiles = map[string]Profile{ requirementsFn: ociRequirements, apptainerOption: "--oci", optionForCommands: []string{"shell", "exec", "run", "test", "instance start"}, + oci: true, }, ociRootProfile: { name: "OCIRoot", @@ -145,6 +152,7 @@ var OCIProfiles = map[string]Profile{ requirementsFn: ociRequirements, apptainerOption: "--oci", optionForCommands: []string{"shell", "exec", "run", "test", "instance start"}, + oci: true, }, ociFakerootProfile: { name: "OCIFakeroot", @@ -155,6 +163,7 @@ var OCIProfiles = map[string]Profile{ requirementsFn: ociRequirements, apptainerOption: "--oci --fakeroot", optionForCommands: []string{"shell", "exec", "run", "test", "instance start"}, + oci: true, }, } @@ -176,6 +185,11 @@ func (p Profile) Privileged() bool { return p.privileged } +// OCI returns whether the profile is using an OCI runtime, rather than the apptainer native runtime. +func (p Profile) OCI() bool { + return p.oci +} + // Requirements calls the different require.* functions // necessary for running an E2E test under this profile. func (p Profile) Requirements(t *testing.T) { diff --git a/internal/app/apptainer/oci_linux.go b/internal/app/apptainer/oci_linux.go index 2e1732a202..d102aee784 100644 --- a/internal/app/apptainer/oci_linux.go +++ b/internal/app/apptainer/oci_linux.go @@ -14,9 +14,12 @@ package apptainer import ( "context" + "fmt" + "github.com/apptainer/apptainer/internal/pkg/buildcfg" "github.com/apptainer/apptainer/internal/pkg/runtime/launcher/oci" ocibundle "github.com/apptainer/apptainer/pkg/ocibundle/sif" + "github.com/apptainer/apptainer/pkg/util/apptainerconf" ) // OciArgs contains CLI arguments @@ -34,52 +37,92 @@ type OciArgs struct { // OciRun runs a container (equivalent to create/start/delete) func OciRun(ctx context.Context, containerID string, args *OciArgs) error { - return oci.Run(ctx, containerID, args.BundlePath, args.PidFile) + systemdCgroups, err := systemdCgroups() + if err != nil { + return err + } + return oci.Run(ctx, containerID, args.BundlePath, args.PidFile, systemdCgroups) } // OciCreate creates a container from an OCI bundle func OciCreate(containerID string, args *OciArgs) error { - return oci.Create(containerID, args.BundlePath) + systemdCgroups, err := systemdCgroups() + if err != nil { + return err + } + return oci.Create(containerID, args.BundlePath, systemdCgroups) } // OciStart starts a previously create container func OciStart(containerID string) error { - return oci.Start(containerID) + systemdCgroups, err := systemdCgroups() + if err != nil { + return err + } + return oci.Start(containerID, systemdCgroups) } // OciDelete deletes container resources func OciDelete(ctx context.Context, containerID string) error { - return oci.Delete(ctx, containerID) + systemdCgroups, err := systemdCgroups() + if err != nil { + return err + } + return oci.Delete(ctx, containerID, systemdCgroups) } // OciExec executes a command in a container func OciExec(containerID string, cmdArgs []string) error { - return oci.Exec(containerID, cmdArgs) + systemdCgroups, err := systemdCgroups() + if err != nil { + return err + } + return oci.Exec(containerID, cmdArgs, systemdCgroups) } // OciKill kills container process func OciKill(containerID string, killSignal string) error { - return oci.Kill(containerID, killSignal) + systemdCgroups, err := systemdCgroups() + if err != nil { + return err + } + return oci.Kill(containerID, killSignal, systemdCgroups) } // OciPause pauses processes in a container func OciPause(containerID string) error { - return oci.Pause(containerID) + systemdCgroups, err := systemdCgroups() + if err != nil { + return err + } + return oci.Pause(containerID, systemdCgroups) } // OciResume pauses processes in a container func OciResume(containerID string) error { - return oci.Resume(containerID) + systemdCgroups, err := systemdCgroups() + if err != nil { + return err + } + return oci.Resume(containerID, systemdCgroups) } // OciState queries container state func OciState(containerID string, args *OciArgs) error { - return oci.State(containerID) + systemdCgroups, err := systemdCgroups() + if err != nil { + return err + } + return oci.State(containerID, systemdCgroups) } // OciUpdate updates container cgroups resources func OciUpdate(containerID string, args *OciArgs) error { - return oci.Update(containerID, args.FromFile) + systemdCgroups, err := systemdCgroups() + if err != nil { + return err + } + return oci.Update(containerID, args.FromFile, systemdCgroups) } // OciMount mount a SIF image to create an OCI bundle @@ -99,3 +142,14 @@ func OciUmount(bundle string) error { } return d.Delete() } + +func systemdCgroups() (use bool, err error) { + cfg := apptainerconf.GetCurrentConfig() + if cfg == nil { + cfg, err = apptainerconf.Parse(buildcfg.APPTAINER_CONF_FILE) + if err != nil { + return false, fmt.Errorf("unable to parse apptainer configuration file: %w", err) + } + } + return cfg.SystemdCgroups, nil +} diff --git a/internal/pkg/cgroups/manager_linux.go b/internal/pkg/cgroups/manager_linux.go index e630b93b6e..8dc9717c9e 100644 --- a/internal/pkg/cgroups/manager_linux.go +++ b/internal/pkg/cgroups/manager_linux.go @@ -14,7 +14,6 @@ import ( "fmt" "os" "path/filepath" - "strconv" "strings" "github.com/apptainer/apptainer/internal/pkg/util/env" @@ -344,15 +343,8 @@ func NewManagerWithSpec(spec *specs.LinuxResources, pid int, group string, syste if pid == 0 { return nil, fmt.Errorf("a pid is required to create a new cgroup") } - if group == "" && !systemd { - group = filepath.Join("/apptainer", strconv.Itoa(pid)) - } - if group == "" && systemd { - if os.Getuid() == 0 { - group = "system.slice:apptainer:" + strconv.Itoa(pid) - } else { - group = "user.slice:apptainer:" + strconv.Itoa(pid) - } + if group == "" { + group = DefaultPathForPid(systemd, pid) } sylog.Debugf("Creating cgroups manager for %s", group) diff --git a/internal/pkg/cgroups/util.go b/internal/pkg/cgroups/util.go index 2c27fdcf82..96ddc7a340 100644 --- a/internal/pkg/cgroups/util.go +++ b/internal/pkg/cgroups/util.go @@ -11,6 +11,9 @@ package cgroups import ( "fmt" + "os" + "path/filepath" + "strconv" lccgroups "github.com/opencontainers/runc/libcontainer/cgroups" ) @@ -51,3 +54,24 @@ func pidToPath(pid int) (path string, err error) { } return path, nil } + +// DefaultPathForPid returns a default cgroup path for a given PID. +func DefaultPathForPid(systemd bool, pid int) (group string) { + // Default naming is pid of first process added to cgroup. + strPid := strconv.Itoa(pid) + // Request is for an empty cgroup... name it for the requestor's (our) pid. + if pid == -1 { + strPid = "parent-" + strconv.Itoa(os.Getpid()) + } + + if systemd { + if os.Getuid() == 0 { + group = "system.slice:apptainer:" + strPid + } else { + group = "user.slice:apptainer:" + strPid + } + } else { + group = filepath.Join("/apptainer", strPid) + } + return group +} diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index f611f3d71d..5489b43e1a 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -24,6 +24,7 @@ import ( "github.com/apptainer/apptainer/internal/pkg/buildcfg" "github.com/apptainer/apptainer/internal/pkg/cache" + "github.com/apptainer/apptainer/internal/pkg/cgroups" "github.com/apptainer/apptainer/internal/pkg/runtime/launcher" "github.com/apptainer/apptainer/internal/pkg/util/fs/files" "github.com/apptainer/apptainer/internal/pkg/util/user" @@ -162,10 +163,6 @@ func checkOpts(lo launcher.Options) error { badOpt = append(badOpt, "NoUmask") } - if lo.CGroupsJSON != "" { - badOpt = append(badOpt, "CGroupsJSON") - } - // ConfigFile always set by CLI. We should support only the default from build time. if lo.ConfigFile != "" && lo.ConfigFile != buildcfg.APPTAINER_CONF_FILE { badOpt = append(badOpt, "ConfigFile") @@ -225,6 +222,15 @@ func (l *Launcher) createSpec() (*specs.Spec, error) { } spec.Mounts = mounts + cgPath, resources, err := l.getCgroup() + if err != nil { + return nil, err + } + if cgPath != "" { + spec.Linux.CgroupsPath = cgPath + spec.Linux.Resources = resources + } + return &spec, nil } @@ -404,9 +410,10 @@ func (l *Launcher) Exec(ctx context.Context, image string, process string, args if os.Getuid() == 0 { // Direct execution of runc/crun run. - err = Run(ctx, id.String(), b.Path(), "") + err = Run(ctx, id.String(), b.Path(), "", l.apptainerConf.SystemdCgroups) } else { // Reexec apptainer oci run in a userns with mappings. + // Note - the oci run command will pull out the SystemdCgroups setting from config. err = RunNS(ctx, id.String(), b.Path(), "") } var exitErr *exec.ExitError @@ -416,6 +423,19 @@ func (l *Launcher) Exec(ctx context.Context, image string, process string, args return err } +// getCgroup will return a cgroup path and resources for the runtime to create. +func (l *Launcher) getCgroup() (path string, resources *specs.LinuxResources, err error) { + if l.cfg.CGroupsJSON == "" { + return "", nil, nil + } + path = cgroups.DefaultPathForPid(l.apptainerConf.SystemdCgroups, -1) + resources, err = cgroups.UnmarshalJSONResources(l.cfg.CGroupsJSON) + if err != nil { + return "", nil, err + } + return path, resources, nil +} + func mergeMap(a map[string]string, b map[string]string) map[string]string { for k, v := range b { a[k] = v diff --git a/internal/pkg/runtime/launcher/oci/oci_conmon_linux.go b/internal/pkg/runtime/launcher/oci/oci_conmon_linux.go index ba69214991..2b3a9c261c 100644 --- a/internal/pkg/runtime/launcher/oci/oci_conmon_linux.go +++ b/internal/pkg/runtime/launcher/oci/oci_conmon_linux.go @@ -38,7 +38,7 @@ type ociError struct { } // Create creates a container from an OCI bundle -func Create(containerID, bundlePath string) error { +func Create(containerID, bundlePath string, systemdCgroups bool) error { conmon, err := bin.FindBin("conmon") if err != nil { return err @@ -120,6 +120,10 @@ func Create(containerID, bundlePath string) error { "--exit-command-arg", containerID, } + if systemdCgroups { + cmdArgs = append(cmdArgs, "--systemd-cgroup") + } + cmd := exec.Command(conmon, cmdArgs...) cmd.Dir = absBundle cmd.Env = append(cmd.Env, fmt.Sprintf("_OCI_SYNCPIPE=%d", 3), fmt.Sprintf("_OCI_STARTPIPE=%d", 4)) @@ -157,7 +161,7 @@ func Create(containerID, bundlePath string) error { // We check for errors from runc (which conmon invokes) via the sync pipe pid, err := readConmonPipeData(syncParent, path.Join(sd, runcLogFile)) if err != nil { - if err2 := Delete(context.TODO(), containerID); err2 != nil { + if err2 := Delete(context.TODO(), containerID, systemdCgroups); err2 != nil { sylog.Errorf("Removing container %s from runtime after creation failed", containerID) } return err diff --git a/internal/pkg/runtime/launcher/oci/oci_runc_linux.go b/internal/pkg/runtime/launcher/oci/oci_runc_linux.go index cc3a5d693b..fa919dcc34 100644 --- a/internal/pkg/runtime/launcher/oci/oci_runc_linux.go +++ b/internal/pkg/runtime/launcher/oci/oci_runc_linux.go @@ -28,7 +28,7 @@ import ( ) // Delete deletes container resources -func Delete(ctx context.Context, containerID string) error { +func Delete(ctx context.Context, containerID string, systemdCgroups bool) error { runtimeBin, err := runtime() if err != nil { return err @@ -40,9 +40,11 @@ func Delete(ctx context.Context, containerID string) error { runtimeArgs := []string{ "--root", rsd, - "delete", - containerID, } + if systemdCgroups { + runtimeArgs = append(runtimeArgs, "--systemd-cgroup") + } + runtimeArgs = append(runtimeArgs, "delete", containerID) cmd := exec.Command(runtimeBin, runtimeArgs...) cmd.Stdout = os.Stdout @@ -75,7 +77,7 @@ func Delete(ctx context.Context, containerID string) error { } // Exec executes a command in a container -func Exec(containerID string, cmdArgs []string) error { +func Exec(containerID string, cmdArgs []string, systemdCgroups bool) error { runtimeBin, err := runtime() if err != nil { return err @@ -87,9 +89,11 @@ func Exec(containerID string, cmdArgs []string) error { runtimeArgs := []string{ "--root", rsd, - "exec", - containerID, } + if systemdCgroups { + runtimeArgs = append(runtimeArgs, "--systemd-cgroup") + } + runtimeArgs = append(runtimeArgs, "exec", containerID) runtimeArgs = append(runtimeArgs, cmdArgs...) cmd := exec.Command(runtimeBin, runtimeArgs...) cmd.Stdout = os.Stdout @@ -100,7 +104,7 @@ func Exec(containerID string, cmdArgs []string) error { } // Kill kills container process -func Kill(containerID string, killSignal string) error { +func Kill(containerID string, killSignal string, systemdCgroups bool) error { runtimeBin, err := runtime() if err != nil { return err @@ -126,7 +130,7 @@ func Kill(containerID string, killSignal string) error { } // Pause pauses processes in a container -func Pause(containerID string) error { +func Pause(containerID string, systemdCgroups bool) error { runtimeBin, err := runtime() if err != nil { return err @@ -138,9 +142,11 @@ func Pause(containerID string) error { runtimeArgs := []string{ "--root", rsd, - "pause", - containerID, } + if systemdCgroups { + runtimeArgs = append(runtimeArgs, "--systemd-cgroup") + } + runtimeArgs = append(runtimeArgs, "pause", containerID) cmd := exec.Command(runtimeBin, runtimeArgs...) cmd.Stdout = os.Stdout @@ -151,7 +157,7 @@ func Pause(containerID string) error { } // Resume pauses processes in a container -func Resume(containerID string) error { +func Resume(containerID string, systemdCgroups bool) error { runtimeBin, err := runtime() if err != nil { return err @@ -163,9 +169,11 @@ func Resume(containerID string) error { runtimeArgs := []string{ "--root", rsd, - "resume", - containerID, } + if systemdCgroups { + runtimeArgs = append(runtimeArgs, "--systemd-cgroup") + } + runtimeArgs = append(runtimeArgs, "resume", containerID) cmd := exec.Command(runtimeBin, runtimeArgs...) cmd.Stdout = os.Stdout @@ -176,7 +184,7 @@ func Resume(containerID string) error { } // Run runs a container (equivalent to create/start/delete) -func Run(ctx context.Context, containerID, bundlePath, pidFile string) error { +func Run(ctx context.Context, containerID, bundlePath, pidFile string, systemdCgroups bool) error { runtimeBin, err := runtime() if err != nil { return err @@ -197,12 +205,15 @@ func Run(ctx context.Context, containerID, bundlePath, pidFile string) error { runtimeArgs := []string{ "--root", rsd, - "run", - "-b", absBundle, } + if systemdCgroups { + runtimeArgs = append(runtimeArgs, "--systemd-cgroup") + } + runtimeArgs = append(runtimeArgs, "run", "-b", absBundle) if pidFile != "" { runtimeArgs = append(runtimeArgs, "--pid-file="+pidFile) } + runtimeArgs = append(runtimeArgs, containerID) cmd := exec.Command(runtimeBin, runtimeArgs...) cmd.Stdout = os.Stdout @@ -237,9 +248,13 @@ func RunNS(ctx context.Context, containerID, bundlePath, pidFile string) error { sylog.Debugf("Calling fakeroot engine to execute %q", strings.Join(args, " ")) cfg := &config.Common{ - EngineName: fakerootConfig.Name, - ContainerID: "fakeroot", - EngineConfig: &fakerootConfig.EngineConfig{Args: args, NoPIDNS: true}, + EngineName: fakerootConfig.Name, + ContainerID: "fakeroot", + EngineConfig: &fakerootConfig.EngineConfig{ + Envs: os.Environ(), + Args: args, + NoPIDNS: true, + }, } return starter.Run( @@ -252,7 +267,7 @@ func RunNS(ctx context.Context, containerID, bundlePath, pidFile string) error { } // Start starts a previously created container -func Start(containerID string) error { +func Start(containerID string, systemdCgroups bool) error { runtimeBin, err := runtime() if err != nil { return err @@ -264,9 +279,11 @@ func Start(containerID string) error { runtimeArgs := []string{ "--root", rsd, - "start", - containerID, } + if systemdCgroups { + runtimeArgs = append(runtimeArgs, "--systemd-cgroup") + } + runtimeArgs = append(runtimeArgs, "start", containerID) cmd := exec.Command(runtimeBin, runtimeArgs...) cmd.Stdout = os.Stdout @@ -277,7 +294,7 @@ func Start(containerID string) error { } // State queries container state -func State(containerID string) error { +func State(containerID string, systemdCgroups bool) error { runtimeBin, err := runtime() if err != nil { return err @@ -289,9 +306,11 @@ func State(containerID string) error { runtimeArgs := []string{ "--root", rsd, - "state", - containerID, } + if systemdCgroups { + runtimeArgs = append(runtimeArgs, "--systemd-cgroup") + } + runtimeArgs = append(runtimeArgs, "state", containerID) cmd := exec.Command(runtimeBin, runtimeArgs...) cmd.Stdout = os.Stdout @@ -302,7 +321,7 @@ func State(containerID string) error { } // Update updates container cgroups resources -func Update(containerID, cgFile string) error { +func Update(containerID, cgFile string, systemdCgroups bool) error { runtimeBin, err := runtime() if err != nil { return err @@ -314,10 +333,11 @@ func Update(containerID, cgFile string) error { runtimeArgs := []string{ "--root", rsd, - "update", - "-r", cgFile, - containerID, } + if systemdCgroups { + runtimeArgs = append(runtimeArgs, "--systemd-cgroup") + } + runtimeArgs = append(runtimeArgs, "update", "-r", cgFile, containerID) cmd := exec.Command(runtimeBin, runtimeArgs...) cmd.Stdout = os.Stdout From 01c98205b9da46a89558851ff4004e7a92f65366 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Mon, 6 Mar 2023 14:26:30 +0000 Subject: [PATCH 067/114] fix: fall back to cgroupfs for OCI non-root, non cgroups v2 For the `OCI` command group, we need cgroups v2 unified mode in order to call `runc` with `--systemd-cgroup` as a non-root user. Fall back to cgroupfs operation if this constraint is not satisified. This fixes a failure when running a container using `runc`. `crun` was falling back to cgroupfs itself, even when `--systemd-cgroup` was specified. Fixes sylabs/singularity#1408 Signed-off-by: Edita Kizinevic --- internal/app/apptainer/oci_linux.go | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/internal/app/apptainer/oci_linux.go b/internal/app/apptainer/oci_linux.go index d102aee784..faa803d730 100644 --- a/internal/app/apptainer/oci_linux.go +++ b/internal/app/apptainer/oci_linux.go @@ -20,6 +20,8 @@ import ( "github.com/apptainer/apptainer/internal/pkg/runtime/launcher/oci" ocibundle "github.com/apptainer/apptainer/pkg/ocibundle/sif" "github.com/apptainer/apptainer/pkg/util/apptainerconf" + "github.com/apptainer/apptainer/pkg/util/namespaces" + lccgroups "github.com/opencontainers/runc/libcontainer/cgroups" ) // OciArgs contains CLI arguments @@ -151,5 +153,18 @@ func systemdCgroups() (use bool, err error) { return false, fmt.Errorf("unable to parse apptainer configuration file: %w", err) } } - return cfg.SystemdCgroups, nil + + useSystemd := cfg.SystemdCgroups + + // As non-root, we need cgroups v2 unified mode for systemd support. + // Fall back to cgroupfs if this is not available. + hostUID, err := namespaces.HostUID() + if err != nil { + return false, fmt.Errorf("while finding host uid: %w", err) + } + if hostUID != 0 && !lccgroups.IsCgroup2UnifiedMode() { + useSystemd = false + } + + return useSystemd, nil } From d59ca94230bdaae3809614ac9517a471b4cc6c2d Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Wed, 8 Mar 2023 13:03:29 +0000 Subject: [PATCH 068/114] oci: drop warning about runc functionality Drop the warning that not all OCI functionality is supported with runc. Some things turned out to be SingularityCE bugs. Some things are very version specific. A new enough runc will work. Fixes sylabs/singularity#1425 Signed-off-by: Edita Kizinevic --- internal/pkg/runtime/launcher/oci/oci_linux.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/pkg/runtime/launcher/oci/oci_linux.go b/internal/pkg/runtime/launcher/oci/oci_linux.go index f78d9e0cd7..45cc4c560a 100644 --- a/internal/pkg/runtime/launcher/oci/oci_linux.go +++ b/internal/pkg/runtime/launcher/oci/oci_linux.go @@ -51,7 +51,7 @@ func runtime() (path string, err error) { return } sylog.Debugf("While finding crun: %s", err) - sylog.Warningf("crun not found. Will attempt to use runc, but not all functionality is supported.") + sylog.Debugf("Falling back to runc as OCI runtime.") return bin.FindBin("runc") } From 267615e7939a8046bb12a93be9dc3626fb3e6a46 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Fri, 17 Mar 2023 10:42:03 +0000 Subject: [PATCH 069/114] rpm: fix: correct BuildRequires/Requires for SLES * On SLES we intend to require runc, not crun. * squashfs had a duplicate Requires, instead of a BuildRequires. Fixes sylabs/singularity#1453 Signed-off-by: Edita Kizinevic --- dist/rpm/apptainer.spec.in | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dist/rpm/apptainer.spec.in b/dist/rpm/apptainer.spec.in index 54062229be..5742b620da 100644 --- a/dist/rpm/apptainer.spec.in +++ b/dist/rpm/apptainer.spec.in @@ -101,10 +101,10 @@ BuildRequires: cryptsetup BuildRequires: libseccomp-devel Requires: conmon # crun requirement not satisfied on EL7 or SLES default repos - use runc there. -%if "%{_target_vendor}" == "suse" || 0%{?rhel} > 7 -Requires: crun -%else +%if "%{_target_vendor}" == "suse" || 0%{?rhel} < 8 Requires: runc +%else +Requires: crun %endif Requires: cryptsetup %if "%{?squashfuse_version}" != "" From 310374308a7c33657c7f64f3eaa7ec106c82769e Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Fri, 24 Feb 2023 17:50:46 +0000 Subject: [PATCH 070/114] e2e: refactor test image handling Refactor test image handling so that: * We create the OCI archive, and Docker archive, used in the e2e tests with a container/image Copy from our local registry busybox image. * We use DockerHub credentials for e2e.CopyOCIImage. * We replace a couple of uses of `library://` images with our local registry ORAS SIF, avoiding some additional network access. Fixes sylabs/singularity#1364 Signed-off-by: Edita Kizinevic --- e2e/actions/oci.go | 41 +++++++------------ e2e/cgroups/cgroups.go | 16 ++++---- e2e/env/oci.go | 12 +++--- e2e/internal/e2e/env.go | 3 +- e2e/internal/e2e/image.go | 84 ++++++++++++++++++++++++--------------- e2e/suite.go | 18 ++++++--- 6 files changed, 96 insertions(+), 78 deletions(-) diff --git a/e2e/actions/oci.go b/e2e/actions/oci.go index 98e9e6c219..01ecf4615b 100644 --- a/e2e/actions/oci.go +++ b/e2e/actions/oci.go @@ -19,25 +19,14 @@ import ( "github.com/apptainer/apptainer/internal/pkg/util/fs" ) -const ( - dockerArchiveURI = "https://github.com/apptainer/apptainer/releases/download/v0.1.0/alpine-docker-save.tar" -) - func (c actionTests) actionOciRun(t *testing.T) { - e2e.EnsureOCIImage(t, c.env) + e2e.EnsureOCIArchive(t, c.env) + e2e.EnsureDockerArchive(t, c.env) - // Prepare docker-archive source - tmpDir := t.TempDir() - dockerArchive := filepath.Join(tmpDir, "docker-archive.tar") - err := e2e.DownloadFile(dockerArchiveURI, dockerArchive) - if err != nil { - t.Fatalf("Could not download docker archive test file: %v", err) - } - defer os.Remove(dockerArchive) // Prepare oci source (oci directory layout) ociLayout := t.TempDir() - cmd := exec.Command("tar", "-C", ociLayout, "-xf", c.env.OCIImagePath) - err = cmd.Run() + cmd := exec.Command("tar", "-C", ociLayout, "-xf", c.env.OCIArchivePath) + err := cmd.Run() if err != nil { t.Fatalf("Error extracting oci archive to layout: %v", err) } @@ -50,12 +39,12 @@ func (c actionTests) actionOciRun(t *testing.T) { }{ { name: "docker-archive", - imageRef: "docker-archive:" + dockerArchive, + imageRef: "docker-archive:" + c.env.DockerArchivePath, exit: 0, }, { name: "oci-archive", - imageRef: "oci-archive:" + c.env.OCIImagePath, + imageRef: "oci-archive:" + c.env.OCIArchivePath, exit: 0, }, { @@ -99,9 +88,9 @@ func (c actionTests) actionOciRun(t *testing.T) { // exec tests min fuctionality for apptainer exec func (c actionTests) actionOciExec(t *testing.T) { - e2e.EnsureOCIImage(t, c.env) + e2e.EnsureOCIArchive(t, c.env) - imageRef := "oci-archive:" + c.env.OCIImagePath + imageRef := "oci-archive:" + c.env.OCIArchivePath tests := []struct { name string @@ -173,7 +162,7 @@ func (c actionTests) actionOciExec(t *testing.T) { // Shell interaction tests func (c actionTests) actionOciShell(t *testing.T) { - e2e.EnsureOCIImage(t, c.env) + e2e.EnsureOCIArchive(t, c.env) tests := []struct { name string @@ -183,7 +172,7 @@ func (c actionTests) actionOciShell(t *testing.T) { }{ { name: "ShellExit", - argv: []string{"oci-archive:" + c.env.OCIImagePath}, + argv: []string{"oci-archive:" + c.env.OCIArchivePath}, consoleOps: []e2e.ApptainerConsoleOp{ // "cd /" to work around issue where a long // working directory name causes the test @@ -199,7 +188,7 @@ func (c actionTests) actionOciShell(t *testing.T) { }, { name: "ShellBadCommand", - argv: []string{"oci-archive:" + c.env.OCIImagePath}, + argv: []string{"oci-archive:" + c.env.OCIArchivePath}, consoleOps: []e2e.ApptainerConsoleOp{ e2e.ConsoleSendLine("_a_fake_command"), e2e.ConsoleSendLine("exit"), @@ -226,8 +215,8 @@ func (c actionTests) actionOciShell(t *testing.T) { } func (c actionTests) actionOciNetwork(t *testing.T) { - e2e.EnsureOCIImage(t, c.env) - imageRef := "oci-archive:" + c.env.OCIImagePath + e2e.EnsureOCIArchive(t, c.env) + imageRef := "oci-archive:" + c.env.OCIArchivePath tests := []struct { name string @@ -287,8 +276,8 @@ func (c actionTests) actionOciNetwork(t *testing.T) { //nolint:maintidx func (c actionTests) actionOciBinds(t *testing.T) { - e2e.EnsureOCIImage(t, c.env) - imageRef := "oci-archive:" + c.env.OCIImagePath + e2e.EnsureOCIArchive(t, c.env) + imageRef := "oci-archive:" + c.env.OCIArchivePath workspace, cleanup := e2e.MakeTempDir(t, c.env.TestDir, "bind-workspace-", "") defer e2e.Privileged(cleanup) diff --git a/e2e/cgroups/cgroups.go b/e2e/cgroups/cgroups.go index 645511159c..885c818b3b 100644 --- a/e2e/cgroups/cgroups.go +++ b/e2e/cgroups/cgroups.go @@ -384,18 +384,18 @@ func (c *ctx) actionApply(t *testing.T, profile e2e.Profile, imageRef string) { func (c *ctx) actionApplyRoot(t *testing.T) { e2e.EnsureImage(t, c.env) - e2e.EnsureOCIImage(t, c.env) + e2e.EnsureOCIArchive(t, c.env) t.Run(e2e.RootProfile.String(), func(t *testing.T) { c.actionApply(t, e2e.RootProfile, c.env.ImagePath) }) t.Run(e2e.OCIRootProfile.String(), func(t *testing.T) { - c.actionApply(t, e2e.OCIRootProfile, "oci-archive:"+c.env.OCIImagePath) + c.actionApply(t, e2e.OCIRootProfile, "oci-archive:"+c.env.OCIArchivePath) }) } func (c *ctx) actionApplyRootless(t *testing.T) { e2e.EnsureImage(t, c.env) - e2e.EnsureOCIImage(t, c.env) + e2e.EnsureOCIArchive(t, c.env) for _, profile := range []e2e.Profile{e2e.UserProfile, e2e.UserNamespaceProfile, e2e.FakerootProfile} { t.Run(profile.String(), func(t *testing.T) { c.actionApply(t, profile, c.env.ImagePath) @@ -403,7 +403,7 @@ func (c *ctx) actionApplyRootless(t *testing.T) { } for _, profile := range []e2e.Profile{e2e.OCIUserProfile, e2e.OCIFakerootProfile} { t.Run(profile.String(), func(t *testing.T) { - c.actionApply(t, profile, "oci-archive:"+c.env.OCIImagePath) + c.actionApply(t, profile, "oci-archive:"+c.env.OCIArchivePath) }) } } @@ -626,18 +626,18 @@ func (c *ctx) actionFlagV2(t *testing.T, tt resourceFlagTest, profile e2e.Profil func (c *ctx) actionFlagsRoot(t *testing.T) { e2e.EnsureImage(t, c.env) - e2e.EnsureOCIImage(t, c.env) + e2e.EnsureOCIArchive(t, c.env) t.Run(e2e.RootProfile.String(), func(t *testing.T) { c.actionFlags(t, e2e.RootProfile, c.env.ImagePath) }) t.Run(e2e.OCIRootProfile.String(), func(t *testing.T) { - c.actionFlags(t, e2e.OCIRootProfile, "oci-archive:"+c.env.OCIImagePath) + c.actionFlags(t, e2e.OCIRootProfile, "oci-archive:"+c.env.OCIArchivePath) }) } func (c *ctx) actionFlagsRootless(t *testing.T) { e2e.EnsureImage(t, c.env) - e2e.EnsureOCIImage(t, c.env) + e2e.EnsureOCIArchive(t, c.env) for _, profile := range []e2e.Profile{e2e.UserProfile, e2e.UserNamespaceProfile, e2e.FakerootProfile} { t.Run(profile.String(), func(t *testing.T) { c.actionFlags(t, profile, c.env.ImagePath) @@ -645,7 +645,7 @@ func (c *ctx) actionFlagsRootless(t *testing.T) { } for _, profile := range []e2e.Profile{e2e.OCIUserProfile, e2e.OCIFakerootProfile} { t.Run(profile.String(), func(t *testing.T) { - c.actionFlags(t, profile, "oci-archive:"+c.env.OCIImagePath) + c.actionFlags(t, profile, "oci-archive:"+c.env.OCIArchivePath) }) } } diff --git a/e2e/env/oci.go b/e2e/env/oci.go index 7632b52a85..9d7610dcde 100644 --- a/e2e/env/oci.go +++ b/e2e/env/oci.go @@ -19,8 +19,8 @@ import ( ) func (c ctx) ociApptainerEnv(t *testing.T) { - e2e.EnsureOCIImage(t, c.env) - defaultImage := "oci-archive:" + c.env.OCIImagePath + e2e.EnsureOCIArchive(t, c.env) + defaultImage := "oci-archive:" + c.env.OCIArchivePath // Append or prepend this path. partialPath := "/foo" @@ -87,8 +87,8 @@ func (c ctx) ociApptainerEnv(t *testing.T) { } func (c ctx) ociEnvOption(t *testing.T) { - e2e.EnsureOCIImage(t, c.env) - defaultImage := "oci-archive:" + c.env.OCIImagePath + e2e.EnsureOCIArchive(t, c.env) + defaultImage := "oci-archive:" + c.env.OCIArchivePath tests := []struct { name string @@ -197,8 +197,8 @@ func (c ctx) ociEnvOption(t *testing.T) { } func (c ctx) ociEnvFile(t *testing.T) { - e2e.EnsureOCIImage(t, c.env) - defaultImage := "oci-archive:" + c.env.OCIImagePath + e2e.EnsureOCIArchive(t, c.env) + defaultImage := "oci-archive:" + c.env.OCIArchivePath dir, cleanup := e2e.MakeTempDir(t, c.env.TestDir, "envfile-", "") defer cleanup(t) diff --git a/e2e/internal/e2e/env.go b/e2e/internal/e2e/env.go index f67f826e79..9a99c9cafd 100644 --- a/e2e/internal/e2e/env.go +++ b/e2e/internal/e2e/env.go @@ -18,8 +18,9 @@ type TestEnv struct { SingularityImagePath string // Path to a Singularity image for legacy tests DebianImagePath string // Path to an image containing a Debian distribution with libc compatible to the host libc OrasTestImage string // URI to SIF image pushed into local registry with ORAS - OCIImagePath string + OCIArchivePath string // Path to test OCI archive tar file TestDir string // Path to the directory from which an Apptainer command needs to be executed + DockerArchivePath string // Path to test Docker archive tar file TestRegistry string // Host:Port of local registry TestRegistryImage string // URI to OCI image pushed into local registry HomeDir string // HomeDir sets the home directory that will be used for the execution of a command diff --git a/e2e/internal/e2e/image.go b/e2e/internal/e2e/image.go index 367de54140..ad2d602209 100644 --- a/e2e/internal/e2e/image.go +++ b/e2e/internal/e2e/image.go @@ -34,8 +34,6 @@ import ( "github.com/containers/image/v5/types" ) -const ociArchiveURI = "https://github.com/apptainer/apptainer/releases/download/v0.1.0/alpine-oci-archive.tar" - var ( ensureMutex sync.Mutex pullMutex sync.Mutex @@ -288,28 +286,6 @@ func BusyboxSIF(t *testing.T) string { return busyboxSIF } -func parseRef(refString string) (ref types.ImageReference, err error) { - parts := strings.SplitN(refString, ":", 2) - if len(parts) < 2 { - return nil, fmt.Errorf("could not parse image ref: %s", refString) - } - - switch parts[0] { - case "docker": - ref, err = docker.ParseReference(parts[1]) - case "docker-archive": - ref, err = dockerarchive.ParseReference(parts[1]) - case "oci": - ref, err = ocilayout.ParseReference(parts[1]) - case "oci-archive": - ref, err = ociarchive.ParseReference(parts[1]) - default: - return nil, fmt.Errorf("cannot create an OCI container from %s source", parts[0]) - } - - return ref, err -} - func DownloadFile(url string, path string) error { dl, err := os.Create(path) if err != nil { @@ -330,13 +306,13 @@ func DownloadFile(url string, path string) error { return nil } -// EnsureImage checks if e2e OCI test image is available, and fetches +// EnsureImage checks if e2e OCI test archive is available, and fetches // it otherwise. -func EnsureOCIImage(t *testing.T, env TestEnv) { +func EnsureOCIArchive(t *testing.T, env TestEnv) { ensureMutex.Lock() defer ensureMutex.Unlock() - switch _, err := os.Stat(env.OCIImagePath); { + switch _, err := os.Stat(env.OCIArchivePath); { case err == nil: // OK: file exists, return return @@ -347,13 +323,59 @@ func EnsureOCIImage(t *testing.T, env TestEnv) { default: // FATAL: something else is wrong t.Fatalf("Failed when checking image %q: %+v\n", - env.OCIImagePath, + env.OCIArchivePath, err) } // Prepare oci-archive source - err := DownloadFile(ociArchiveURI, env.OCIImagePath) - if err != nil { - t.Fatalf("Could not download oci archive test file: %v", err) + t.Logf("Copying %s to %s", env.TestRegistryImage, "oci-archive:"+env.OCIArchivePath) + CopyImage(t, env.TestRegistryImage, "oci-archive:"+env.OCIArchivePath, true, false) +} + +// EnsureDockerArchive checks if e2e Docker test archive is available, and fetches +// it otherwise. +func EnsureDockerArchive(t *testing.T, env TestEnv) { + ensureMutex.Lock() + defer ensureMutex.Unlock() + + switch _, err := os.Stat(env.DockerArchivePath); { + case err == nil: + // OK: file exists, return + return + + case os.IsNotExist(err): + // OK: file does not exist, continue + + default: + // FATAL: something else is wrong + t.Fatalf("Failed when checking image %q: %+v\n", + env.DockerArchivePath, + err) + } + + // Prepare oci-archive source + t.Logf("Copying %s to %s", env.TestRegistryImage, "docker-archive:"+env.DockerArchivePath) + CopyImage(t, env.TestRegistryImage, "docker-archive:"+env.DockerArchivePath, true, false) +} + +func parseRef(refString string) (ref types.ImageReference, err error) { + parts := strings.SplitN(refString, ":", 2) + if len(parts) < 2 { + return nil, fmt.Errorf("could not parse image ref: %s", refString) + } + + switch parts[0] { + case "docker": + ref, err = docker.ParseReference(parts[1]) + case "docker-archive": + ref, err = dockerarchive.ParseReference(parts[1]) + case "oci": + ref, err = ocilayout.ParseReference(parts[1]) + case "oci-archive": + ref, err = ociarchive.ParseReference(parts[1]) + default: + return nil, fmt.Errorf("cannot create an OCI container from %s source", parts[0]) } + + return ref, err } diff --git a/e2e/suite.go b/e2e/suite.go index 4c30147665..b28e22b3d2 100644 --- a/e2e/suite.go +++ b/e2e/suite.go @@ -164,12 +164,6 @@ func Run(t *testing.T) { testenv.SingularityImagePath = path.Join(name, "test-singularity.sif") defer os.Remove(testenv.SingularityImagePath) - // OCI Test image - ociImagePath := path.Join(name, "oci.tar") - t.Log("Path to test OCI image:", ociImagePath) - testenv.OCIImagePath = ociImagePath - defer os.Remove(ociImagePath) - testenv.DebianImagePath = path.Join(name, "test-debian.sif") defer os.Remove(testenv.DebianImagePath) @@ -194,11 +188,23 @@ func Run(t *testing.T) { t.Log("Path to test image:", imagePath) testenv.ImagePath = imagePath + // OCI Archive test image path, built on demand by e2e.EnsureOCIArchive + ociArchivePath := path.Join(name, "oci.tar") + t.Log("Path to test OCI archive:", ociArchivePath) + testenv.OCIArchivePath = ociArchivePath + + // Docker Archive test image path, built on demand by e2e.EnsureDockerArhive + dockerArchivePath := path.Join(name, "docker.tar") + t.Log("Path to test Docker archive:", dockerArchivePath) + testenv.DockerArchivePath = dockerArchivePath + // Local registry ORAS SIF image, built on demand by e2e.EnsureORASImage testenv.OrasTestImage = fmt.Sprintf("oras://%s/oras_test_sif:latest", testenv.TestRegistry) t.Cleanup(func() { os.Remove(imagePath) + os.Remove(ociArchivePath) + os.Remove(dockerArchivePath) }) suite := testhelper.NewSuite(t, testenv) From 7117ded748fe0eaed82cb401c6c6a0f5b8e4c72d Mon Sep 17 00:00:00 2001 From: preminger Date: Mon, 27 Mar 2023 09:27:15 -0400 Subject: [PATCH 071/114] oci: implemented CDI device mapping (sylabs/singularity#1459) * oci: implemented CDI device mapping * first batch of post-comments revisions * revisions following elezar's comments * added `--device` to CHANGELOG * fixed out-of-order calls to addCDIDevices & Update * added sync.Once getting & refreshing CDI registry * cdi unit-test stub * added some more cdi unit-tests * more unit-tests * better testing for empty mounts lists * finishing touches on cdi unit-test * first stab at cdi e2e-test * DT's temp fix for userns mapping limitation * ignore VSCode debugging build targets * renamed CDI json template file * added initial deviceNode testing * changed flag to --cdi-dirs (from --cdidirs) Signed-off-by: Edita Kizinevic --- .gitignore | 9 + CHANGELOG.md | 4 + LICENSE_DEPENDENCIES.md | 30 ++ cmd/internal/cli/action_flags.go | 26 +- cmd/internal/cli/actions.go | 2 + e2e/actions/actions.go | 1 + e2e/actions/oci.go | 220 +++++++++++++++ go.mod | 4 + go.sum | 8 + .../runtime/launcher/native/launcher_linux.go | 7 + .../pkg/runtime/launcher/oci/cdi_linux.go | 60 ++++ .../runtime/launcher/oci/cdi_linux_test.go | 263 ++++++++++++++++++ .../runtime/launcher/oci/launcher_linux.go | 13 +- .../pkg/runtime/launcher/oci/spec_linux.go | 1 + internal/pkg/runtime/launcher/options.go | 22 ++ test/cdi/cditemplate.json.tpl | 18 ++ test/cdi/kmsg.json | 51 ++++ test/cdi/tmpmount.json | 45 +++ 18 files changed, 781 insertions(+), 3 deletions(-) create mode 100644 internal/pkg/runtime/launcher/oci/cdi_linux.go create mode 100644 internal/pkg/runtime/launcher/oci/cdi_linux_test.go create mode 100644 test/cdi/cditemplate.json.tpl create mode 100644 test/cdi/kmsg.json create mode 100644 test/cdi/tmpmount.json diff --git a/.gitignore b/.gitignore index e8e1026cca..ebaba871bc 100644 --- a/.gitignore +++ b/.gitignore @@ -110,3 +110,12 @@ pkg/library/client/test[0-9]* /debian LICENSE_DEPENDENCIES.csv + +# VSCode debugging build targets +__debug_bin +*/__debug_bin +*/*/__debug_bin +*/*/*/__debug_bin +*/*/*/*/__debug_bin +*/*/*/*/*/__debug_bin +*/*/*/*/*/*/__debug_bin diff --git a/CHANGELOG.md b/CHANGELOG.md index fff053cdc9..11e8c6d3d7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -46,6 +46,10 @@ For older changes see the [archived Singularity change log](https://github.com/a the container. - `--apply-cgroups`, and the `--cpu*`, `--blkio*`, `--memory*`, `--pids-limit` flags to apply resource limits. +- Added `--device` flag to "action" commands (`run`/`exec`/`shell`) when run in + OCI mode (`--oci`). Currently supports passing one or more (comma-separated) + fully-qualified CDI device names, and those devices will then be made + available inside the container. ### New Features & Functionality diff --git a/LICENSE_DEPENDENCIES.md b/LICENSE_DEPENDENCIES.md index 5e6581f94b..247cf4ffd6 100644 --- a/LICENSE_DEPENDENCIES.md +++ b/LICENSE_DEPENDENCIES.md @@ -17,6 +17,12 @@ The dependencies and their licenses are as follows: **License URL:** +## github.com/container-orchestrated-devices/container-device-interface + +**License:** Apache-2.0 + +**License URL:** + ## github.com/containerd/containerd **License:** Apache-2.0 @@ -293,6 +299,12 @@ The dependencies and their licenses are as follows: **License URL:** +## github.com/opencontainers/runtime-tools + +**License:** Apache-2.0 + +**License URL:** + ## github.com/opencontainers/selinux **License:** Apache-2.0 @@ -533,6 +545,12 @@ The dependencies and their licenses are as follows: **License URL:** +## github.com/fsnotify/fsnotify + +**License:** BSD-3-Clause + +**License URL:** + ## github.com/gogo/protobuf/proto **License:** BSD-3-Clause @@ -635,6 +653,12 @@ The dependencies and their licenses are as follows: **Project URL:** +## golang.org/x/mod/semver + +**License:** BSD-3-Clause + +**Project URL:** + ## golang.org/x/net **License:** BSD-3-Clause @@ -929,6 +953,12 @@ The dependencies and their licenses are as follows: **Project URL:** +## sigs.k8s.io/yaml + +**License:** MIT + +**Project URL:** + ## github.com/gosimple/slug **License:** MPL-2.0 diff --git a/cmd/internal/cli/action_flags.go b/cmd/internal/cli/action_flags.go index 60b38f1243..c3704c1534 100644 --- a/cmd/internal/cli/action_flags.go +++ b/cmd/internal/cli/action_flags.go @@ -42,6 +42,8 @@ var ( noMount []string dmtcpLaunch string dmtcpRestart string + device []string + cdiDirs []string isBoot bool isFakeroot bool @@ -116,7 +118,7 @@ var actionBindFlag = cmdline.Flag{ DefaultValue: cmdline.StringArray{}, // to allow commas in bind path Name: "bind", ShortHand: "B", - Usage: "a user-bind path specification. spec has the format src[:dest[:opts]], where src and dest are outside and inside paths. If dest is not given, it is set equal to src. Mount options ('opts') may be specified as 'ro' (read-only) or 'rw' (read/write, which is the default). Multiple bind paths can be given by a comma separated list.", + Usage: "a user-bind path specification. spec has the format src[:dest[:opts]], where src and dest are outside and inside paths. If dest is not given, it is set equal to src. Mount options ('opts') may be specified as 'ro' (read-only) or 'rw' (read/write, which is the default). Multiple bind paths can be given by a comma separated list.", EnvKeys: []string{"BIND", "BINDPATH"}, Tag: "", EnvHandler: cmdline.EnvAppendValue, @@ -141,7 +143,7 @@ var actionHomeFlag = cmdline.Flag{ DefaultValue: CurrentUser.HomeDir, Name: "home", ShortHand: "H", - Usage: "a home directory specification. spec can either be a src path or src:dest pair. src is the source path of the home directory outside the container and dest overrides the home directory within the container.", + Usage: "a home directory specification. spec can either be a src path or src:dest pair. src is the source path of the home directory outside the container and dest overrides the home directory within the container.", EnvKeys: []string{"HOME"}, Tag: "", } @@ -887,6 +889,24 @@ var actionOCIFlag = cmdline.Flag{ EnvKeys: []string{"OCI"}, } +// --device +var actionDevice = cmdline.Flag{ + ID: "actionDevice", + Value: &device, + DefaultValue: []string{}, + Name: "device", + Usage: "fully-qualified CDI device name(s). A fully-qualified CDI device name consists of a VENDOR, CLASS, and NAME, which are combined as follows: /= (e.g. vendor.com/device=mydevice). Multiple fully-qualified CDI device names can be given as a comma separated list.", +} + +// --cdi-dirs +var actionCdiDirs = cmdline.Flag{ + ID: "actionCdiDirs", + Value: &cdiDirs, + DefaultValue: []string{}, + Name: "cdi-dirs", + Usage: "comma-separated list of directories in which CDI should look for device definition JSON files. If omitted, default will be: /etc/cdi,/var/run/cdi", +} + func init() { addCmdInit(func(cmdManager *cmdline.CommandManager) { cmdManager.RegisterCmd(ExecCmd) @@ -985,5 +1005,7 @@ func init() { cmdManager.RegisterFlagForCmd(&actionIgnoreUsernsFlag, actionsInstanceCmd...) cmdManager.RegisterFlagForCmd(&actionUnderlayFlag, actionsInstanceCmd...) cmdManager.RegisterFlagForCmd(&actionOCIFlag, actionsCmd...) + cmdManager.RegisterFlagForCmd(&actionDevice, actionsCmd...) + cmdManager.RegisterFlagForCmd(&actionCdiDirs, actionsCmd...) }) } diff --git a/cmd/internal/cli/actions.go b/cmd/internal/cli/actions.go index 814875b3e5..05d66abe12 100644 --- a/cmd/internal/cli/actions.go +++ b/cmd/internal/cli/actions.go @@ -407,6 +407,8 @@ func launchContainer(cmd *cobra.Command, image string, containerCmd string, cont launcher.OptUseBuildConfig(useBuildConfig), launcher.OptTmpDir(tmpDir), launcher.OptUnderlay(underlay), + launcher.OptDevice(device), + launcher.OptCdiDirs(cdiDirs), } var l launcher.Launcher diff --git a/e2e/actions/actions.go b/e2e/actions/actions.go index 8a29e1fee8..94a8b97f02 100644 --- a/e2e/actions/actions.go +++ b/e2e/actions/actions.go @@ -2917,5 +2917,6 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { "ociShell": c.actionOciShell, // apptainer shell --oci "ociNetwork": c.actionOciNetwork, // apptainer exec --oci --net "ociBinds": c.actionOciBinds, // apptainer exec --oci --bind / --mount + "ociCdi": c.actionOciCdi, // apptainer exec --oci --cdi } } diff --git a/e2e/actions/oci.go b/e2e/actions/oci.go index 01ecf4615b..ce0c55bf60 100644 --- a/e2e/actions/oci.go +++ b/e2e/actions/oci.go @@ -10,13 +10,18 @@ package actions import ( + "encoding/json" + "fmt" "os" "os/exec" "path/filepath" + "strings" "testing" + "text/template" "github.com/apptainer/apptainer/e2e/internal/e2e" "github.com/apptainer/apptainer/internal/pkg/util/fs" + cdispecs "github.com/container-orchestrated-devices/container-device-interface/specs-go" ) func (c actionTests) actionOciRun(t *testing.T) { @@ -480,3 +485,218 @@ func (c actionTests) actionOciBinds(t *testing.T) { }) } } + +func (c actionTests) actionOciCdi(t *testing.T) { + // Grab the reference OCI archive we're going to use + e2e.EnsureOCIArchive(t, c.env) + imageRef := "oci-archive:" + c.env.OCIArchivePath + + // Set up a custom subtestWorkspace object that will holds the collection of temporary directories (nested under the main temporary directory, mainDir) that each test will use. + type subtestWorkspace struct { + mainDir string + jsonsDir string + mountDirs []string + } + + // Create a function to create a fresh subtestWorkspace, with distinct temporary directories, that each individual subtest will use + setupIndivSubtestWorkspace := func(t *testing.T, numMountDirs int) *subtestWorkspace { + stws := subtestWorkspace{} + mainDir, cleanup := e2e.MakeTempDir(t, c.env.TestDir, "", "") + t.Cleanup(func() { + if !t.Failed() { + e2e.Privileged(cleanup) + } + }) + stws.mainDir = mainDir + + // No need to do anything with the cleanup functions returned here, because the directories created are all going to be children of (tw.)mainDir, whose cleanup was already registered above. + stws.jsonsDir, _ = e2e.MakeTempDir(t, stws.mainDir, "cdi-jsons-", "") + stws.mountDirs = make([]string, 0, numMountDirs) + for len(stws.mountDirs) < numMountDirs { + dir, _ := e2e.MakeTempDir(t, stws.mainDir, fmt.Sprintf("mount-dir-%d-", len(stws.mountDirs)+1), "") + // Make writable to all, due to current nested userns mapping restrictions. + // Will work without this once crun-specific single mapping is present. + os.Chmod(dir, 0o777) + stws.mountDirs = append(stws.mountDirs, dir) + } + + return &stws + } + + // Set up the JSON template that we're going to populate on a per-subtest basis with particular CDI spec values + e2eMountTemplateFilename := "cditemplate.json.tpl" + cdiJSONTemplateFilePath := filepath.Join("..", "test", "cdi", e2eMountTemplateFilename) + funcMap := template.FuncMap{ + // The name "title" is what the function will be called in the template text. + "tojson": func(o any) string { + s, _ := json.Marshal(o) + return string(s) + }, + } + cdiJSONTemplate, err := template.New(e2eMountTemplateFilename).Funcs(funcMap).ParseFiles(cdiJSONTemplateFilePath) + if err != nil { + t.Errorf("Could not read JSON template for CDI e2e tests from file %#v", cdiJSONTemplateFilePath) + return + } + + // The set of actual subtests + var wantUID uint32 = 1000 + var wantGID uint32 = 1000 + tests := []struct { + name string + devices []string + wantExit int + postRun func(t *testing.T) + DeviceNodes []cdispecs.DeviceNode + Mounts []cdispecs.Mount + Env []string + }{ + { + name: "ValidMounts", + devices: []string{ + "apptainertesting.sylabs.io/device=TesterDevice", + }, + wantExit: 0, + DeviceNodes: []cdispecs.DeviceNode{}, + Mounts: []cdispecs.Mount{ + { + ContainerPath: "/tmp/mount1", + Options: []string{"rw", "bind", "users"}, + }, + { + ContainerPath: "/tmp/mount3", + Options: []string{"rw", "bind", "users"}, + }, + { + ContainerPath: "/tmp/mount13", + Options: []string{"rw", "bind", "users"}, + }, + { + ContainerPath: "/tmp/mount17", + Options: []string{"rw", "bind", "users"}, + }, + }, + Env: []string{ + "ABCD=QWERTY", + "EFGH=ASDFGH", + "IJKL=ZXCVBN", + }, + }, + { + name: "InvalidDevice", + devices: []string{ + "apptainertesting.sylabs.io/device=DoesNotExist", + }, + wantExit: 255, + DeviceNodes: []cdispecs.DeviceNode{}, + Mounts: []cdispecs.Mount{}, + Env: []string{}, + }, + { + name: "KmsgDevice", + devices: []string{ + "apptainertesting.sylabs.io/device=TesterDevice", + }, + wantExit: 0, + DeviceNodes: []cdispecs.DeviceNode{ + { + HostPath: "/dev/kmsg", + Path: "/dev/kmsg", + Permissions: "rw", + Type: "c", + UID: &wantUID, + GID: &wantGID, + }, + }, + }, + } + + for _, profile := range e2e.OCIProfiles { + t.Run(profile.String(), func(t *testing.T) { + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + stws := setupIndivSubtestWorkspace(t, len(tt.Mounts)) + + // Populate the HostPath values we're going to feed into the CDI JSON template, based on the subtestWorkspace we just created + for i, d := range stws.mountDirs { + tt.Mounts[i].HostPath = d + } + + // Inject this subtest's values into the template to create the CDI JSON file + cdiJSONFilePath := filepath.Join(stws.jsonsDir, fmt.Sprintf("%s-cdi.json", tt.name)) + cdiJSONFile, err := os.OpenFile(cdiJSONFilePath, os.O_CREATE|os.O_WRONLY, 0o644) + if err != nil { + t.Errorf("could not create file %#v for writing CDI JSON: %v", cdiJSONFilePath, err) + } + if err = cdiJSONTemplate.Execute(cdiJSONFile, tt); err != nil { + t.Errorf("error executing template %#v to create CDI JSON: %v", cdiJSONTemplateFilePath, err) + return + } + cdiJSONFile.Close() + + // Create a list of test strings, each of which will be echoed into a separate file in a separate mount in the container. + testfileStrings := make([]string, 0, len(tt.Mounts)) + for i := range tt.Mounts { + testfileStrings = append(testfileStrings, fmt.Sprintf("test_string_for_mount_%d_in_test_%s", i, tt.name)) + } + + // Generate the command to be executed in the container + // Start by printing all environment variables, to test using e2e.ContainMatch conditions later + execCmd := "/bin/env" + for _, d := range tt.DeviceNodes { + testFlag := "-f" + switch d.Type { + case "c": + testFlag = "-c" + } + execCmd += fmt.Sprintf(" && test %s %s", testFlag, d.Path) + } + for i, m := range tt.Mounts { + // Add a separate teststring echo statement for each mount + execCmd += fmt.Sprintf(" && echo %s > %s/testfile_%d", testfileStrings[i], m.ContainerPath, i) + } + + // Create a postRun function to check that the testfiles written to the container mounts made their way to the right host temporary directories + testMountsAndEnv := func(t *testing.T) { + for i, m := range tt.Mounts { + testfileFilename := filepath.Join(m.HostPath, fmt.Sprintf("testfile_%d", i)) + b, err := os.ReadFile(testfileFilename) + if err != nil { + t.Errorf("could not read testfile %s", testfileFilename) + return + } + + s := string(b) + if s != testfileStrings[i]+"\n" { + t.Errorf("mismatched testfileString; expected %#v, got %#v (mount: %#v)", s, testfileStrings[i], m) + } + } + } + + // Create a set of e2e.ApptainerCmdResultOp objects to test that environment variables have been correctly injected into the container + envExpects := make([]e2e.ApptainerCmdResultOp, 0, len(tt.Env)) + for _, e := range tt.Env { + envExpects = append(envExpects, e2e.ExpectOutput(e2e.ContainMatch, e)) + } + + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithCommand("exec"), + e2e.WithArgs( + "--device", + strings.Join(tt.devices, ","), + "--cdi-dirs", + stws.jsonsDir, + imageRef, + "/bin/sh", "-c", execCmd), + e2e.WithProfile(profile), + e2e.ExpectExit(tt.wantExit, envExpects...), + e2e.PostRun(tt.postRun), + e2e.PostRun(testMountsAndEnv), + ) + }) + } + }) + } +} diff --git a/go.mod b/go.mod index 88d2a35eb0..9b78730459 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/buger/goterm v1.0.4 github.com/buger/jsonparser v1.1.1 github.com/cenkalti/backoff/v4 v4.2.1 + github.com/container-orchestrated-devices/container-device-interface v0.5.4 github.com/containerd/containerd v1.7.2 github.com/containernetworking/cni v1.1.2 github.com/containernetworking/plugins v1.3.0 @@ -37,6 +38,7 @@ require ( github.com/opencontainers/umoci v0.4.7 github.com/pelletier/go-toml/v2 v2.0.8 github.com/pkg/errors v0.9.1 + github.com/samber/lo v1.38.1 github.com/seccomp/containers-golang v0.6.0 github.com/seccomp/libseccomp-golang v0.10.0 github.com/shopspring/decimal v1.3.1 @@ -95,6 +97,7 @@ require ( github.com/docker/go-metrics v0.0.1 // indirect github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7 // indirect github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -182,6 +185,7 @@ require ( google.golang.org/protobuf v1.30.0 // indirect gopkg.in/go-jose/go-jose.v2 v2.6.1 // indirect gopkg.in/square/go-jose.v2 v2.6.0 // indirect + sigs.k8s.io/yaml v1.3.0 // indirect ) replace oras.land/oras-go => github.com/sylabs/oras-go v1.2.4-0.20230628133146-a64659fc0454 diff --git a/go.sum b/go.sum index 7c1f775e78..2dab1f4fa4 100644 --- a/go.sum +++ b/go.sum @@ -230,6 +230,8 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb h1:EDmT6Q9Zs+SbUoc7Ik9EfrFqcylYqgPZ9ANSbTAntnE= +github.com/container-orchestrated-devices/container-device-interface v0.5.4 h1:PqQGqJqQttMP5oJ/qNGEg8JttlHqGY3xDbbcKb5T9E8= +github.com/container-orchestrated-devices/container-device-interface v0.5.4/go.mod h1:DjE95rfPiiSmG7uVXtg0z6MnPm/Lx4wxKCIts0ZE0vg= github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= @@ -467,6 +469,7 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7 h1:LofdAjjjqCSXMwLGgOgnE+rdPuvX9DxCqaHwKy7i/ko= github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= @@ -1087,6 +1090,8 @@ github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1/go.mod h1:Z0q5wiB github.com/safchain/ethtool v0.3.0 h1:gimQJpsI6sc1yIqP/y8GYgiXn/NjgvpM0RNoWLVVmP0= github.com/safchain/ethtool v0.3.0/go.mod h1:SA9BwrgyAqNo7M+uaL6IYbxpm5wk3L7Mm6ocLW+CJUs= github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= +github.com/samber/lo v1.38.1 h1:j2XEAqXKb09Am4ebOg31SpvzUTTs6EN3VfgeLUhPdXM= +github.com/samber/lo v1.38.1/go.mod h1:+m/ZKRl6ClXCE2Lgf3MsQlWfh4bn1bz6CXEOxnEXnEA= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= @@ -1591,6 +1596,7 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220906165534-d0df966e6959/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1950,3 +1956,5 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/internal/pkg/runtime/launcher/native/launcher_linux.go b/internal/pkg/runtime/launcher/native/launcher_linux.go index e7afef09db..e38c710cc2 100644 --- a/internal/pkg/runtime/launcher/native/launcher_linux.go +++ b/internal/pkg/runtime/launcher/native/launcher_linux.go @@ -80,6 +80,13 @@ func NewLauncher(opts ...launcher.Option) (*Launcher, error) { return nil, fmt.Errorf("%w", err) } } + if len(lo.Devices) > 0 { + return nil, fmt.Errorf("CDI device mappings unsupported in native launcher") + } + + if len(lo.CdiDirs) > 0 { + return nil, fmt.Errorf("CDI device mappings unsupported in native launcher") + } // Initialize empty default Apptainer Engine and OCI configuration engineConfig := apptainerConfig.NewConfig() diff --git a/internal/pkg/runtime/launcher/oci/cdi_linux.go b/internal/pkg/runtime/launcher/oci/cdi_linux.go new file mode 100644 index 0000000000..436edb715f --- /dev/null +++ b/internal/pkg/runtime/launcher/oci/cdi_linux.go @@ -0,0 +1,60 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +// Includes code from https://github.com/containers/podman +// Released under the Apache License Version 2.0 + +package oci + +import ( + "fmt" + "sync" + + "github.com/container-orchestrated-devices/container-device-interface/pkg/cdi" + "github.com/opencontainers/runtime-spec/specs-go" +) + +// A container to hold the CDI registry, plus a sync.Once object to ensure we only have to ask for it once +var regSyncContainer struct { + reg cdi.Registry + initOnce sync.Once + err error +} + +// addCDIDevices adds an array of CDI devices to an existing spec. +// Accepts optional, variable number of cdi.Option arguments (to which cdi.WithAutoRefresh(false) will be prepended). Note that due to the use of a sync.Once initialization strategy, these options will only have an effect if this is the first call made to addCDIDevices(). +func addCDIDevices(spec *specs.Spec, cdiDevices []string, cdiRegOptions ...cdi.Option) error { + regSyncContainer.initOnce.Do(func() { + // Get the CDI registry, passing a cdi.WithAutoRefresh(false) option so that CDI registry files are not scanned asynchronously. (We are about to call a manual refresh, below.) + realCDIOptions := append([]cdi.Option{cdi.WithAutoRefresh(false)}, cdiRegOptions...) + regSyncContainer.reg = cdi.GetRegistry(realCDIOptions...) + regSyncContainer.err = regSyncContainer.reg.Refresh() + }) + + if regSyncContainer.err != nil { + return fmt.Errorf("Error encountered refreshing the CDI registry during initialization: %v", regSyncContainer.err) + } + + for _, cdiDevice := range cdiDevices { + if !isCDIDevice(cdiDevice) { + return fmt.Errorf("string %#v does not represent a valid CDI device", cdiDevice) + } + } + + if _, err := regSyncContainer.reg.InjectDevices(spec, cdiDevices...); err != nil { + return fmt.Errorf("Error encountered setting up CDI devices: %w", err) + } + + return nil +} + +// isCDIDevice checks whether a string is a valid CDI device selector. +func isCDIDevice(str string) bool { + return cdi.IsQualifiedName(str) +} diff --git a/internal/pkg/runtime/launcher/oci/cdi_linux_test.go b/internal/pkg/runtime/launcher/oci/cdi_linux_test.go new file mode 100644 index 0000000000..c1c5f25f7b --- /dev/null +++ b/internal/pkg/runtime/launcher/oci/cdi_linux_test.go @@ -0,0 +1,263 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2022, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +// Package oci implements a Launcher that will configure and launch a container +// with an OCI runtime. It also provides implementations of OCI state +// transitions that can be called directly, Create/Start/Kill etc. +package oci + +import ( + "path/filepath" + "reflect" + "sort" + "testing" + + "github.com/container-orchestrated-devices/container-device-interface/pkg/cdi" + "github.com/opencontainers/runtime-spec/specs-go" + "github.com/samber/lo" +) + +var specDirs = []string{filepath.Join("..", "..", "..", "..", "..", "test", "cdi")} + +type mountsList []specs.Mount + +func (a mountsList) Len() int { return len(a) } +func (a mountsList) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a mountsList) Less(i, j int) bool { return a[i].Destination < a[j].Destination } + +func Test_addCDIDevice(t *testing.T) { + var wantUID uint32 = 1000 + var wantGID uint32 = 1000 + tests := []struct { + name string + devices []string + wantDevices []specs.LinuxDevice + wantMounts mountsList + wantErr bool + wantEnv []string + }{ + { + name: "ValidOneDeviceKmsg", + devices: []string{ + "apptainertesting.sylabs.io/device=kmsgDevice", + }, + wantDevices: []specs.LinuxDevice{ + { + Path: "/dev/kmsg", + Type: "c", + Major: 1, + Minor: 11, + FileMode: nil, + UID: &wantUID, + GID: &wantGID, + }, + }, + wantMounts: mountsList{ + { + Source: "/tmp", + Destination: "/tmpmountforkmsg", + Type: "", + Options: []string{"rw"}, + }, + }, + wantEnv: []string{ + "FOO=VALID_SPEC", + "BAR=BARVALUE1", + }, + }, + { + name: "ValidTmpDevices", + devices: []string{ + "apptainertesting.sylabs.io/device=tmpmountDevice17", + "apptainertesting.sylabs.io/device=tmpmountDevice1", + }, + wantDevices: []specs.LinuxDevice{}, + wantMounts: mountsList{ + { + Source: "/tmp", + Destination: "/tmpmount1", + Type: "", + Options: []string{"ro"}, + }, + { + Source: "/tmp", + Destination: "/tmpmount3", + Type: "", + Options: []string{"rbind", "nosuid", "nodev"}, + }, + { + Source: "/tmp", + Destination: "/tmpmount13", + Type: "", + Options: []string{"rw"}, + }, + { + Source: "/tmp", + Destination: "/tmpmount17", + Type: "", + Options: []string{"r"}, + }, + }, + wantEnv: []string{ + "ABCD=QWERTY", + "EFGH=ASDFGH", + "IJKL=ZXCVBN", + "FOO=VALID_SPEC", + "BAR=BARVALUE1", + }, + }, + { + name: "ValidTmpDevicesFromOneJSON", + devices: []string{ + "apptainertesting.sylabs.io/device=tmpmountDevice1", + }, + wantDevices: []specs.LinuxDevice{}, + wantMounts: mountsList{ + { + Source: "/tmp", + Destination: "/tmpmount1", + Type: "", + Options: []string{"ro"}, + }, + { + Source: "/tmp", + Destination: "/tmpmount3", + Type: "", + Options: []string{"rbind", "nosuid", "nodev"}, + }, + { + Source: "/tmp", + Destination: "/tmpmount13", + Type: "", + Options: []string{"rw"}, + }, + }, + wantEnv: []string{ + "ABCD=QWERTY", + "EFGH=ASDFGH", + "IJKL=ZXCVBN", + }, + }, + { + name: "ValidMixedDevices", + devices: []string{ + "apptainertesting.sylabs.io/device=tmpmountDevice17", + "apptainertesting.sylabs.io/device=kmsgDevice", + "apptainertesting.sylabs.io/device=tmpmountDevice1", + }, + wantDevices: []specs.LinuxDevice{ + { + Path: "/dev/kmsg", + Type: "c", + Major: 1, + Minor: 11, + FileMode: nil, + UID: &wantUID, + GID: &wantGID, + }, + }, + wantMounts: mountsList{ + { + Source: "/tmp", + Destination: "/tmpmount1", + Type: "", + Options: []string{"ro"}, + }, + { + Source: "/tmp", + Destination: "/tmpmount3", + Type: "", + Options: []string{"rbind", "nosuid", "nodev"}, + }, + { + Source: "/tmp", + Destination: "/tmpmount13", + Type: "", + Options: []string{"rw"}, + }, + { + Source: "/tmp", + Destination: "/tmpmount17", + Type: "", + Options: []string{"r"}, + }, + { + Source: "/tmp", + Destination: "/tmpmountforkmsg", + Type: "", + Options: []string{"rw"}, + }, + }, + wantEnv: []string{ + "ABCD=QWERTY", + "EFGH=ASDFGH", + "IJKL=ZXCVBN", + "FOO=VALID_SPEC", + "BAR=BARVALUE1", + }, + }, + { + name: "InvalidNameOneDevice", + devices: []string{ + "apptainertesting.sylabs.io/device=noSuchDevice", + }, + wantErr: true, + }, + { + name: "InvalidNameSeveralDevices", + devices: []string{ + "apptainertesting.sylabs.io/device=noSuchDevice", + "apptainertesting.sylabs.io/device=noSuchDeviceEither", + }, + wantErr: true, + }, + { + name: "InvalidNameAmongValids", + devices: []string{ + "apptainertesting.sylabs.io/device=tmpmountDevice17", + "apptainertesting.sylabs.io/device=noSuchDevice", + "apptainertesting.sylabs.io/device=tmpmountDevice1", + "apptainertesting.sylabs.io/device=kmsgDevice", + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + spec := minimalSpec() + err := addCDIDevices(&spec, tt.devices, cdi.WithSpecDirs(specDirs...)) + if (err != nil) != tt.wantErr { + t.Errorf("addCDIDevices() mismatched error values; expected %v, got %v.", tt.wantErr, err) + } + + // We need this if-statement because the comparison below is done with reflection, and so a nil array and a non-nil but zero-length array will be considered different (which is not what we want here) + if (len(tt.wantMounts) > 0) || (len(spec.Mounts) > 0) { + // Note that the current implementation of OCI/CDI sorts the mounts generated by the set of mapped devices, therefore we compare against a sorted list. + sort.Sort(tt.wantMounts) + if !reflect.DeepEqual(mountsList(spec.Mounts), tt.wantMounts) { + t.Errorf("addCDIDevices() mismatched mounts; expected %v, got %v.", tt.wantMounts, spec.Mounts) + } + } + + envMissing := hashingListSubtract(tt.wantEnv, spec.Process.Env) + if len(envMissing) > 0 { + t.Errorf("addCDIDevices() mismatched environment variables; expected, but did not find, the following environment variables: %v", envMissing) + } + }) + } +} + +// hashingListSubtract is a utility-function for subtracting a list from another list, using map's internal hashing function to do this more efficiently than lo.Difference or lo.Without (which only assume comparable, and thus run in quadratic time). +func hashingListSubtract[T comparable](toSubstractFrom []T, toSubstract []T) []T { + subtractionMap := lo.FromEntries(lo.Map(toSubstractFrom, func(item T, _ int) lo.Entry[T, bool] { + return lo.Entry[T, bool]{Key: item, Value: true} + })) + + return lo.Keys(lo.OmitByKeys(subtractionMap, toSubstract)) +} diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index 5489b43e1a..7022e8ef0a 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -33,6 +33,7 @@ import ( "github.com/apptainer/apptainer/pkg/ocibundle/tools" "github.com/apptainer/apptainer/pkg/sylog" "github.com/apptainer/apptainer/pkg/util/apptainerconf" + "github.com/container-orchestrated-devices/container-device-interface/pkg/cdi" "github.com/google/uuid" "github.com/opencontainers/runtime-spec/specs-go" ) @@ -210,7 +211,7 @@ func checkOpts(lo launcher.Options) error { // createSpec creates an initial OCI runtime specification, suitable to launch a // container. This spec excludes the Process config, as this has to be computed // where the image config is available, to account for the image's CMD / -// ENTRYPOINT / ENV / USER. +// ENTRYPOINT / ENV / USER. See finalizeSpec() function. func (l *Launcher) createSpec() (*specs.Spec, error) { spec := minimalSpec() @@ -296,6 +297,16 @@ func (l *Launcher) finalizeSpec(ctx context.Context, b ocibundle.Bundle, spec *s return err } spec.Process = specProcess + + if len(l.cfg.CdiDirs) > 0 { + err = addCDIDevices(spec, l.cfg.Devices, cdi.WithSpecDirs(l.cfg.CdiDirs...)) + } else { + err = addCDIDevices(spec, l.cfg.Devices) + } + if err != nil { + return err + } + if err := b.Update(ctx, spec); err != nil { return err } diff --git a/internal/pkg/runtime/launcher/oci/spec_linux.go b/internal/pkg/runtime/launcher/oci/spec_linux.go index 243a8fd8d9..dac2ccdea4 100644 --- a/internal/pkg/runtime/launcher/oci/spec_linux.go +++ b/internal/pkg/runtime/launcher/oci/spec_linux.go @@ -14,6 +14,7 @@ import ( "github.com/apptainer/apptainer/internal/pkg/runtime/launcher" "github.com/apptainer/apptainer/pkg/sylog" + "github.com/opencontainers/runtime-spec/specs-go" ) diff --git a/internal/pkg/runtime/launcher/options.go b/internal/pkg/runtime/launcher/options.go index ef7d2462ae..d9a51fb3b0 100644 --- a/internal/pkg/runtime/launcher/options.go +++ b/internal/pkg/runtime/launcher/options.go @@ -153,6 +153,12 @@ type Options struct { // SysContext holds Docker/OCI image handling configuration. // This will be used by a launcher handling OCI images directly. SysContext *types.SystemContext + + // Devices contains the list of device mappings (if any), e.g. CDI mappings. + Devices []string + + // CdiDirs contains the list of directories in which CDI should look for device definition JSON files + CdiDirs []string } type Option func(co *Options) error @@ -558,3 +564,19 @@ func OptSysContext(sc *types.SystemContext) Option { return nil } } + +// OptDevice sets CDI device mappings to apply. +func OptDevice(op []string) Option { + return func(lo *Options) error { + lo.Devices = op + return nil + } +} + +// OptCdiDirs sets CDI spec search-directories to apply. +func OptCdiDirs(op []string) Option { + return func(lo *Options) error { + lo.CdiDirs = op + return nil + } +} diff --git a/test/cdi/cditemplate.json.tpl b/test/cdi/cditemplate.json.tpl new file mode 100644 index 0000000000..59dd541438 --- /dev/null +++ b/test/cdi/cditemplate.json.tpl @@ -0,0 +1,18 @@ +{ + "cdiVersion": "0.5.0", + "kind": "apptainertesting.sylabs.io/device", + + "devices": [ + { + "name": "TesterDevice", + "containerEdits": { + "deviceNodes": {{tojson .DeviceNodes}}, + "mounts": {{tojson .Mounts}} + } + } + ], + + "containerEdits": { + "env": {{tojson .Env}} + } +} diff --git a/test/cdi/kmsg.json b/test/cdi/kmsg.json new file mode 100644 index 0000000000..78f2035bb7 --- /dev/null +++ b/test/cdi/kmsg.json @@ -0,0 +1,51 @@ +{ + "cdiVersion": "0.5.0", + "kind": "apptainertesting.sylabs.io/device", + + "devices": [ + { + "name": "kmsgDevice", + "containerEdits": { + "deviceNodes": [ + { + "hostPath": "/dev/kmsg", + "path": "/dev/kmsg", + "permissions": "rw", + "uid": 1000, + "gid": 1000 + } + ], + "mounts": [ + { + "containerPath": "/tmpmountforkmsg", + "options": [ + "rw" + ], + "hostPath": "/tmp" + } + ] + } + }, + { + "name": "tmpmountDevice17", + "containerEdits": { + "mounts": [ + { + "containerPath": "/tmpmount17", + "options": [ + "r" + ], + "hostPath": "/tmp" + } + ] + } + } + ], + + "containerEdits": { + "env": [ + "FOO=VALID_SPEC", + "BAR=BARVALUE1" + ] + } +} diff --git a/test/cdi/tmpmount.json b/test/cdi/tmpmount.json new file mode 100644 index 0000000000..77501d5628 --- /dev/null +++ b/test/cdi/tmpmount.json @@ -0,0 +1,45 @@ +{ + "cdiVersion": "0.5.0", + "kind": "apptainertesting.sylabs.io/device", + + "devices": [ + { + "name": "tmpmountDevice1", + "containerEdits": { + "mounts": [ + { + "containerPath": "/tmpmount13", + "options": [ + "rw" + ], + "hostPath": "/tmp" + }, + { + "containerPath": "/tmpmount3", + "options": [ + "rbind", + "nosuid", + "nodev" + ], + "hostPath": "/tmp" + }, + { + "containerPath": "/tmpmount1", + "options": [ + "ro" + ], + "hostPath": "/tmp" + } + ] + } + } + ], + + "containerEdits": { + "env": [ + "ABCD=QWERTY", + "EFGH=ASDFGH", + "IJKL=ZXCVBN" + ] + } +} From 2b2572c0626358d51efc7c00c5b21029e73ee043 Mon Sep 17 00:00:00 2001 From: preminger Date: Thu, 30 Mar 2023 09:54:09 -0400 Subject: [PATCH 072/114] oci: support --hostname (sylabs/singularity#1495) * oci: support --hostname * generalize --hostname inferring --uts across native & oci Signed-off-by: Edita Kizinevic --- CHANGELOG.md | 2 ++ cmd/internal/cli/action_flags.go | 2 +- cmd/internal/cli/actions.go | 12 +++++++++++- e2e/actions/actions.go | 17 +++++++++++++---- e2e/actions/oci.go | 17 +++++++++++++---- .../runtime/launcher/native/launcher_linux.go | 8 ++++++-- .../pkg/runtime/launcher/oci/launcher_linux.go | 12 +++++++++--- 7 files changed, 55 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 11e8c6d3d7..350684f585 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,8 @@ For older changes see the [archived Singularity change log](https://github.com/a OCI mode (`--oci`). Currently supports passing one or more (comma-separated) fully-qualified CDI device names, and those devices will then be made available inside the container. +- OCI mode now supports `--hostname` (requires UTS namespace, therefore this + flag will infer `--uts` if running in OCI mode). ### New Features & Functionality diff --git a/cmd/internal/cli/action_flags.go b/cmd/internal/cli/action_flags.go index c3704c1534..8bf5d8a674 100644 --- a/cmd/internal/cli/action_flags.go +++ b/cmd/internal/cli/action_flags.go @@ -235,7 +235,7 @@ var actionHostnameFlag = cmdline.Flag{ Value: &hostname, DefaultValue: "", Name: "hostname", - Usage: "set container hostname", + Usage: "set container hostname. Infers --uts.", EnvKeys: []string{"HOSTNAME"}, Tag: "", } diff --git a/cmd/internal/cli/actions.go b/cmd/internal/cli/actions.go index 05d66abe12..40a1d9eac9 100644 --- a/cmd/internal/cli/actions.go +++ b/cmd/internal/cli/actions.go @@ -53,7 +53,12 @@ func getCacheHandle(cfg cache.Config) *cache.Handle { return h } -// actionPreRun will run replaceURIWithImage and will also do the proper path unsetting +// actionPreRun will: +// - run replaceURIWithImage; +// - do the proper path unsetting; +// - and implement flag inferences for: +// --compat +// --hostname func actionPreRun(cmd *cobra.Command, args []string) { // For compatibility - we still set USER_PATH so it will be visible in the // container, and can be used there if needed. USER_PATH is not used by @@ -75,6 +80,11 @@ func actionPreRun(cmd *cobra.Command, args []string) { noUmask = true noEval = true } + + // --hostname requires UTS namespace + if len(hostname) > 0 { + utsNamespace = true + } } func handleOCI(ctx context.Context, imgCache *cache.Handle, cmd *cobra.Command, pullFrom string) (string, error) { diff --git a/e2e/actions/actions.go b/e2e/actions/actions.go index 94a8b97f02..1e0f0280cc 100644 --- a/e2e/actions/actions.go +++ b/e2e/actions/actions.go @@ -124,9 +124,10 @@ func (c actionTests) actionExec(t *testing.T) { homePath := filepath.Join("/home", basename) tests := []struct { - name string - argv []string - exit int + name string + argv []string + exit int + wantOutputs []e2e.ApptainerCmdResultOp }{ { name: "NoCommand", @@ -270,6 +271,14 @@ func (c actionTests) actionExec(t *testing.T) { argv: []string{"--no-home", c.env.ImagePath, "ls", "-ld", user.Dir}, exit: 1, }, + { + name: "Hostname", + argv: []string{"--hostname", "whats-in-a-native-name", c.env.ImagePath, "hostname"}, + exit: 0, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ExactMatch, "whats-in-a-native-name"), + }, + }, } for _, tt := range tests { @@ -280,7 +289,7 @@ func (c actionTests) actionExec(t *testing.T) { e2e.WithCommand("exec"), e2e.WithDir("/tmp"), e2e.WithArgs(tt.argv...), - e2e.ExpectExit(tt.exit), + e2e.ExpectExit(tt.exit, tt.wantOutputs...), ) } } diff --git a/e2e/actions/oci.go b/e2e/actions/oci.go index ce0c55bf60..f5431fc88c 100644 --- a/e2e/actions/oci.go +++ b/e2e/actions/oci.go @@ -98,9 +98,10 @@ func (c actionTests) actionOciExec(t *testing.T) { imageRef := "oci-archive:" + c.env.OCIArchivePath tests := []struct { - name string - argv []string - exit int + name string + argv []string + exit int + wantOutputs []e2e.ApptainerCmdResultOp }{ { name: "NoCommand", @@ -147,6 +148,14 @@ func (c actionTests) actionOciExec(t *testing.T) { argv: []string{"--uts", imageRef, "true"}, exit: 0, }, + { + name: "Hostname", + argv: []string{"--hostname", "whats-in-an-oci-name", imageRef, "hostname"}, + exit: 0, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ExactMatch, "whats-in-an-oci-name"), + }, + }, } for _, profile := range e2e.OCIProfiles { t.Run(profile.String(), func(t *testing.T) { @@ -158,7 +167,7 @@ func (c actionTests) actionOciExec(t *testing.T) { e2e.WithCommand("exec"), e2e.WithDir("/tmp"), e2e.WithArgs(tt.argv...), - e2e.ExpectExit(tt.exit), + e2e.ExpectExit(tt.exit, tt.wantOutputs...), ) } }) diff --git a/internal/pkg/runtime/launcher/native/launcher_linux.go b/internal/pkg/runtime/launcher/native/launcher_linux.go index e38c710cc2..305584fe05 100644 --- a/internal/pkg/runtime/launcher/native/launcher_linux.go +++ b/internal/pkg/runtime/launcher/native/launcher_linux.go @@ -301,7 +301,11 @@ func (l *Launcher) Exec(ctx context.Context, image string, process string, args // If user wants to set a hostname, it requires the UTS namespace. if l.cfg.Hostname != "" { - l.cfg.Namespaces.UTS = true + // This is a sanity-check; actionPreRun in actions.go should have prevented this scenario from arising. + if !l.cfg.Namespaces.UTS { + return fmt.Errorf("internal error: trying to set hostname without UTS namespace") + } + l.engineConfig.SetHostname(l.cfg.Hostname) } @@ -388,7 +392,7 @@ func (l *Launcher) Exec(ctx context.Context, image string, process string, args if l.cfg.Boot { l.cfg.Namespaces.UTS = true l.cfg.Namespaces.Net = true - if l.cfg.Hostname == "" { + if len(l.cfg.Hostname) < 1 { l.engineConfig.SetHostname(instanceName) } if !l.cfg.KeepPrivs { diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index 7022e8ef0a..ab7223780b 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -135,9 +135,6 @@ func checkOpts(lo launcher.Options) error { if len(lo.NetworkArgs) > 0 { badOpt = append(badOpt, "NetworkArgs") } - if lo.Hostname != "" { - badOpt = append(badOpt, "Hostname") - } if lo.DNS != "" { badOpt = append(badOpt, "DNS") } @@ -217,6 +214,15 @@ func (l *Launcher) createSpec() (*specs.Spec, error) { spec = addNamespaces(spec, l.cfg.Namespaces) + if len(l.cfg.Hostname) > 0 { + // This is a sanity-check; actionPreRun in actions.go should have prevented this scenario from arising. + if !l.cfg.Namespaces.UTS { + return nil, fmt.Errorf("internal error: trying to set hostname without UTS namespace") + } + + spec.Hostname = l.cfg.Hostname + } + mounts, err := l.getMounts() if err != nil { return nil, err From 2b03310e8bf05ff4a6d684a3ee93bbdccd553ca0 Mon Sep 17 00:00:00 2001 From: preminger Date: Fri, 31 Mar 2023 08:47:59 -0400 Subject: [PATCH 073/114] oci: support --scratch (sylabs/singularity#1498) Signed-off-by: Edita Kizinevic --- CHANGELOG.md | 4 +- e2e/actions/actions.go | 33 +++++++++++--- e2e/actions/oci.go | 43 ++++++++++++++++--- .../runtime/launcher/oci/launcher_linux.go | 3 -- .../pkg/runtime/launcher/oci/mounts_linux.go | 24 +++++++++++ 5 files changed, 93 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 350684f585..2bdd4486cc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -51,7 +51,9 @@ For older changes see the [archived Singularity change log](https://github.com/a fully-qualified CDI device names, and those devices will then be made available inside the container. - OCI mode now supports `--hostname` (requires UTS namespace, therefore this - flag will infer `--uts` if running in OCI mode). + flag will infer `--uts`). +- OCI mode now supports `--scratch` (shorthand: `-S`) to mount a tmpfs scratch + directory in the container. ### New Features & Functionality diff --git a/e2e/actions/actions.go b/e2e/actions/actions.go index 1e0f0280cc..1d8aec2a2a 100644 --- a/e2e/actions/actions.go +++ b/e2e/actions/actions.go @@ -1174,10 +1174,11 @@ func (c actionTests) actionBinds(t *testing.T) { } tests := []struct { - name string - args []string - postRun func(*testing.T) - exit int + name string + args []string + wantOutputs []e2e.ApptainerCmdResultOp + postRun func(*testing.T) + exit int }{ { name: "NonExistentSource", @@ -1740,6 +1741,28 @@ func (c actionTests) actionBinds(t *testing.T) { postRun: checkHostDir(filepath.Join(hostWorkDir, "var_tmp", "canary/dir")), exit: 0, }, + { + name: "IsScratchTmpfs", + args: []string{ + "--scratch", "/name-of-a-scratch", + sandbox, + "mount", + }, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.RegexMatch, `\btmpfs on /name-of-a-scratch\b`), + }, + exit: 0, + }, + { + name: "BindOverScratch", + args: []string{ + "--scratch", "/name-of-a-scratch", + "--bind", hostCanaryDir + ":/name-of-a-scratch", + sandbox, + "test", "-f", "/name-of-a-scratch/file", + }, + exit: 0, + }, { name: "ScratchTmpfsBind", args: []string{ @@ -1872,7 +1895,7 @@ func (c actionTests) actionBinds(t *testing.T) { e2e.WithCommand("exec"), e2e.WithArgs(tt.args...), e2e.PostRun(tt.postRun), - e2e.ExpectExit(tt.exit), + e2e.ExpectExit(tt.exit, tt.wantOutputs...), ) } }) diff --git a/e2e/actions/oci.go b/e2e/actions/oci.go index f5431fc88c..ed5d70f502 100644 --- a/e2e/actions/oci.go +++ b/e2e/actions/oci.go @@ -346,10 +346,11 @@ func (c actionTests) actionOciBinds(t *testing.T) { } tests := []struct { - name string - args []string - postRun func(*testing.T) - exit int + name string + args []string + wantOutputs []e2e.ApptainerCmdResultOp + postRun func(*testing.T) + exit int }{ { name: "NonExistentSource", @@ -448,6 +449,38 @@ func (c actionTests) actionOciBinds(t *testing.T) { postRun: checkHostFile(filepath.Join(hostCanaryDir, "nested")), exit: 0, }, + { + name: "IsScratchTmpfs", + args: []string{ + "--scratch", "/name-of-a-scratch", + imageRef, + "mount", + }, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.RegexMatch, `\btmpfs on /name-of-a-scratch\b`), + }, + exit: 0, + }, + { + name: "BindOverScratch", + args: []string{ + "--scratch", "/name-of-a-scratch", + "--bind", hostCanaryDir + ":/name-of-a-scratch", + imageRef, + "test", "-f", "/name-of-a-scratch/file", + }, + exit: 0, + }, + { + name: "ScratchTmpfsBind", + args: []string{ + "--scratch", "/scratch", + "--bind", hostCanaryDir + ":/scratch/dir", + imageRef, + "test", "-f", "/scratch/dir/file", + }, + exit: 0, + }, // For the --mount variants we are really just verifying the CLI // acceptance of one or more --mount flags. Translation from --mount // strings to BindPath structs is checked in unit tests. The @@ -488,7 +521,7 @@ func (c actionTests) actionOciBinds(t *testing.T) { e2e.WithCommand("exec"), e2e.WithArgs(tt.args...), e2e.PostRun(tt.postRun), - e2e.ExpectExit(tt.exit), + e2e.ExpectExit(tt.exit, tt.wantOutputs...), ) } }) diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index ab7223780b..1e609d7970 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -86,9 +86,6 @@ func checkOpts(lo launcher.Options) error { if len(lo.OverlayPaths) > 0 { badOpt = append(badOpt, "OverlayPaths") } - if len(lo.ScratchDirs) > 0 { - badOpt = append(badOpt, "ScratchDirs") - } if lo.WorkDir != "" { badOpt = append(badOpt, "WorkDir") } diff --git a/internal/pkg/runtime/launcher/oci/mounts_linux.go b/internal/pkg/runtime/launcher/oci/mounts_linux.go index 51c6fddbe7..c8aac4808e 100644 --- a/internal/pkg/runtime/launcher/oci/mounts_linux.go +++ b/internal/pkg/runtime/launcher/oci/mounts_linux.go @@ -40,6 +40,9 @@ func (l *Launcher) getMounts() ([]specs.Mount, error) { if err := l.addHomeMount(mounts); err != nil { return nil, fmt.Errorf("while configuring home mount: %w", err) } + if err := l.addScratchMounts(mounts); err != nil { + return nil, fmt.Errorf("while configuring scratch mount(s): %w", err) + } if err := l.addBindMounts(mounts); err != nil { return nil, fmt.Errorf("while configuring bind mount(s): %w", err) } @@ -210,6 +213,27 @@ func (l *Launcher) addHomeMount(mounts *[]specs.Mount) error { return nil } +// addScratchMounts adds tmpfs mounts for scratch directories in the container. +func (l *Launcher) addScratchMounts(mounts *[]specs.Mount) error { + for _, s := range l.cfg.ScratchDirs { + *mounts = append(*mounts, + specs.Mount{ + Destination: s, + Type: "tmpfs", + Source: "tmpfs", + Options: []string{ + "nosuid", + "relatime", + "nodev", + fmt.Sprintf("size=%dm", l.apptainerConf.SessiondirMaxSize), + }, + }, + ) + } + + return nil +} + func (l *Launcher) addBindMounts(mounts *[]specs.Mount) error { // First get binds from -B/--bind and env var binds, err := bind.ParseBindPath(l.cfg.BindPaths) From c3f315df426b3aafec5b9d92c6d3b1565193e9bb Mon Sep 17 00:00:00 2001 From: preminger Date: Thu, 13 Apr 2023 12:37:03 -0400 Subject: [PATCH 074/114] oci: support --pwd (sylabs/singularity#1558) Signed-off-by: Edita Kizinevic --- CHANGELOG.md | 1 + e2e/actions/oci.go | 8 ++++++++ internal/pkg/runtime/launcher/oci/launcher_linux.go | 3 --- internal/pkg/runtime/launcher/oci/process_linux.go | 4 ++++ 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2bdd4486cc..f2b7a3c0a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -54,6 +54,7 @@ For older changes see the [archived Singularity change log](https://github.com/a flag will infer `--uts`). - OCI mode now supports `--scratch` (shorthand: `-S`) to mount a tmpfs scratch directory in the container. +- Support `--pwd` in OCI mode. ### New Features & Functionality diff --git a/e2e/actions/oci.go b/e2e/actions/oci.go index ed5d70f502..5354af066a 100644 --- a/e2e/actions/oci.go +++ b/e2e/actions/oci.go @@ -156,6 +156,14 @@ func (c actionTests) actionOciExec(t *testing.T) { e2e.ExpectOutput(e2e.ExactMatch, "whats-in-an-oci-name"), }, }, + { + name: "Pwd", + argv: []string{"--pwd", "/tmp", imageRef, "pwd"}, + exit: 0, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ExactMatch, "/tmp"), + }, + }, } for _, profile := range e2e.OCIProfiles { t.Run(profile.String(), func(t *testing.T) { diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index 1e609d7970..489b041e52 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -166,9 +166,6 @@ func checkOpts(lo launcher.Options) error { if lo.ShellPath != "" { badOpt = append(badOpt, "ShellPath") } - if lo.PwdPath != "" { - badOpt = append(badOpt, "PwdPath") - } if lo.Boot { badOpt = append(badOpt, "Boot") diff --git a/internal/pkg/runtime/launcher/oci/process_linux.go b/internal/pkg/runtime/launcher/oci/process_linux.go index 59b268eb9b..25d957ebce 100644 --- a/internal/pkg/runtime/launcher/oci/process_linux.go +++ b/internal/pkg/runtime/launcher/oci/process_linux.go @@ -92,6 +92,10 @@ func getProcessArgs(imageSpec imgspecv1.Image, process string, args []string) [] // getProcessCwd computes the Cwd that the container process should start in. // Currently this is the user's tmpfs home directory (see --containall). func (l *Launcher) getProcessCwd() (dir string, err error) { + if len(l.cfg.PwdPath) > 1 { + return l.cfg.PwdPath, nil + } + if l.cfg.Fakeroot { return "/root", nil } From f6c60a8fe6ab50e602e59f8f466e346145631d17 Mon Sep 17 00:00:00 2001 From: preminger Date: Mon, 3 Apr 2023 10:49:16 -0400 Subject: [PATCH 075/114] oci: support --home (sylabs/singularity#1500) Signed-off-by: Edita Kizinevic --- CHANGELOG.md | 7 ++ e2e/actions/actions.go | 11 ++- e2e/actions/oci.go | 80 +++++++++++++++++++ e2e/internal/e2e/apptainercmd.go | 6 +- .../runtime/launcher/native/launcher_linux.go | 4 +- .../runtime/launcher/oci/launcher_linux.go | 16 +--- .../pkg/runtime/launcher/oci/mounts_linux.go | 67 ++++++++++++---- .../pkg/runtime/launcher/oci/process_linux.go | 16 +--- 8 files changed, 155 insertions(+), 52 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f2b7a3c0a6..cacca387f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -55,6 +55,13 @@ For older changes see the [archived Singularity change log](https://github.com/a - OCI mode now supports `--scratch` (shorthand: `-S`) to mount a tmpfs scratch directory in the container. - Support `--pwd` in OCI mode. +- OCI mode now supports `--home`. Supplying a single location (e.g. + `--home /myhomedir`) will result in a new tmpfs directory being created at the + specified location inside the container, and that dir being set as the + in-container user's home dir. Supplying two locations separated by a colon + (e.g. `--home /home/user:/myhomedir`) will result in the first location on the + host being bind-mounted as the second location in-container, and set as + the in-container user's home dir. ### New Features & Functionality diff --git a/e2e/actions/actions.go b/e2e/actions/actions.go index 1d8aec2a2a..c27859222b 100644 --- a/e2e/actions/actions.go +++ b/e2e/actions/actions.go @@ -104,7 +104,11 @@ func (c actionTests) actionExec(t *testing.T) { if err != nil { t.Fatal(err) } - defer os.RemoveAll(testdata) + t.Cleanup(func() { + if !t.Failed() { + os.RemoveAll(testdata) + } + }) testdataTmp := filepath.Join(testdata, "tmp") if err := os.Mkdir(testdataTmp, 0o755); err != nil { @@ -238,7 +242,10 @@ func (c actionTests) actionExec(t *testing.T) { }, { name: "Home", - argv: []string{"--home", testdata, c.env.ImagePath, "test", "-f", tmpfile.Name()}, + argv: []string{"--home", "/myhomeloc", c.env.ImagePath, "env"}, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.RegexMatch, `\bHOME=/myhomeloc\b`), + }, exit: 0, }, { diff --git a/e2e/actions/oci.go b/e2e/actions/oci.go index 5354af066a..6bb1bfe96e 100644 --- a/e2e/actions/oci.go +++ b/e2e/actions/oci.go @@ -97,6 +97,32 @@ func (c actionTests) actionOciExec(t *testing.T) { imageRef := "oci-archive:" + c.env.OCIArchivePath + // Create a temp testfile + testdata, err := fs.MakeTmpDir(c.env.TestDir, "testdata", 0o755) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if !t.Failed() { + os.RemoveAll(testdata) + } + }) + + testdataTmp := filepath.Join(testdata, "tmp") + if err := os.Mkdir(testdataTmp, 0o755); err != nil { + t.Fatal(err) + } + + // Create a temp testfile + tmpfile, err := fs.MakeTmpFile(testdataTmp, "testApptainerExec.", 0o644) + if err != nil { + t.Fatal(err) + } + tmpfile.Close() + + basename := filepath.Base(tmpfile.Name()) + homePath := filepath.Join("/home", basename) + tests := []struct { name string argv []string @@ -143,6 +169,30 @@ func (c actionTests) actionOciExec(t *testing.T) { argv: []string{imageRef, "/bin/sh", "-c", "touch $HOME"}, exit: 0, }, + { + name: "Home", + argv: []string{"--home", "/myhomeloc", imageRef, "sh", "-c", "env; mount"}, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.RegexMatch, `\bHOME=/myhomeloc\b`), + e2e.ExpectOutput(e2e.RegexMatch, `\btmpfs on /myhomeloc\b`), + }, + exit: 0, + }, + { + name: "HomePath", + argv: []string{"--home", testdataTmp + ":/home", imageRef, "test", "-f", homePath}, + exit: 0, + }, + { + name: "HomeTmp", + argv: []string{"--home", "/tmp", imageRef, "true"}, + exit: 0, + }, + { + name: "HomeTmpExplicit", + argv: []string{"--home", "/tmp:/home", imageRef, "true"}, + exit: 0, + }, { name: "UTSNamespace", argv: []string{"--uts", imageRef, "true"}, @@ -315,11 +365,16 @@ func (c actionTests) actionOciBinds(t *testing.T) { canaryDirBind := hostCanaryDir + ":" + contCanaryDir canaryDirMount := "type=bind,source=" + hostCanaryDir + ",destination=" + contCanaryDir + hostHomeDir := filepath.Join(workspace, "home") + createWorkspaceDirs := func(t *testing.T) { e2e.Privileged(func(t *testing.T) { if err := os.RemoveAll(hostCanaryDir); err != nil && !os.IsNotExist(err) { t.Fatalf("failed to delete canary_dir: %s", err) } + if err := os.RemoveAll(hostHomeDir); err != nil && !os.IsNotExist(err) { + t.Fatalf("failed to delete workspace home: %s", err) + } })(t) if err := fs.Mkdir(hostCanaryDir, 0o777); err != nil { @@ -331,6 +386,9 @@ func (c actionTests) actionOciBinds(t *testing.T) { if err := os.Chmod(hostCanaryFile, 0o777); err != nil { t.Fatalf("failed to apply permissions on canary_file: %s", err) } + if err := fs.Mkdir(hostHomeDir, 0o777); err != nil { + t.Fatalf("failed to create workspace home directory: %s", err) + } } checkHostFn := func(path string, fn func(string) bool) func(*testing.T) { @@ -489,6 +547,28 @@ func (c actionTests) actionOciBinds(t *testing.T) { }, exit: 0, }, + { + name: "CustomHomeOneToOne", + args: []string{ + "--home", hostHomeDir + ":" + hostHomeDir, + "--bind", hostCanaryDir + ":" + filepath.Join(hostHomeDir, "canary121RO"), + imageRef, + "test", "-f", filepath.Join(hostHomeDir, "canary121RO/file"), + }, + postRun: checkHostDir(filepath.Join(hostHomeDir, "canary121RO")), + exit: 0, + }, + { + name: "CustomHomeBind", + args: []string{ + "--home", hostHomeDir + ":/home/e2e", + "--bind", hostCanaryDir + ":/home/e2e/canaryRO", + imageRef, + "test", "-f", "/home/e2e/canaryRO/file", + }, + postRun: checkHostDir(filepath.Join(hostHomeDir, "canaryRO")), + exit: 0, + }, // For the --mount variants we are really just verifying the CLI // acceptance of one or more --mount flags. Translation from --mount // strings to BindPath structs is checked in unit tests. The diff --git a/e2e/internal/e2e/apptainercmd.go b/e2e/internal/e2e/apptainercmd.go index e08bd729c3..23c6bda17c 100644 --- a/e2e/internal/e2e/apptainercmd.go +++ b/e2e/internal/e2e/apptainercmd.go @@ -108,7 +108,7 @@ func (r *ApptainerCmdResult) expectMatch(mt MatchType, stream streamType, patter // get rid of the trailing newline if strings.TrimSuffix(output, "\n") != pattern { return errors.Errorf( - "Command %q:\nExpect %s stream exact match:\n%s\nCommand %s output:\n%s", + "Command %q:\nExpect %s stream exact match:\n%s\nCommand %s stream:\n%s", r.FullCmd, streamName, pattern, streamName, output, ) } @@ -122,7 +122,7 @@ func (r *ApptainerCmdResult) expectMatch(mt MatchType, stream streamType, patter case UnwantedExactMatch: if strings.TrimSuffix(output, "\n") == pattern { return errors.Errorf( - "Command %q:\nExpect %s stream not matching:\n%s\nCommand %s output:\n%s", + "Command %q:\nExpect %s stream not matching:\n%s\nCommand %s stream:\n%s", r.FullCmd, streamName, pattern, streamName, output, ) } @@ -136,7 +136,7 @@ func (r *ApptainerCmdResult) expectMatch(mt MatchType, stream streamType, patter } if !matched { return errors.Errorf( - "Command %q:\nExpect %s stream match regular expression:\n%s\nCommand %s output:\n%s", + "Command %q:\nExpect %s stream match regular expression:\n%s\nCommand %s stream:\n%s", r.FullCmd, streamName, pattern, streamName, output, ) } diff --git a/internal/pkg/runtime/launcher/native/launcher_linux.go b/internal/pkg/runtime/launcher/native/launcher_linux.go index 305584fe05..e260f9de68 100644 --- a/internal/pkg/runtime/launcher/native/launcher_linux.go +++ b/internal/pkg/runtime/launcher/native/launcher_linux.go @@ -793,8 +793,8 @@ func (l *Launcher) setHome() error { // Handle any user request to override the home directory source/dest homeSlice := strings.Split(l.cfg.HomeDir, ":") - if len(homeSlice) > 2 || len(homeSlice) == 0 { - return fmt.Errorf("home argument has incorrect number of elements: %v", len(homeSlice)) + if len(homeSlice) < 1 || len(homeSlice) > 2 { + return fmt.Errorf("home argument has incorrect number of elements: %v", homeSlice) } l.engineConfig.SetHomeSource(homeSlice[0]) if len(homeSlice) == 1 { diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index 489b041e52..17f6aa8ea5 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -27,7 +27,6 @@ import ( "github.com/apptainer/apptainer/internal/pkg/cgroups" "github.com/apptainer/apptainer/internal/pkg/runtime/launcher" "github.com/apptainer/apptainer/internal/pkg/util/fs/files" - "github.com/apptainer/apptainer/internal/pkg/util/user" "github.com/apptainer/apptainer/pkg/ocibundle" "github.com/apptainer/apptainer/pkg/ocibundle/native" "github.com/apptainer/apptainer/pkg/ocibundle/tools" @@ -90,12 +89,6 @@ func checkOpts(lo launcher.Options) error { badOpt = append(badOpt, "WorkDir") } - // Home is always sent from the CLI, and must be valid as an option, but - // CustomHome signifies if it was a user specified value which we don't - // support (yet). - if lo.CustomHome { - badOpt = append(badOpt, "CustomHome") - } if lo.NoHome { badOpt = append(badOpt, "NoHome") } @@ -316,7 +309,7 @@ func (l *Launcher) finalizeSpec(ctx context.Context, b ocibundle.Bundle, spec *s if targetUID == 0 || containerUser { return nil } - // Otherewise, add to the passwd and group files in the container. + // Otherwise, add to the passwd and group files in the container. if err := l.updatePasswdGroup(tools.RootFs(b.Path()).Path(), targetUID, targetGID); err != nil { return err } @@ -331,13 +324,8 @@ func (l *Launcher) updatePasswdGroup(rootfs string, uid, gid uint32) error { containerPasswd := filepath.Join(rootfs, "etc", "passwd") containerGroup := filepath.Join(rootfs, "etc", "group") - pw, err := user.CurrentOriginal() - if err != nil { - return err - } - sylog.Debugf("Updating passwd file: %s", containerPasswd) - content, err := files.Passwd(containerPasswd, pw.Dir, int(uid), nil) + content, err := files.Passwd(containerPasswd, l.cfg.HomeDir, int(uid), nil) if err != nil { sylog.Warningf("%s", err) } else if err := os.WriteFile(containerPasswd, content, 0o755); err != nil { diff --git a/internal/pkg/runtime/launcher/oci/mounts_linux.go b/internal/pkg/runtime/launcher/oci/mounts_linux.go index c8aac4808e..ae23c34623 100644 --- a/internal/pkg/runtime/launcher/oci/mounts_linux.go +++ b/internal/pkg/runtime/launcher/oci/mounts_linux.go @@ -176,29 +176,61 @@ func (l *Launcher) addSysMount(mounts *[]specs.Mount) { // emulating `--compat` / `--containall`, so the user must specifically bind in // their home directory from the host for it to be available. func (l *Launcher) addHomeMount(mounts *[]specs.Mount) error { - if l.cfg.Fakeroot { - *mounts = append(*mounts, - specs.Mount{ - Destination: "/root", - Type: "tmpfs", - Source: "tmpfs", - Options: []string{ - "nosuid", - "relatime", - "mode=755", - fmt.Sprintf("size=%dm", l.apptainerConf.SessiondirMaxSize), - }, - }) - return nil - } - + // Get the host user's data pw, err := user.CurrentOriginal() if err != nil { return err } + + if l.cfg.CustomHome { + // Handle any user request to override the home directory source/dest + homeSlice := strings.Split(l.cfg.HomeDir, ":") + if len(homeSlice) < 1 || len(homeSlice) > 2 { + return fmt.Errorf("home argument has incorrect number of elements: %v", homeSlice) + } + homeSrc := homeSlice[0] + l.cfg.HomeDir = homeSrc + + // User requested more than just a custom home dir; they want to bind-mount a directory in the host to this custom home dir. + // This means the home dir, as viewed from inside the container, is actually the second member of the ":"-separated slice. The first member is actually just the source portion of the requested bind-mount. + if len(homeSlice) > 1 { + homeDest := homeSlice[1] + l.cfg.HomeDir = homeDest + + // Since the home dir is a bind-mount in this case, we don't have to mount a tmpfs directory for the in-container home dir, and we can just do the bind-mount & return. + return addBindMount(mounts, bind.Path{ + Source: homeSrc, + Destination: homeDest, + }) + } + } else { + // If we're running in fake-root mode (and we haven't requested a custom home dir), we do need to create a tmpfs mount for the home dir, but it's a special case (because of its location & permissions), so we handle that here & return. + if l.cfg.Fakeroot { + l.cfg.HomeDir = "/root" + *mounts = append(*mounts, + specs.Mount{ + Destination: "/root", + Type: "tmpfs", + Source: "tmpfs", + Options: []string{ + "nosuid", + "relatime", + "mode=755", + fmt.Sprintf("size=%dm", l.apptainerConf.SessiondirMaxSize), + }, + }) + + return nil + } + + // No fakeroot and no custom home dir - so the in-container home dir will be named the same as the host user's home dir. (Though note that it'll still be a tmpfs mount! It'll get mounted by the catch-all mount append, below.) + l.cfg.HomeDir = pw.Dir + } + + // If we've not hit a special case (bind-mounted custom home dir, or fakeroot), then create a tmpfs mount as a home dir in the requested location (whether it's custom or not; by this point, l.cfg.HomeDir will reflect the right value). *mounts = append(*mounts, specs.Mount{ - Destination: pw.Dir, + Destination: l.cfg.HomeDir, Type: "tmpfs", Source: "tmpfs", Options: []string{ @@ -210,6 +242,7 @@ func (l *Launcher) addHomeMount(mounts *[]specs.Mount) error { fmt.Sprintf("gid=%d", pw.GID), }, }) + return nil } diff --git a/internal/pkg/runtime/launcher/oci/process_linux.go b/internal/pkg/runtime/launcher/oci/process_linux.go index 25d957ebce..902120f0db 100644 --- a/internal/pkg/runtime/launcher/oci/process_linux.go +++ b/internal/pkg/runtime/launcher/oci/process_linux.go @@ -20,7 +20,6 @@ import ( "github.com/apptainer/apptainer/internal/pkg/runtime/engine/config/oci/generate" "github.com/apptainer/apptainer/internal/pkg/util/env" "github.com/apptainer/apptainer/internal/pkg/util/shell/interpreter" - "github.com/apptainer/apptainer/internal/pkg/util/user" imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/opencontainers/runtime-spec/specs-go" "golang.org/x/term" @@ -91,20 +90,9 @@ func getProcessArgs(imageSpec imgspecv1.Image, process string, args []string) [] // getProcessCwd computes the Cwd that the container process should start in. // Currently this is the user's tmpfs home directory (see --containall). +// Because this is called after mounts have already been computed, we can count on l.cfg.HomeDir containing the right value, incorporating any custom home dir overrides (i.e., --home). func (l *Launcher) getProcessCwd() (dir string, err error) { - if len(l.cfg.PwdPath) > 1 { - return l.cfg.PwdPath, nil - } - - if l.cfg.Fakeroot { - return "/root", nil - } - - pw, err := user.CurrentOriginal() - if err != nil { - return "", err - } - return pw.Dir, nil + return l.cfg.HomeDir, nil } // getReverseUserMaps returns uid and gid mappings that re-map container uid to target From 0e2dba6fbcff9cd88d19cd9edf3af8ad3be1ebd9 Mon Sep 17 00:00:00 2001 From: preminger Date: Mon, 3 Apr 2023 10:44:09 -0400 Subject: [PATCH 076/114] oci: improvements to cdi support * added --cdi-dirs to changelog * improved comments in actionOciCdi e2e-test Signed-off-by: Edita Kizinevic --- CHANGELOG.md | 4 ++++ e2e/actions/oci.go | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cacca387f1..b86aa8109a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -50,6 +50,10 @@ For older changes see the [archived Singularity change log](https://github.com/a OCI mode (`--oci`). Currently supports passing one or more (comma-separated) fully-qualified CDI device names, and those devices will then be made available inside the container. +- Added `--cdi-dirs` flag to override the default search locations for CDI + json files, allowing, for example, users who don't have root access on their + host machine to nevertheless create CDI mappings (into containers run with + `--fakeroot`, for example). - OCI mode now supports `--hostname` (requires UTS namespace, therefore this flag will infer `--uts`). - OCI mode now supports `--scratch` (shorthand: `-S`) to mount a tmpfs scratch diff --git a/e2e/actions/oci.go b/e2e/actions/oci.go index 6bb1bfe96e..1f5ed893ce 100644 --- a/e2e/actions/oci.go +++ b/e2e/actions/oci.go @@ -773,6 +773,8 @@ func (c actionTests) actionOciCdi(t *testing.T) { // Generate the command to be executed in the container // Start by printing all environment variables, to test using e2e.ContainMatch conditions later execCmd := "/bin/env" + + // Add commands to test the presence of mapped devices. for _, d := range tt.DeviceNodes { testFlag := "-f" switch d.Type { @@ -781,6 +783,8 @@ func (c actionTests) actionOciCdi(t *testing.T) { } execCmd += fmt.Sprintf(" && test %s %s", testFlag, d.Path) } + + // Add commands to test the presence, and functioning, of mounts. for i, m := range tt.Mounts { // Add a separate teststring echo statement for each mount execCmd += fmt.Sprintf(" && echo %s > %s/testfile_%d", testfileStrings[i], m.ContainerPath, i) @@ -809,6 +813,7 @@ func (c actionTests) actionOciCdi(t *testing.T) { envExpects = append(envExpects, e2e.ExpectOutput(e2e.ContainMatch, e)) } + // Run the subtest. c.env.RunApptainer( t, e2e.AsSubtest(tt.name), From de564cb4b7086135165a2b64060634f9b902123a Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Wed, 5 Apr 2023 12:16:15 +0100 Subject: [PATCH 077/114] fix: oci: explicitly request userns for inner reverse idmap When running in --oci mode as a normal user, Singularity sets up an outer userns that provides an unprivileged root. When --fakeroot is not also requested, runc / crun then map back to the user's own uid/gid via an inner reverse mapping specified in config.json. This mapping was not being applied correctly, as a userns was not requested in the config.json for all situations where a reverse mapping was requested. Ensure that if we specify a mapping in config.json then we request a userns so that it is applied. Cover this behavior in an e2e-test by writing to a host dir from the container, and checking ownership of the file on the host. Fixes sylabs/singularity#1517 Signed-off-by: Edita Kizinevic --- e2e/actions/actions.go | 1 + e2e/actions/oci.go | 43 +++++++++++++++++++ .../runtime/launcher/oci/launcher_linux.go | 5 +++ internal/pkg/util/fs/helper.go | 11 ++++- internal/pkg/util/fs/helper_linux_test.go | 11 ++++- 5 files changed, 69 insertions(+), 2 deletions(-) diff --git a/e2e/actions/actions.go b/e2e/actions/actions.go index c27859222b..77cc34cddb 100644 --- a/e2e/actions/actions.go +++ b/e2e/actions/actions.go @@ -2957,5 +2957,6 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { "ociNetwork": c.actionOciNetwork, // apptainer exec --oci --net "ociBinds": c.actionOciBinds, // apptainer exec --oci --bind / --mount "ociCdi": c.actionOciCdi, // apptainer exec --oci --cdi + "ociIDMaps": c.actionOciIDMaps, // check uid/gid mapping on host for --oci as user / --fakeroot } } diff --git a/e2e/actions/oci.go b/e2e/actions/oci.go index 1f5ed893ce..68a7b5ccec 100644 --- a/e2e/actions/oci.go +++ b/e2e/actions/oci.go @@ -835,3 +835,46 @@ func (c actionTests) actionOciCdi(t *testing.T) { }) } } + +// Check that both root via fakeroot and user without fakeroot are mapped to +// uid/gid on host, by writing a file out to host and checking ownership. +func (c actionTests) actionOciIDMaps(t *testing.T) { + e2e.EnsureOCIArchive(t, c.env) + imageRef := "oci-archive:" + c.env.OCIArchivePath + + bindDir, cleanup := e2e.MakeTempDir(t, c.env.TestDir, "usermap", "") + t.Cleanup(func() { + if !t.Failed() { + cleanup(t) + } + }) + + for _, profile := range []e2e.Profile{e2e.OCIUserProfile, e2e.OCIFakerootProfile} { + t.Run(profile.String(), func(t *testing.T) { + cmdArgs := []string{ + "-B", fmt.Sprintf("%s:/test", bindDir), + imageRef, + "/bin/touch", fmt.Sprintf("/test/%s", profile.String()), + } + c.env.RunApptainer( + t, + e2e.AsSubtest(profile.String()), + e2e.WithProfile(profile), + e2e.WithCommand("exec"), + e2e.WithArgs(cmdArgs...), + e2e.ExpectExit(0), + e2e.PostRun(func(t *testing.T) { + fp := filepath.Join(bindDir, profile.String()) + expectUID := profile.HostUser(t).UID + expectGID := profile.HostUser(t).GID + if !fs.IsOwner(fp, expectUID) { + t.Errorf("%s not owned by uid %d", fp, expectUID) + } + if !fs.IsGroup(fp, expectGID) { + t.Errorf("%s not owned by gid %d", fp, expectGID) + } + }), + ) + }) + } +} diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index 17f6aa8ea5..3098eac168 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -278,6 +278,11 @@ func (l *Launcher) finalizeSpec(ctx context.Context, b ocibundle.Bundle, spec *s } spec.Linux.UIDMappings = uidMap spec.Linux.GIDMappings = gidMap + // Must add userns to the runc/crun applied config for the inner reverse uid/gid mapping to work. + spec.Linux.Namespaces = append( + spec.Linux.Namespaces, + specs.LinuxNamespace{Type: specs.UserNamespace}, + ) } u := specs.User{ diff --git a/internal/pkg/util/fs/helper.go b/internal/pkg/util/fs/helper.go index f44b5032e1..3af4f7d7b2 100644 --- a/internal/pkg/util/fs/helper.go +++ b/internal/pkg/util/fs/helper.go @@ -92,7 +92,7 @@ func IsLink(name string) bool { return info.Mode()&os.ModeSymlink != 0 } -// IsOwner check if name component is owned by user identified with uid. +// IsOwner checks if named file is owned by user identified with uid. func IsOwner(name string, uid uint32) bool { info, err := os.Stat(name) if err != nil { @@ -101,6 +101,15 @@ func IsOwner(name string, uid uint32) bool { return info.Sys().(*syscall.Stat_t).Uid == uid } +// IsGroup checks if named file is owned by group identified with gid. +func IsGroup(name string, gid uint32) bool { + info, err := os.Stat(name) + if err != nil { + return false + } + return info.Sys().(*syscall.Stat_t).Gid == gid +} + // IsExec check if name component has executable bit permission set. func IsExec(name string) bool { info, err := os.Stat(name) diff --git a/internal/pkg/util/fs/helper_linux_test.go b/internal/pkg/util/fs/helper_linux_test.go index fa07485722..4af90fe61d 100644 --- a/internal/pkg/util/fs/helper_linux_test.go +++ b/internal/pkg/util/fs/helper_linux_test.go @@ -167,7 +167,16 @@ func TestIsOwner(t *testing.T) { defer test.ResetPrivilege(t) if IsOwner("/etc/passwd", 0) != true { - t.Errorf("IsOwner returns false for /etc/passwd owner") + t.Errorf("IsOwner returns false for /etc/passwd root ownership") + } +} + +func TestIsGroup(t *testing.T) { + test.DropPrivilege(t) + defer test.ResetPrivilege(t) + + if IsGroup("/etc/passwd", 0) != true { + t.Errorf("IsGroup returns false for /etc/passwd root group ownership") } } From ff5bd89ecebe5e7143c4aeb7f63df9f0343b2408 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Tue, 11 Apr 2023 14:12:42 +0100 Subject: [PATCH 078/114] fix: oci: reverse uid/gid maps now honour target IDs In `getReverseUserMaps`,we were taking `targetUID` and `targetGID` parameters that define the UID and GID the container will be entereed as. However, the mappings returned were based on the current host UID and GID, and not the target IDs. Ensure that the user maps use the `targetUID` and `targetGID`. Split the function up so that the core computation of the user maps can be tested. The tests help to explain what the intended functionality here is... which is beneficial as it's somewhat complex. Fixes sylabs/singularity#1519 Signed-off-by: Edita Kizinevic --- .../runtime/launcher/oci/launcher_linux.go | 2 +- .../pkg/runtime/launcher/oci/process_linux.go | 43 ++++++------- .../launcher/oci/process_linux_test.go | 60 +++++++++++++++++++ 3 files changed, 84 insertions(+), 21 deletions(-) diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index 3098eac168..dbfd1b026a 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -272,7 +272,7 @@ func (l *Launcher) finalizeSpec(ctx context.Context, b ocibundle.Bundle, spec *s } if targetUID != 0 && currentUID != 0 { - uidMap, gidMap, err := l.getReverseUserMaps(targetUID, targetGID) + uidMap, gidMap, err := getReverseUserMaps(currentUID, targetUID, targetGID) if err != nil { return err } diff --git a/internal/pkg/runtime/launcher/oci/process_linux.go b/internal/pkg/runtime/launcher/oci/process_linux.go index 902120f0db..855415a266 100644 --- a/internal/pkg/runtime/launcher/oci/process_linux.go +++ b/internal/pkg/runtime/launcher/oci/process_linux.go @@ -100,11 +100,9 @@ func (l *Launcher) getProcessCwd() (dir string, err error) { // userns from which the OCI runtime is launched. // // e.g. host 1001 -> fakeroot userns 0 -> container targetUID -func (l *Launcher) getReverseUserMaps(targetUID, targetGID uint32) (uidMap, gidMap []specs.LinuxIDMapping, err error) { - uid := uint32(os.Getuid()) - gid := uint32(os.Getgid()) +func getReverseUserMaps(hostUID, targetUID, targetGID uint32) (uidMap, gidMap []specs.LinuxIDMapping, err error) { // Get user's configured subuid & subgid ranges - subuidRange, err := fakeroot.GetIDRange(fakeroot.SubUIDFile, uid) + subuidRange, err := fakeroot.GetIDRange(fakeroot.SubUIDFile, hostUID) if err != nil { return nil, nil, err } @@ -112,7 +110,7 @@ func (l *Launcher) getReverseUserMaps(targetUID, targetGID uint32) (uidMap, gidM if subuidRange.Size < 65536 { return nil, nil, fmt.Errorf("subuid range size (%d) must be at least 65536", subuidRange.Size) } - subgidRange, err := fakeroot.GetIDRange(fakeroot.SubGIDFile, uid) + subgidRange, err := fakeroot.GetIDRange(fakeroot.SubGIDFile, hostUID) if err != nil { return nil, nil, err } @@ -120,22 +118,27 @@ func (l *Launcher) getReverseUserMaps(targetUID, targetGID uint32) (uidMap, gidM return nil, nil, fmt.Errorf("subuid range size (%d) must be at least 65536", subgidRange.Size) } - if uid < subuidRange.Size { + uidMap, gidMap = reverseMapByRange(targetUID, targetGID, *subuidRange, *subgidRange) + return uidMap, gidMap, nil +} + +func reverseMapByRange(targetUID, targetGID uint32, subuidRange, subgidRange specs.LinuxIDMapping) (uidMap, gidMap []specs.LinuxIDMapping) { + if targetUID < subuidRange.Size { uidMap = []specs.LinuxIDMapping{ { ContainerID: 0, HostID: 1, - Size: uid, + Size: targetUID, }, { - ContainerID: uid, + ContainerID: targetUID, HostID: 0, Size: 1, }, { - ContainerID: uid + 1, - HostID: uid + 1, - Size: subuidRange.Size - uid, + ContainerID: targetUID + 1, + HostID: targetUID + 1, + Size: subuidRange.Size - targetUID, }, } } else { @@ -146,29 +149,29 @@ func (l *Launcher) getReverseUserMaps(targetUID, targetGID uint32) (uidMap, gidM Size: subuidRange.Size, }, { - ContainerID: uid, + ContainerID: targetUID, HostID: 0, Size: 1, }, } } - if gid < subgidRange.Size { + if targetGID < subgidRange.Size { gidMap = []specs.LinuxIDMapping{ { ContainerID: 0, HostID: 1, - Size: gid, + Size: targetGID, }, { - ContainerID: gid, + ContainerID: targetGID, HostID: 0, Size: 1, }, { - ContainerID: gid + 1, - HostID: gid + 1, - Size: subgidRange.Size - gid, + ContainerID: targetGID + 1, + HostID: targetGID + 1, + Size: subgidRange.Size - targetGID, }, } } else { @@ -179,14 +182,14 @@ func (l *Launcher) getReverseUserMaps(targetUID, targetGID uint32) (uidMap, gidM Size: subgidRange.Size, }, { - ContainerID: gid, + ContainerID: targetGID, HostID: 0, Size: 1, }, } } - return uidMap, gidMap, nil + return uidMap, gidMap } // getProcessEnv combines the image config ENV with the ENV requested at runtime. diff --git a/internal/pkg/runtime/launcher/oci/process_linux_test.go b/internal/pkg/runtime/launcher/oci/process_linux_test.go index 6c4050e159..e2251881a6 100644 --- a/internal/pkg/runtime/launcher/oci/process_linux_test.go +++ b/internal/pkg/runtime/launcher/oci/process_linux_test.go @@ -17,6 +17,7 @@ import ( "testing" imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/opencontainers/runtime-spec/specs-go" ) func TestApptainerEnvMap(t *testing.T) { @@ -380,3 +381,62 @@ func TestGetProcessEnv(t *testing.T) { }) } } + +func TestLauncher_reverseMapByRange(t *testing.T) { + tests := []struct { + name string + targetUID uint32 + targetGID uint32 + subUIDMap specs.LinuxIDMapping + subGIDMap specs.LinuxIDMapping + wantUIDMap []specs.LinuxIDMapping + wantGIDMap []specs.LinuxIDMapping + wantErr bool + }{ + { + // TargetID is smaller than size of subuid/subgid map. + name: "LowTargetID", + targetUID: 1000, + targetGID: 2000, + subUIDMap: specs.LinuxIDMapping{HostID: 1000, ContainerID: 100000, Size: 65536}, + subGIDMap: specs.LinuxIDMapping{HostID: 2000, ContainerID: 200000, Size: 65536}, + wantUIDMap: []specs.LinuxIDMapping{ + {ContainerID: 0, HostID: 1, Size: 1000}, + {ContainerID: 1000, HostID: 0, Size: 1}, + {ContainerID: 1001, HostID: 1001, Size: 64536}, + }, + wantGIDMap: []specs.LinuxIDMapping{ + {ContainerID: 0, HostID: 1, Size: 2000}, + {ContainerID: 2000, HostID: 0, Size: 1}, + {ContainerID: 2001, HostID: 2001, Size: 63536}, + }, + }, + { + // TargetID is higher than size of subuid/subgid map. + name: "HighTargetID", + targetUID: 70000, + targetGID: 80000, + subUIDMap: specs.LinuxIDMapping{HostID: 1000, ContainerID: 100000, Size: 65536}, + subGIDMap: specs.LinuxIDMapping{HostID: 2000, ContainerID: 200000, Size: 65536}, + wantUIDMap: []specs.LinuxIDMapping{ + {ContainerID: 0, HostID: 1, Size: 65536}, + {ContainerID: 70000, HostID: 0, Size: 1}, + }, + wantGIDMap: []specs.LinuxIDMapping{ + {ContainerID: 0, HostID: 1, Size: 65536}, + {ContainerID: 80000, HostID: 0, Size: 1}, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotUIDMap, gotGIDMap := reverseMapByRange(tt.targetUID, tt.targetGID, tt.subUIDMap, tt.subGIDMap) + if !reflect.DeepEqual(gotUIDMap, tt.wantUIDMap) { + t.Errorf("Launcher.getReverseUserMaps() gotUidMap = %v, want %v", gotUIDMap, tt.wantUIDMap) + } + if !reflect.DeepEqual(gotGIDMap, tt.wantGIDMap) { + t.Errorf("Launcher.getReverseUserMaps() gotGidMap = %v, want %v", gotGIDMap, tt.wantGIDMap) + } + }) + } +} From 569b626d5b05db8d8d4f013aa54e5562f9865968 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Tue, 11 Apr 2023 15:45:36 +0100 Subject: [PATCH 079/114] fix: oci: enter cgroup before executing crun as non-root When executed from a root-owned cgroup, such as the session scope resulting from a bare ssh login, crun will fail to create our requested container cgroup. If we are running as non-root, create and move into a user-owned cgroup, so that there's a common user-owned ancestor. This avoids the `crun` error. Note that no workaround is needed for `runc` as it is able to create the requested container cgroup without any issue. Fixes sylabs/singularity#1538 Signed-off-by: Edita Kizinevic --- .../runtime/launcher/oci/launcher_linux.go | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index dbfd1b026a..1278a9bc1e 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -359,6 +359,11 @@ func (l *Launcher) Exec(ctx context.Context, image string, process string, args return fmt.Errorf("launcher SysContext must be set for OCI image handling") } + // If we need to, enter a new cgroup to workaround an issue with crun container cgroup creation (#1538). + if err := l.crunNestCgroup(); err != nil { + return fmt.Errorf("while applying crun cgroup workaround: %w", err) + } + bundleDir, err := os.MkdirTemp("", "oci-bundle") if err != nil { return nil @@ -440,6 +445,38 @@ func (l *Launcher) getCgroup() (path string, resources *specs.LinuxResources, er return path, resources, nil } +// crunNestCgroup will check whether we are using crun, and enter a cgroup if running as a non-root user. +// This is required to satisfy a common user-owned ancestor cgroup requirement on e.g. bare ssh logins. +// See: https://github.com/sylabs/singularity/issues/1538 +func (l *Launcher) crunNestCgroup() error { + r, err := runtime() + if err != nil { + return err + } + + // No workaround required for runc. + if filepath.Base(r) == "runc" { + return nil + } + + // No workaround required if we are run as root. + if os.Getuid() == 0 { + return nil + } + + // We are running crun as a user. Enter a cgroup now. + pid := os.Getpid() + sylog.Debugf("crun workaround - adding process %d to sibling cgroup", pid) + manager, err := cgroups.NewManagerWithSpec(&specs.LinuxResources{}, pid, "", l.apptainerConf.SystemdCgroups) + if err != nil { + return fmt.Errorf("couldn't create cgroup manager: %w", err) + } + cgPath, _ := manager.GetCgroupRelPath() + sylog.Debugf("In sibling cgroup: %s", cgPath) + + return nil +} + func mergeMap(a map[string]string, b map[string]string) map[string]string { for k, v := range b { a[k] = v From 3ee67e0110a7d1754f1b97afa3a2b30357dc47a1 Mon Sep 17 00:00:00 2001 From: Omer Preminger Date: Wed, 12 Apr 2023 19:58:39 -0400 Subject: [PATCH 080/114] fix: TERM not set in OCI containers Signed-off-by: Edita Kizinevic --- e2e/docker/docker.go | 1 + e2e/docker/regressions.go | 59 +++++++++++++++++++ .../pkg/runtime/launcher/oci/process_linux.go | 7 +++ 3 files changed, 67 insertions(+) diff --git a/e2e/docker/docker.go b/e2e/docker/docker.go index 0f2014403d..01d885b7e9 100644 --- a/e2e/docker/docker.go +++ b/e2e/docker/docker.go @@ -957,6 +957,7 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { // Regressions t.Run("issue 4524", c.issue4524) t.Run("issue 1286", c.issue1286) + t.Run("issue 1528", c.issue1528) }, // Tests that are especially slow, or run against a local docker // registry, can be run in parallel, with `--disable-cache` used within diff --git a/e2e/docker/regressions.go b/e2e/docker/regressions.go index 4bdee283c7..dc2cec4129 100644 --- a/e2e/docker/regressions.go +++ b/e2e/docker/regressions.go @@ -227,3 +227,62 @@ func (c ctx) issue1286(t *testing.T) { ) } } + +// https://github.com/sylabs/singularity/issues/1528 +// Check that host's TERM value gets passed to OCI container. +func (c ctx) issue1528(t *testing.T) { + e2e.EnsureOCIArchive(t, c.env) + + imageRef := "oci-archive:" + c.env.OCIArchivePath + + _, wasHostTermSet := os.LookupEnv("TERM") + if !wasHostTermSet { + if err := os.Setenv("TERM", "xterm"); err != nil { + t.Errorf("could not set TERM environment variable on host") + } + defer os.Unsetenv("TERM") + } + + singEnvTermPrevious, wasHostSingEnvTermSet := os.LookupEnv("APPTAINERENV_TERM") + if wasHostSingEnvTermSet { + if err := os.Unsetenv("APPTAINERENV_TERM"); err != nil { + t.Errorf("could not unset APPTAINERENV_TERM environment variable on host") + } + defer os.Setenv("APPTAINERENV_TERM", singEnvTermPrevious) + } else { + defer os.Unsetenv("APPTAINERENV_TERM") + } + + envTerm := os.Getenv("TERM") + wantTermString := fmt.Sprintf("TERM=%s\n", envTerm) + for _, profile := range e2e.OCIProfiles { + t.Run(profile.String(), func(t *testing.T) { + c.env.RunApptainer( + t, + e2e.AsSubtest("issue1528"), + e2e.WithProfile(profile), + e2e.WithCommand("exec"), + e2e.WithArgs(imageRef, "env"), + e2e.ExpectExit(0, e2e.ExpectOutput(e2e.ContainMatch, wantTermString)), + ) + }) + } + + singEnvTerm := envTerm + "testsuffix" + if err := os.Setenv("APPTAINERENV_TERM", singEnvTerm); err != nil { + t.Errorf("could not set APPTAINERENV_TERM environment variable on host") + } + wantTermString = fmt.Sprintf("TERM=%s\n", singEnvTerm) + for _, profile := range e2e.OCIProfiles { + t.Run(profile.String(), func(t *testing.T) { + c.env.RunApptainer( + t, + e2e.AsSubtest("issue1528override"), + e2e.WithProfile(profile), + e2e.WithCommand("exec"), + e2e.WithArgs(imageRef, "env"), + e2e.ExpectExit(0, e2e.ExpectOutput(e2e.ContainMatch, wantTermString)), + ) + }) + } +} diff --git a/internal/pkg/runtime/launcher/oci/process_linux.go b/internal/pkg/runtime/launcher/oci/process_linux.go index 855415a266..78402e320f 100644 --- a/internal/pkg/runtime/launcher/oci/process_linux.go +++ b/internal/pkg/runtime/launcher/oci/process_linux.go @@ -31,6 +31,13 @@ func (l *Launcher) getProcess(ctx context.Context, imgSpec imgspecv1.Image, imag // Assemble the runtime & user-requested environment, which will be merged // with the image ENV and set in the container at runtime. rtEnv := defaultEnv(image, bundle) + + // Propagate TERM from host. Doing this here means it can be overridden by APPTAINERENV_TERM. + hostTerm, isHostTermSet := os.LookupEnv("TERM") + if isHostTermSet { + rtEnv["TERM"] = hostTerm + } + // APPTAINERENV_ has lowest priority rtEnv = mergeMap(rtEnv, apptainerEnvMap()) // --env-file can override APPTAINERENV_ From 27de69c455d095ccaa183ce275d91b92a03ddf10 Mon Sep 17 00:00:00 2001 From: Omer Preminger Date: Wed, 12 Apr 2023 13:29:10 -0400 Subject: [PATCH 081/114] oci: support resolv.conf and --dns Signed-off-by: Edita Kizinevic --- CHANGELOG.md | 4 ++ .../runtime/launcher/oci/launcher_linux.go | 40 +++++++++++++++++-- 2 files changed, 41 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b86aa8109a..185aece478 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -66,6 +66,10 @@ For older changes see the [archived Singularity change log](https://github.com/a (e.g. `--home /home/user:/myhomedir`) will result in the first location on the host being bind-mounted as the second location in-container, and set as the in-container user's home dir. +- OCI mode now handles `--dns` and `resolv.conf` on par with native mode: the + `--dns` flag can be used to pass a comma-separated list of DNS servers that + will be used in the container; if this flag is not used, the container will + use the same `resolv.conf` settings as the host. ### New Features & Functionality diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index 1278a9bc1e..84d90fec55 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -16,6 +16,7 @@ import ( "context" "errors" "fmt" + "net" "os" "os/exec" "path/filepath" @@ -125,9 +126,6 @@ func checkOpts(lo launcher.Options) error { if len(lo.NetworkArgs) > 0 { badOpt = append(badOpt, "NetworkArgs") } - if lo.DNS != "" { - badOpt = append(badOpt, "DNS") - } if lo.AddCaps != "" { badOpt = append(badOpt, "AddCaps") @@ -309,6 +307,11 @@ func (l *Launcher) finalizeSpec(ctx context.Context, b ocibundle.Bundle, spec *s return err } + // Prepare DNS settings for the container. + if err := l.prepareResolvConf(tools.RootFs(b.Path()).Path()); err != nil { + return err + } + // If we are entering as root, or a USER defined in the container, then passwd/group // information should be present already. if targetUID == 0 || containerUser { @@ -318,6 +321,7 @@ func (l *Launcher) finalizeSpec(ctx context.Context, b ocibundle.Bundle, spec *s if err := l.updatePasswdGroup(tools.RootFs(b.Path()).Path(), targetUID, targetGID); err != nil { return err } + return nil } @@ -348,6 +352,36 @@ func (l *Launcher) updatePasswdGroup(rootfs string, uid, gid uint32) error { return nil } +func (l *Launcher) prepareResolvConf(rootfs string) error { + hostResolvConfPath := "/etc/resolv.conf" + containerResolvConfPath := filepath.Join(rootfs, "etc", "resolv.conf") + + var resolvConfData []byte + var err error + if len(l.cfg.DNS) > 0 { + dns := strings.Replace(l.cfg.DNS, " ", "", -1) + ips := strings.Split(dns, ",") + for _, ip := range ips { + if net.ParseIP(ip) == nil { + return fmt.Errorf("DNS nameserver %v is not a valid IP address", ip) + } + line := fmt.Sprintf("nameserver %s\n", ip) + resolvConfData = append(resolvConfData, line...) + } + } else { + resolvConfData, err = os.ReadFile(hostResolvConfPath) + if err != nil { + return fmt.Errorf("could not read host's resolv.conf file: %w", err) + } + } + + if err := os.WriteFile(containerResolvConfPath, resolvConfData, 0o755); err != nil { + return fmt.Errorf("while writing container's resolv.conf file: %v", err) + } + + return nil +} + // Exec will interactively execute a container via the runc low-level runtime. // image is a reference to an OCI image, e.g. docker://ubuntu or oci:/tmp/mycontainer func (l *Launcher) Exec(ctx context.Context, image string, process string, args []string, instanceName string) error { From 98039877411ead71617810b8cea9698cb8f56331 Mon Sep 17 00:00:00 2001 From: Omer Preminger Date: Wed, 12 Apr 2023 18:06:31 -0400 Subject: [PATCH 082/114] warn instead of failing when container lacks /etc (cf. issue1286) Signed-off-by: Edita Kizinevic --- internal/pkg/runtime/launcher/oci/launcher_linux.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index 84d90fec55..49ae11220c 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -354,6 +354,7 @@ func (l *Launcher) updatePasswdGroup(rootfs string, uid, gid uint32) error { func (l *Launcher) prepareResolvConf(rootfs string) error { hostResolvConfPath := "/etc/resolv.conf" + containerEtc := filepath.Join(rootfs, "etc") containerResolvConfPath := filepath.Join(rootfs, "etc", "resolv.conf") var resolvConfData []byte @@ -375,6 +376,12 @@ func (l *Launcher) prepareResolvConf(rootfs string) error { } } + stat, err := os.Stat(containerEtc) + if os.IsNotExist(err) || !stat.IsDir() { + sylog.Warningf("container does not contain an /etc directory; skipping resolve.conf configuration") + return nil + } + if err := os.WriteFile(containerResolvConfPath, resolvConfData, 0o755); err != nil { return fmt.Errorf("while writing container's resolv.conf file: %v", err) } From 359200e598dc9071dff2ddae26bf9caa8d582ee9 Mon Sep 17 00:00:00 2001 From: Omer Preminger Date: Wed, 12 Apr 2023 18:14:45 -0400 Subject: [PATCH 083/114] added rudimentary tests for --dns Signed-off-by: Edita Kizinevic --- e2e/actions/actions.go | 16 ++++++++++++++++ e2e/actions/oci.go | 16 ++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/e2e/actions/actions.go b/e2e/actions/actions.go index 77cc34cddb..3d73090ba7 100644 --- a/e2e/actions/actions.go +++ b/e2e/actions/actions.go @@ -286,6 +286,22 @@ func (c actionTests) actionExec(t *testing.T) { e2e.ExpectOutput(e2e.ExactMatch, "whats-in-a-native-name"), }, }, + { + name: "ResolvConfGoogle", + argv: []string{"--dns", "8.8.8.8,8.8.4.4", c.env.ImagePath, "nslookup", "w3.org"}, + exit: 0, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.RegexMatch, `^(\s*)Server:(\s+)(8\.8\.8\.8|8\.8\.4\.4)(\s*)\n`), + }, + }, + { + name: "ResolvConfCloudflare", + argv: []string{"--dns", "1.1.1.1", c.env.ImagePath, "nslookup", "w3.org"}, + exit: 0, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.RegexMatch, `^(\s*)Server:(\s+)(1\.1\.1\.1)(\s*)\n`), + }, + }, } for _, tt := range tests { diff --git a/e2e/actions/oci.go b/e2e/actions/oci.go index 68a7b5ccec..f7d630d201 100644 --- a/e2e/actions/oci.go +++ b/e2e/actions/oci.go @@ -214,6 +214,22 @@ func (c actionTests) actionOciExec(t *testing.T) { e2e.ExpectOutput(e2e.ExactMatch, "/tmp"), }, }, + { + name: "ResolvConfGoogle", + argv: []string{"--dns", "8.8.8.8,8.8.4.4", imageRef, "nslookup", "w3.org"}, + exit: 0, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.RegexMatch, `^(\s*)Server:(\s+)(8\.8\.8\.8|8\.8\.4\.4)(\s*)\n`), + }, + }, + { + name: "ResolvConfCloudflare", + argv: []string{"--dns", "1.1.1.1", imageRef, "nslookup", "w3.org"}, + exit: 0, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.RegexMatch, `^(\s*)Server:(\s+)(1\.1\.1\.1)(\s*)\n`), + }, + }, } for _, profile := range e2e.OCIProfiles { t.Run(profile.String(), func(t *testing.T) { From 6cc1809e4f929e49e4714469fd9ec7e4cbd2ec09 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Thu, 13 Apr 2023 10:16:49 +0100 Subject: [PATCH 084/114] fix: oci: honour config passwd / config group directives Ensure that the `config passwd` and `config group` directives in `singularity.conf` are honoured in `--oci` mode. Add e2e config tests for OCI mode that test this. Signed-off-by: Edita Kizinevic --- e2e/config/config.go | 1 + e2e/config/oci.go | 273 ++++++++++++++++++ .../runtime/launcher/oci/launcher_linux.go | 32 +- 3 files changed, 294 insertions(+), 12 deletions(-) create mode 100644 e2e/config/oci.go diff --git a/e2e/config/config.go b/e2e/config/config.go index 8d58cf203e..0b365877b8 100644 --- a/e2e/config/config.go +++ b/e2e/config/config.go @@ -1003,5 +1003,6 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { "config file": c.configFile, // test --config file option "config global": np(c.configGlobal), // test various global configuration "config global combination": np(c.configGlobalCombination), // test various global configuration with combination + "oci config global": np(c.ociConfigGlobal), // test various global configuration for OCI mode } } diff --git a/e2e/config/oci.go b/e2e/config/oci.go new file mode 100644 index 0000000000..bee10dcca1 --- /dev/null +++ b/e2e/config/oci.go @@ -0,0 +1,273 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2019-2023, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package config + +import ( + "fmt" + "testing" + + "github.com/apptainer/apptainer/e2e/internal/e2e" +) + +//nolint:maintidx +func (c configTests) ociConfigGlobal(t *testing.T) { + e2e.EnsureOCIArchive(t, c.env) + archiveRef := "oci-archive:" + c.env.OCIArchivePath + + setDirective := func(t *testing.T, directive, value string) { + c.env.RunApptainer( + t, + e2e.WithProfile(e2e.RootProfile), + e2e.WithCommand("config global"), + e2e.WithArgs("--set", directive, value), + e2e.ExpectExit(0), + ) + } + resetDirective := func(t *testing.T, directive string) { + c.env.RunApptainer( + t, + e2e.WithProfile(e2e.RootProfile), + e2e.WithCommand("config global"), + e2e.WithArgs("--reset", directive), + e2e.ExpectExit(0), + ) + } + + tests := []struct { + name string + argv []string + profile e2e.Profile + addRequirementsFn func(*testing.T) + cwd string + directive string + directiveValue string + exit int + resultOp e2e.ApptainerCmdResultOp + }{ + // { + // name: "AllowPidNsNo", + // argv: []string{"--pid", "--no-init", archiveRef, "/bin/sh", "-c", "echo $$"}, + // profile: e2e.OCIUserProfile, + // directive: "allow pid ns", + // directiveValue: "no", + // exit: 0, + // resultOp: e2e.ExpectOutput(e2e.UnwantedExactMatch, "1"), + // }, + // { + // name: "AllowPidNsYes", + // argv: []string{"--pid", "--no-init", archiveRef, "/bin/sh", "-c", "echo $$"}, + // profile: e2e.OCIUserProfile, + // directive: "allow pid ns", + // directiveValue: "yes", + // exit: 0, + // resultOp: e2e.ExpectOutput(e2e.ExactMatch, "1"), + // }, + { + name: "ConfigPasswdNo", + argv: []string{archiveRef, "grep", + fmt.Sprintf("%s:x:%d", e2e.OCIUserProfile.ContainerUser(t).Name, e2e.OCIUserProfile.ContainerUser(t).UID), + "/etc/passwd"}, + profile: e2e.OCIUserProfile, + directive: "config passwd", + directiveValue: "no", + exit: 1, + }, + { + name: "ConfigPasswdYes", + argv: []string{archiveRef, "grep", + fmt.Sprintf("%s:x:%d", e2e.OCIUserProfile.ContainerUser(t).Name, e2e.OCIUserProfile.ContainerUser(t).UID), + "/etc/passwd"}, + profile: e2e.OCIUserProfile, + directive: "config passwd", + directiveValue: "yes", + exit: 0, + }, + { + name: "ConfigGroupNo", + argv: []string{archiveRef, "grep", + fmt.Sprintf("x:%d:%s", e2e.OCIUserProfile.ContainerUser(t).GID, e2e.OCIUserProfile.ContainerUser(t).Name), + "/etc/group"}, + profile: e2e.OCIUserProfile, + directive: "config group", + directiveValue: "no", + exit: 1, + }, + { + name: "ConfigGroupYes", + argv: []string{archiveRef, "grep", + fmt.Sprintf("x:%d:%s", e2e.OCIUserProfile.ContainerUser(t).GID, e2e.OCIUserProfile.ContainerUser(t).Name), + "/etc/group"}, + profile: e2e.OCIUserProfile, + directive: "config group", + directiveValue: "yes", + exit: 0, + }, + // { + // name: "ConfigResolvConfNo", + // argv: []string{archiveRef, "grep", "/etc/resolv.conf.*- tmpfs", "/proc/self/mountinfo"}, + // profile: e2e.OCIUserProfile, + // directive: "config resolv_conf", + // directiveValue: "no", + // exit: 1, + // }, + // { + // name: "ConfigResolvConfYes", + // argv: []string{archiveRef, "grep", "/etc/resolv.conf.*- tmpfs", "/proc/self/mountinfo"}, + // profile: e2e.OCIUserProfile, + // directive: "config resolv_conf", + // directiveValue: "yes", + // exit: 0, + // }, + // { + // name: "MountProcNo", + // argv: []string{archiveRef, "test", "-d", "/proc/self"}, + // profile: e2e.OCIUserProfile, + // directive: "mount proc", + // directiveValue: "no", + // exit: 1, + // }, + // { + // name: "MountProcYes", + // argv: []string{archiveRef, "test", "-d", "/proc/self"}, + // profile: e2e.OCIUserProfile, + // directive: "mount proc", + // directiveValue: "yes", + // exit: 0, + // }, + // { + // name: "MountSysNo", + // argv: []string{archiveRef, "test", "-d", "/sys/kernel"}, + // profile: e2e.OCIUserProfile, + // directive: "mount sys", + // directiveValue: "no", + // exit: 1, + // }, + // { + // name: "MountSysYes", + // argv: []string{archiveRef, "test", "-d", "/sys/kernel"}, + // profile: e2e.OCIUserProfile, + // directive: "mount sys", + // directiveValue: "yes", + // exit: 0, + // }, + // { + // name: "MountDevNo", + // argv: []string{archiveRef, "test", "-d", "/dev/pts"}, + // profile: e2e.OCIUserProfile, + // directive: "mount dev", + // directiveValue: "no", + // exit: 1, + // }, + // { + // name: "MountDevMinimal", + // argv: []string{archiveRef, "test", "-b", "/dev/loop0"}, + // profile: e2e.OCIUserProfile, + // directive: "mount dev", + // directiveValue: "minimal", + // exit: 1, + // }, + // { + // name: "MountDevYes", + // argv: []string{archiveRef, "test", "-b", "/dev/loop0"}, + // profile: e2e.OCIUserProfile, + // directive: "mount dev", + // directiveValue: "yes", + // exit: 0, + // }, + // // just test 'mount devpts = no' as yes depends of kernel version + // { + // name: "MountDevPtsNo", + // argv: []string{"-C", archiveRef, "test", "-d", "/dev/pts"}, + // profile: e2e.OCIUserProfile, + // directive: "mount devpts", + // directiveValue: "no", + // exit: 1, + // }, + // { + // name: "MountHomeNo", + // argv: []string{archiveRef, "test", "-d", u.Dir}, + // profile: e2e.OCIUserProfile, + // cwd: "/", + // directive: "mount home", + // directiveValue: "no", + // exit: 1, + // }, + // { + // name: "MountHomeYes", + // argv: []string{archiveRef, "test", "-d", u.Dir}, + // profile: e2e.OCIUserProfile, + // cwd: "/", + // directive: "mount home", + // directiveValue: "yes", + // exit: 0, + // }, + // { + // name: "MountTmpNo", + // argv: []string{archiveRef, "test", "-d", c.env.TestDir}, + // profile: e2e.OCIUserProfile, + // directive: "mount tmp", + // directiveValue: "no", + // exit: 1, + // }, + // { + // name: "MountTmpYes", + // argv: []string{archiveRef, "test", "-d", c.env.TestDir}, + // profile: e2e.OCIUserProfile, + // directive: "mount tmp", + // directiveValue: "yes", + // exit: 0, + // }, + // { + // name: "BindPathPasswd", + // argv: []string{archiveRef, "test", "-f", "/passwd"}, + // profile: e2e.OCIUserProfile, + // directive: "bind path", + // directiveValue: "/etc/passwd:/passwd", + // exit: 0, + // }, + // { + // name: "UserBindControlNo", + // argv: []string{"--bind", "/etc/passwd:/passwd", archiveRef, "test", "-f", "/passwd"}, + // profile: e2e.OCIUserProfile, + // directive: "user bind control", + // directiveValue: "no", + // exit: 1, + // }, + // { + // name: "UserBindControlYes", + // argv: []string{"--bind", "/etc/passwd:/passwd", archiveRef, "test", "-f", "/passwd"}, + // profile: e2e.OCIUserProfile, + // directive: "user bind control", + // directiveValue: "yes", + // exit: 0, + // }, + } + + for _, tt := range tests { + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(tt.profile), + e2e.WithDir(tt.cwd), + e2e.PreRun(func(t *testing.T) { + if tt.addRequirementsFn != nil { + tt.addRequirementsFn(t) + } + setDirective(t, tt.directive, tt.directiveValue) + }), + e2e.PostRun(func(t *testing.T) { + resetDirective(t, tt.directive) + }), + e2e.WithCommand("exec"), + e2e.WithArgs(tt.argv...), + e2e.ExpectExit(tt.exit, tt.resultOp), + ) + } +} diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index 49ae11220c..1f3868cd00 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -333,20 +333,28 @@ func (l *Launcher) updatePasswdGroup(rootfs string, uid, gid uint32) error { containerPasswd := filepath.Join(rootfs, "etc", "passwd") containerGroup := filepath.Join(rootfs, "etc", "group") - sylog.Debugf("Updating passwd file: %s", containerPasswd) - content, err := files.Passwd(containerPasswd, l.cfg.HomeDir, int(uid), nil) - if err != nil { - sylog.Warningf("%s", err) - } else if err := os.WriteFile(containerPasswd, content, 0o755); err != nil { - return fmt.Errorf("while writing passwd file: %w", err) + if l.apptainerConf.ConfigPasswd { + sylog.Debugf("Updating passwd file: %s", containerPasswd) + content, err := files.Passwd(containerPasswd, l.cfg.HomeDir, int(uid), nil) + if err != nil { + sylog.Warningf("%s", err) + } else if err := os.WriteFile(containerPasswd, content, 0o755); err != nil { + return fmt.Errorf("while writing passwd file: %w", err) + } + } else { + sylog.Debugf("Skipping update of %s due to apptainer.conf", containerPasswd) } - sylog.Debugf("Updating group file: %s", containerGroup) - content, err = files.Group(containerGroup, int(uid), []int{int(gid)}, nil) - if err != nil { - sylog.Warningf("%s", err) - } else if err := os.WriteFile(containerGroup, content, 0o755); err != nil { - return fmt.Errorf("while writing passwd file: %w", err) + if l.apptainerConf.ConfigGroup { + sylog.Debugf("Updating group file: %s", containerGroup) + content, err := files.Group(containerGroup, int(uid), []int{int(gid)}, nil) + if err != nil { + sylog.Warningf("%s", err) + } else if err := os.WriteFile(containerGroup, content, 0o755); err != nil { + return fmt.Errorf("while writing passwd file: %w", err) + } + } else { + sylog.Debugf("Skipping update of %s due to apptainer.conf", containerGroup) } return nil From 3527ad6d7af4f691ba3f72ebf0f6bc39b45ccd1a Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Thu, 13 Apr 2023 10:33:32 +0100 Subject: [PATCH 085/114] fix: oci: honour mount proc/sys/tmp/home directives When mount proc/sys/tmp/home directives are set to no in singularity.conf, ensure we don't mount at those locations. Also enables a user bind control test, for which the implementation is already in place and working. Signed-off-by: Edita Kizinevic --- e2e/config/oci.go | 248 +++++++++--------- .../pkg/runtime/launcher/oci/mounts_linux.go | 20 ++ 2 files changed, 150 insertions(+), 118 deletions(-) diff --git a/e2e/config/oci.go b/e2e/config/oci.go index bee10dcca1..1ccfc29c68 100644 --- a/e2e/config/oci.go +++ b/e2e/config/oci.go @@ -125,129 +125,141 @@ func (c configTests) ociConfigGlobal(t *testing.T) { // directiveValue: "yes", // exit: 0, // }, + { + name: "MountProcNo", + argv: []string{archiveRef, "test", "-d", "/proc/self"}, + profile: e2e.OCIUserProfile, + directive: "mount proc", + directiveValue: "no", + exit: 1, + }, + { + name: "MountProcYes", + argv: []string{archiveRef, "test", "-d", "/proc/self"}, + profile: e2e.OCIUserProfile, + directive: "mount proc", + directiveValue: "yes", + exit: 0, + }, + { + name: "MountSysNo", + argv: []string{archiveRef, "test", "-d", "/sys/kernel"}, + profile: e2e.OCIUserProfile, + directive: "mount sys", + directiveValue: "no", + exit: 1, + }, + { + name: "MountSysYes", + argv: []string{archiveRef, "test", "-d", "/sys/kernel"}, + profile: e2e.OCIUserProfile, + directive: "mount sys", + directiveValue: "yes", + exit: 0, + }, + // + // mount dev is not currently honoured. We are mimicking --compat in the + // native runtime, which implies `minimal` here. Using `no` isn't an + // option, as the OCI runtime spec requires certain devices: + // https://github.com/opencontainers/runtime-spec/blob/main/config-linux.md#default-devices + // // { - // name: "MountProcNo", - // argv: []string{archiveRef, "test", "-d", "/proc/self"}, - // profile: e2e.OCIUserProfile, - // directive: "mount proc", - // directiveValue: "no", - // exit: 1, - // }, - // { - // name: "MountProcYes", - // argv: []string{archiveRef, "test", "-d", "/proc/self"}, - // profile: e2e.OCIUserProfile, - // directive: "mount proc", - // directiveValue: "yes", - // exit: 0, - // }, - // { - // name: "MountSysNo", - // argv: []string{archiveRef, "test", "-d", "/sys/kernel"}, - // profile: e2e.OCIUserProfile, - // directive: "mount sys", - // directiveValue: "no", - // exit: 1, - // }, - // { - // name: "MountSysYes", - // argv: []string{archiveRef, "test", "-d", "/sys/kernel"}, - // profile: e2e.OCIUserProfile, - // directive: "mount sys", - // directiveValue: "yes", - // exit: 0, - // }, - // { - // name: "MountDevNo", - // argv: []string{archiveRef, "test", "-d", "/dev/pts"}, - // profile: e2e.OCIUserProfile, - // directive: "mount dev", - // directiveValue: "no", - // exit: 1, - // }, - // { - // name: "MountDevMinimal", - // argv: []string{archiveRef, "test", "-b", "/dev/loop0"}, - // profile: e2e.OCIUserProfile, - // directive: "mount dev", - // directiveValue: "minimal", - // exit: 1, - // }, - // { - // name: "MountDevYes", - // argv: []string{archiveRef, "test", "-b", "/dev/loop0"}, - // profile: e2e.OCIUserProfile, - // directive: "mount dev", - // directiveValue: "yes", - // exit: 0, - // }, - // // just test 'mount devpts = no' as yes depends of kernel version - // { - // name: "MountDevPtsNo", - // argv: []string{"-C", archiveRef, "test", "-d", "/dev/pts"}, - // profile: e2e.OCIUserProfile, - // directive: "mount devpts", - // directiveValue: "no", - // exit: 1, - // }, - // { - // name: "MountHomeNo", - // argv: []string{archiveRef, "test", "-d", u.Dir}, - // profile: e2e.OCIUserProfile, - // cwd: "/", - // directive: "mount home", - // directiveValue: "no", - // exit: 1, - // }, - // { - // name: "MountHomeYes", - // argv: []string{archiveRef, "test", "-d", u.Dir}, - // profile: e2e.OCIUserProfile, - // cwd: "/", - // directive: "mount home", - // directiveValue: "yes", - // exit: 0, - // }, - // { - // name: "MountTmpNo", - // argv: []string{archiveRef, "test", "-d", c.env.TestDir}, - // profile: e2e.OCIUserProfile, - // directive: "mount tmp", - // directiveValue: "no", - // exit: 1, - // }, - // { - // name: "MountTmpYes", - // argv: []string{archiveRef, "test", "-d", c.env.TestDir}, - // profile: e2e.OCIUserProfile, - // directive: "mount tmp", - // directiveValue: "yes", - // exit: 0, - // }, - // { - // name: "BindPathPasswd", - // argv: []string{archiveRef, "test", "-f", "/passwd"}, - // profile: e2e.OCIUserProfile, - // directive: "bind path", - // directiveValue: "/etc/passwd:/passwd", - // exit: 0, - // }, + // name: "MountDevNo", + // argv: []string{archiveRef, "test", "-d", "/dev/pts"}, + // profile: e2e.OCIUserProfile, + // directive: "mount dev", + // directiveValue: "no", + // exit: 1, + // }, { + // name: "MountDevMinimal", + // argv: []string{archiveRef, "test", "-b", "/dev/loop0"}, + // profile: e2e.OCIUserProfile, + // directive: "mount dev", + // directiveValue: "minimal", + // exit: 1, + // }, { + // name: "MountDevYes", + // argv: []string{archiveRef, "test", "-b", "/dev/loop0"}, + // profile: e2e.OCIUserProfile, + // directive: "mount dev", + // directiveValue: "yes", + // exit: 0, + // }, // just test 'mount devpts = no' as yes depends of kernel version // { - // name: "UserBindControlNo", - // argv: []string{"--bind", "/etc/passwd:/passwd", archiveRef, "test", "-f", "/passwd"}, - // profile: e2e.OCIUserProfile, - // directive: "user bind control", - // directiveValue: "no", - // exit: 1, + // name: "MountDevPtsNo", + // argv: []string{"-C", archiveRef, "test", "-d", "/dev/pts"}, + // profile: e2e.OCIUserProfile, + // directive: "mount devpts", + // directiveValue: "no", + // exit: 1, // }, + // + // We have to check for a mount of $HOME, rather than presence of dir, + // as runc/crun will create the dir in the container fs if it doesn't + // exist. + { + name: "MountHomeNo", + argv: []string{archiveRef, "grep", e2e.OCIUserProfile.ContainerUser(t).Dir, "/proc/self/mountinfo"}, + profile: e2e.OCIUserProfile, + cwd: "/", + directive: "mount home", + directiveValue: "no", + exit: 1, + }, + { + name: "MountHomeYes", + argv: []string{archiveRef, "grep", e2e.OCIUserProfile.ContainerUser(t).Dir, "/proc/self/mountinfo"}, + profile: e2e.OCIUserProfile, + cwd: "/", + directive: "mount home", + directiveValue: "yes", + exit: 0, + }, + { + name: "MountTmpNo", + argv: []string{archiveRef, "grep", " /tmp ", "/proc/self/mountinfo"}, + profile: e2e.OCIUserProfile, + directive: "mount tmp", + directiveValue: "no", + exit: 1, + }, + { + name: "MountTmpYes", + argv: []string{archiveRef, "grep", " /tmp ", "/proc/self/mountinfo"}, + profile: e2e.OCIUserProfile, + directive: "mount tmp", + directiveValue: "yes", + exit: 0, + }, + // + // bind path isn't supported at present because we are mimicking + // --compat behaviour in the native runtime. However, we should revisit + // what makes most sense for users here before 4.0. + // // { - // name: "UserBindControlYes", - // argv: []string{"--bind", "/etc/passwd:/passwd", archiveRef, "test", "-f", "/passwd"}, - // profile: e2e.OCIUserProfile, - // directive: "user bind control", - // directiveValue: "yes", - // exit: 0, + // name: "BindPathPasswd", + // argv: []string{archiveRef, "test", "-f", "/passwd"}, + // profile: e2e.OCIUserProfile, + // directive: "bind path", + // directiveValue: "/etc/passwd:/passwd", + // exit: 0, // }, + { + name: "UserBindControlNo", + argv: []string{"--bind", "/etc/passwd:/passwd", archiveRef, "test", "-f", "/passwd"}, + profile: e2e.OCIUserProfile, + directive: "user bind control", + directiveValue: "no", + exit: 1, + }, + { + name: "UserBindControlYes", + argv: []string{"--bind", "/etc/passwd:/passwd", archiveRef, "test", "-f", "/passwd"}, + profile: e2e.OCIUserProfile, + directive: "user bind control", + directiveValue: "yes", + exit: 0, + }, } for _, tt := range tests { diff --git a/internal/pkg/runtime/launcher/oci/mounts_linux.go b/internal/pkg/runtime/launcher/oci/mounts_linux.go index ae23c34623..93a2932348 100644 --- a/internal/pkg/runtime/launcher/oci/mounts_linux.go +++ b/internal/pkg/runtime/launcher/oci/mounts_linux.go @@ -62,6 +62,11 @@ func (l *Launcher) getMounts() ([]specs.Mount, error) { // addTmpMounts adds tmpfs mounts for /tmp and /var/tmp in the container. func (l *Launcher) addTmpMounts(mounts *[]specs.Mount) { + if !l.apptainerConf.MountTmp { + sylog.Debugf("Skipping mount of /tmp due to apptainer.conf") + return + } + *mounts = append(*mounts, specs.Mount{ @@ -143,6 +148,11 @@ func (l *Launcher) addDevMounts(mounts *[]specs.Mount) error { // addProcMount adds the /proc tree in the container. func (l *Launcher) addProcMount(mounts *[]specs.Mount) { + if !l.apptainerConf.MountProc { + sylog.Debugf("Skipping mount of /proc due to apptainer.conf") + return + } + *mounts = append(*mounts, specs.Mount{ Source: "proc", @@ -153,6 +163,11 @@ func (l *Launcher) addProcMount(mounts *[]specs.Mount) { // addSysMount adds the /sys tree in the container. func (l *Launcher) addSysMount(mounts *[]specs.Mount) { + if !l.apptainerConf.MountSys { + sylog.Debugf("Skipping mount of /sys due to apptainer.conf") + return + } + if os.Getuid() == 0 { *mounts = append(*mounts, specs.Mount{ @@ -176,6 +191,11 @@ func (l *Launcher) addSysMount(mounts *[]specs.Mount) { // emulating `--compat` / `--containall`, so the user must specifically bind in // their home directory from the host for it to be available. func (l *Launcher) addHomeMount(mounts *[]specs.Mount) error { + if !l.apptainerConf.MountHome { + sylog.Debugf("Skipping mount of $HOME due to apptainer.conf") + return nil + } + // Get the host user's data pw, err := user.CurrentOriginal() if err != nil { From d4220cde1270f3e103d54e759dd8bb421d548eb7 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Thu, 13 Apr 2023 11:15:56 +0100 Subject: [PATCH 086/114] fix: oci: honour config resolv_conf directive Signed-off-by: Edita Kizinevic --- e2e/config/oci.go | 61 +++++++++++-------- .../runtime/launcher/oci/launcher_linux.go | 5 ++ 2 files changed, 40 insertions(+), 26 deletions(-) diff --git a/e2e/config/oci.go b/e2e/config/oci.go index 1ccfc29c68..0e5295c5d0 100644 --- a/e2e/config/oci.go +++ b/e2e/config/oci.go @@ -71,9 +71,11 @@ func (c configTests) ociConfigGlobal(t *testing.T) { // }, { name: "ConfigPasswdNo", - argv: []string{archiveRef, "grep", + argv: []string{ + archiveRef, "grep", fmt.Sprintf("%s:x:%d", e2e.OCIUserProfile.ContainerUser(t).Name, e2e.OCIUserProfile.ContainerUser(t).UID), - "/etc/passwd"}, + "/etc/passwd", + }, profile: e2e.OCIUserProfile, directive: "config passwd", directiveValue: "no", @@ -81,9 +83,11 @@ func (c configTests) ociConfigGlobal(t *testing.T) { }, { name: "ConfigPasswdYes", - argv: []string{archiveRef, "grep", + argv: []string{ + archiveRef, "grep", fmt.Sprintf("%s:x:%d", e2e.OCIUserProfile.ContainerUser(t).Name, e2e.OCIUserProfile.ContainerUser(t).UID), - "/etc/passwd"}, + "/etc/passwd", + }, profile: e2e.OCIUserProfile, directive: "config passwd", directiveValue: "yes", @@ -91,9 +95,11 @@ func (c configTests) ociConfigGlobal(t *testing.T) { }, { name: "ConfigGroupNo", - argv: []string{archiveRef, "grep", + argv: []string{ + archiveRef, "grep", fmt.Sprintf("x:%d:%s", e2e.OCIUserProfile.ContainerUser(t).GID, e2e.OCIUserProfile.ContainerUser(t).Name), - "/etc/group"}, + "/etc/group", + }, profile: e2e.OCIUserProfile, directive: "config group", directiveValue: "no", @@ -101,30 +107,33 @@ func (c configTests) ociConfigGlobal(t *testing.T) { }, { name: "ConfigGroupYes", - argv: []string{archiveRef, "grep", + argv: []string{ + archiveRef, "grep", fmt.Sprintf("x:%d:%s", e2e.OCIUserProfile.ContainerUser(t).GID, e2e.OCIUserProfile.ContainerUser(t).Name), - "/etc/group"}, + "/etc/group", + }, profile: e2e.OCIUserProfile, directive: "config group", directiveValue: "yes", exit: 0, }, - // { - // name: "ConfigResolvConfNo", - // argv: []string{archiveRef, "grep", "/etc/resolv.conf.*- tmpfs", "/proc/self/mountinfo"}, - // profile: e2e.OCIUserProfile, - // directive: "config resolv_conf", - // directiveValue: "no", - // exit: 1, - // }, - // { - // name: "ConfigResolvConfYes", - // argv: []string{archiveRef, "grep", "/etc/resolv.conf.*- tmpfs", "/proc/self/mountinfo"}, - // profile: e2e.OCIUserProfile, - // directive: "config resolv_conf", - // directiveValue: "yes", - // exit: 0, - // }, + // Test container doesn't have an /etc/resolv.conf, so presence check is okay here. + { + name: "ConfigResolvConfNo", + argv: []string{archiveRef, "test", "-f", "/etc/resolv.conf"}, + profile: e2e.OCIUserProfile, + directive: "config resolv_conf", + directiveValue: "no", + exit: 1, + }, + { + name: "ConfigResolvConfYes", + argv: []string{archiveRef, "test", "-f", "/etc/resolv.conf"}, + profile: e2e.OCIUserProfile, + directive: "config resolv_conf", + directiveValue: "yes", + exit: 0, + }, { name: "MountProcNo", argv: []string{archiveRef, "test", "-d", "/proc/self"}, @@ -158,7 +167,7 @@ func (c configTests) ociConfigGlobal(t *testing.T) { exit: 0, }, // - // mount dev is not currently honoured. We are mimicking --compat in the + // mount dev is not currently honored. We are mimicking --compat in the // native runtime, which implies `minimal` here. Using `no` isn't an // option, as the OCI runtime spec requires certain devices: // https://github.com/opencontainers/runtime-spec/blob/main/config-linux.md#default-devices @@ -233,7 +242,7 @@ func (c configTests) ociConfigGlobal(t *testing.T) { }, // // bind path isn't supported at present because we are mimicking - // --compat behaviour in the native runtime. However, we should revisit + // --compat behavior in the native runtime. However, we should revisit // what makes most sense for users here before 4.0. // // { diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index 1f3868cd00..b1de8c37f7 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -365,6 +365,11 @@ func (l *Launcher) prepareResolvConf(rootfs string) error { containerEtc := filepath.Join(rootfs, "etc") containerResolvConfPath := filepath.Join(rootfs, "etc", "resolv.conf") + if !l.apptainerConf.ConfigResolvConf { + sylog.Debugf("Skipping update of %s due to apptainer.conf", containerResolvConfPath) + return nil + } + var resolvConfData []byte var err error if len(l.cfg.DNS) > 0 { From fc9ad81b95c414fdcd020a9da3c03cca6b2848a4 Mon Sep 17 00:00:00 2001 From: preminger Date: Fri, 14 Apr 2023 12:18:16 -0400 Subject: [PATCH 087/114] fixed typo (resolve.conf -> resolv.conf) (sylabs/singularity#1566) Signed-off-by: Edita Kizinevic --- internal/pkg/runtime/launcher/oci/launcher_linux.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index b1de8c37f7..50b337204b 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -391,7 +391,7 @@ func (l *Launcher) prepareResolvConf(rootfs string) error { stat, err := os.Stat(containerEtc) if os.IsNotExist(err) || !stat.IsDir() { - sylog.Warningf("container does not contain an /etc directory; skipping resolve.conf configuration") + sylog.Warningf("container does not contain an /etc directory; skipping resolv.conf configuration") return nil } From b71fc5822049b768c806e8d08a2b6c9dc5321d5f Mon Sep 17 00:00:00 2001 From: preminger Date: Fri, 14 Apr 2023 11:46:19 -0400 Subject: [PATCH 088/114] pull HashingListSubtract into (new) high-level util package (sylabs/singularity#1562) * pull HashingListSubtract into (new) high-level util package * to-be-squashed: addressing DT's comments Signed-off-by: Edita Kizinevic --- LICENSE_DEPENDENCIES.md | 6 ++ .../runtime/launcher/oci/cdi_linux_test.go | 13 +-- pkg/util/slice/slice.go | 16 +++ pkg/util/slice/slice_test.go | 100 +++++++++++++++++- 4 files changed, 123 insertions(+), 12 deletions(-) diff --git a/LICENSE_DEPENDENCIES.md b/LICENSE_DEPENDENCIES.md index 247cf4ffd6..7ed457ca8a 100644 --- a/LICENSE_DEPENDENCIES.md +++ b/LICENSE_DEPENDENCIES.md @@ -905,6 +905,12 @@ The dependencies and their licenses are as follows: **License URL:** +## github.com/samber/lo + +**License:** MIT + +**License URL:** + ## github.com/secure-systems-lab/go-securesystemslib/dsse **License:** MIT diff --git a/internal/pkg/runtime/launcher/oci/cdi_linux_test.go b/internal/pkg/runtime/launcher/oci/cdi_linux_test.go index c1c5f25f7b..39858c8023 100644 --- a/internal/pkg/runtime/launcher/oci/cdi_linux_test.go +++ b/internal/pkg/runtime/launcher/oci/cdi_linux_test.go @@ -18,9 +18,9 @@ import ( "sort" "testing" + "github.com/apptainer/apptainer/pkg/util/slice" "github.com/container-orchestrated-devices/container-device-interface/pkg/cdi" "github.com/opencontainers/runtime-spec/specs-go" - "github.com/samber/lo" ) var specDirs = []string{filepath.Join("..", "..", "..", "..", "..", "test", "cdi")} @@ -245,19 +245,10 @@ func Test_addCDIDevice(t *testing.T) { } } - envMissing := hashingListSubtract(tt.wantEnv, spec.Process.Env) + envMissing := slice.Subtract(tt.wantEnv, spec.Process.Env) if len(envMissing) > 0 { t.Errorf("addCDIDevices() mismatched environment variables; expected, but did not find, the following environment variables: %v", envMissing) } }) } } - -// hashingListSubtract is a utility-function for subtracting a list from another list, using map's internal hashing function to do this more efficiently than lo.Difference or lo.Without (which only assume comparable, and thus run in quadratic time). -func hashingListSubtract[T comparable](toSubstractFrom []T, toSubstract []T) []T { - subtractionMap := lo.FromEntries(lo.Map(toSubstractFrom, func(item T, _ int) lo.Entry[T, bool] { - return lo.Entry[T, bool]{Key: item, Value: true} - })) - - return lo.Keys(lo.OmitByKeys(subtractionMap, toSubstract)) -} diff --git a/pkg/util/slice/slice.go b/pkg/util/slice/slice.go index c56b4dcfd5..46e0ebba2e 100644 --- a/pkg/util/slice/slice.go +++ b/pkg/util/slice/slice.go @@ -9,6 +9,8 @@ package slice +import "github.com/samber/lo" + // ContainsString returns true if string slice s contains match func ContainsString(s []string, match string) bool { for _, a := range s { @@ -40,3 +42,17 @@ func ContainsInt(s []int, match int) bool { } return false } + +// Subtract removes items in slice b from slice a, returning the result. +// Implemented using a map for greater efficiency than lo.Difference / lo.Without, when operating on large slices. +func Subtract[T comparable](a []T, b []T) []T { + subtractionMap := lo.FromEntries(lo.Map(a, func(item T, _ int) lo.Entry[T, bool] { + return lo.Entry[T, bool]{Key: item, Value: true} + })) + subtractionMap = lo.OmitByKeys(subtractionMap, b) + + return lo.Filter(a, func(x T, _ int) bool { + _, ok := subtractionMap[x] + return ok + }) +} diff --git a/pkg/util/slice/slice_test.go b/pkg/util/slice/slice_test.go index cee3b42e55..fc5e7418f6 100644 --- a/pkg/util/slice/slice_test.go +++ b/pkg/util/slice/slice_test.go @@ -9,7 +9,13 @@ package slice -import "testing" +import ( + "fmt" + "reflect" + "testing" + + "github.com/samber/lo" +) func TestContainsString(t *testing.T) { type args struct { @@ -190,3 +196,95 @@ func TestContainsInt(t *testing.T) { }) } } + +func TestSubtract(t *testing.T) { + type args[T any] struct { + a []T + b []T + want []T + } + intTests := []struct { + name string + args args[int] + }{ + { + name: "Identical", + args: args[int]{ + a: []int{3, 9, 5, 7, 2, 1, 0, 4}, + b: []int{3, 9, 5, 7, 2, 1, 0, 4}, + want: []int{}, + }, + }, + { + name: "EmptyA", + args: args[int]{ + a: []int{}, + b: []int{3, 9, 5, 7, 2, 1, 0, 4}, + want: []int{}, + }, + }, + { + name: "EmptyB", + args: args[int]{ + a: []int{3, 9, 5, 7, 2, 1, 0, 4}, + b: []int{}, + want: []int{3, 9, 5, 7, 2, 1, 0, 4}, + }, + }, + { + name: "EmptyBoth", + args: args[int]{ + a: []int{}, + b: []int{}, + want: []int{}, + }, + }, + { + name: "AsupersetofB", + args: args[int]{ + a: []int{3, 9, 5, 7, 2, 1, 0, 4}, + b: []int{3, 9, 7, 0, 4}, + want: []int{5, 2, 1}, + }, + }, + { + name: "AsubsetofB", + args: args[int]{ + a: []int{5, 2, 1}, + b: []int{5, 7, 2, 1, 0, 4}, + want: []int{}, + }, + }, + { + name: "Intersection", + args: args[int]{ + a: []int{3, 5, 2, 0}, + b: []int{3, 9, 7, 2, 4}, + want: []int{5, 0}, + }, + }, + } + + convertor := func(x int, index int) string { + return fmt.Sprintf("Have an int whose value is %#v, why don't you", x) + } + + for _, tt := range intTests { + t.Run("Int"+tt.name, func(t *testing.T) { + if got := Subtract(tt.args.a, tt.args.b); !reflect.DeepEqual(got, tt.args.want) { + t.Errorf("Subtract(%#v, %#v) = %#v, want %#v", tt.args.a, tt.args.b, got, tt.args.want) + } + }) + + strArgs := args[string]{ + a: lo.Map(tt.args.a, convertor), + b: lo.Map(tt.args.b, convertor), + want: lo.Map(tt.args.want, convertor), + } + t.Run("String"+tt.name, func(t *testing.T) { + if got := Subtract(strArgs.a, strArgs.b); !reflect.DeepEqual(got, strArgs.want) { + t.Errorf("Subtract(%#v, %#v) = %#v, want %#v", strArgs.a, strArgs.b, got, strArgs.want) + } + }) + } +} From ad089bcf5983299fcf15cda0707c51a70ea29632 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Mon, 17 Apr 2023 14:35:25 +0100 Subject: [PATCH 089/114] fix: oci: Don't create cgroup for crun on v1 / cgroupfs If we are running under cgroups v1 or with the cgroupfs manager (i.e. not systemd as cgroup manager), do not attempt to enter a cgroup at startup with crun. We cannot create a cgroup unprivileged in this situation. Under cgroups v1, crun will not perform the cgroups manipulation that leads to the issue we worked around in #1539. Any other issue with the cgroup that we are in at launch cannot be rectified, either. Fixes sylabs/singularity#1569 Signed-off-by: Edita Kizinevic --- .../pkg/runtime/launcher/oci/launcher_linux.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index 50b337204b..951dc1e5d6 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -35,6 +35,7 @@ import ( "github.com/apptainer/apptainer/pkg/util/apptainerconf" "github.com/container-orchestrated-devices/container-device-interface/pkg/cdi" "github.com/google/uuid" + lccgroups "github.com/opencontainers/runc/libcontainer/cgroups" "github.com/opencontainers/runtime-spec/specs-go" ) @@ -499,9 +500,10 @@ func (l *Launcher) getCgroup() (path string, resources *specs.LinuxResources, er return path, resources, nil } -// crunNestCgroup will check whether we are using crun, and enter a cgroup if running as a non-root user. -// This is required to satisfy a common user-owned ancestor cgroup requirement on e.g. bare ssh logins. -// See: https://github.com/sylabs/singularity/issues/1538 +// crunNestCgroup will check whether we are using crun, and enter a cgroup if +// running as a non-root user under cgroups v2, with systemd. This is required +// to satisfy a common user-owned ancestor cgroup requirement on e.g. bare ssh +// logins. See: https://github.com/sylabs/singularity/issues/1538 func (l *Launcher) crunNestCgroup() error { r, err := runtime() if err != nil { @@ -518,6 +520,12 @@ func (l *Launcher) crunNestCgroup() error { return nil } + // We can only create a new cgroup under cgroups v2 with systemd as manager. + // Generally we won't hit the issue that needs a workaround under cgroups v1, so no-op instead of a warning here. + if !(lccgroups.IsCgroup2UnifiedMode() && l.apptainerConf.SystemdCgroups) { + return nil + } + // We are running crun as a user. Enter a cgroup now. pid := os.Getpid() sylog.Debugf("crun workaround - adding process %d to sibling cgroup", pid) From 2decb88f6ec8046363180cd4d68c507b2a29b566 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Fri, 21 Apr 2023 15:11:53 +0100 Subject: [PATCH 090/114] fix: Remove files with restrictive perms from --oci temp rootfs When we run a container in `--oci` mode, we have a temporary rootfs that must be removed when the container exits. A container can contain files / dirs with restrictive permissions that prevent direct removal. Use fs.ForceRemoveAll, which will chmod & remove anything that os.RemoveAll is unable to remove. Fixes sylabs/singularity#1586 Signed-off-by: Edita Kizinevic --- e2e/docker/docker.go | 1 + e2e/docker/regressions.go | 32 +++++++++++++++++++ .../runtime/launcher/oci/launcher_linux.go | 3 +- 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/e2e/docker/docker.go b/e2e/docker/docker.go index 01d885b7e9..14aa50dee9 100644 --- a/e2e/docker/docker.go +++ b/e2e/docker/docker.go @@ -958,6 +958,7 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { t.Run("issue 4524", c.issue4524) t.Run("issue 1286", c.issue1286) t.Run("issue 1528", c.issue1528) + t.Run("issue 1586", c.issue1586) }, // Tests that are especially slow, or run against a local docker // registry, can be run in parallel, with `--disable-cache` used within diff --git a/e2e/docker/regressions.go b/e2e/docker/regressions.go index dc2cec4129..bd7d153cb7 100644 --- a/e2e/docker/regressions.go +++ b/e2e/docker/regressions.go @@ -11,6 +11,7 @@ package docker import ( "fmt" + "io" "os" "path" "path/filepath" @@ -286,3 +287,34 @@ func (c ctx) issue1528(t *testing.T) { }) } } + +// https://github.com/sylabs/singularity/issues/1586 +// In OCI mode, ensure that nothing is left in TMPDIR from a docker:// image with restrictive file permissions. +func (c ctx) issue1586(t *testing.T) { + tmpDir, cleanup := e2e.MakeTempDir(t, c.env.TestDir, "issue1586-", "") + t.Cleanup(func() { + if !t.Failed() { + cleanup(t) + } + }) + + c.env.RunApptainer( + t, + e2e.WithProfile(e2e.OCIUserProfile), + e2e.WithCommand("exec"), + e2e.WithArgs("docker://almalinux:9.1-minimal-20230407", "/bin/true"), + e2e.WithEnv(append(os.Environ(), "TMPDIR="+tmpDir)), + e2e.ExpectExit(0, + e2e.ExpectError(e2e.UnwantedContainMatch, "permission denied"), + ), + ) + + d, err := os.Open(tmpDir) + if err != nil { + t.Errorf("Couldn't open TMPDIR %s: %v", tmpDir, err) + } + defer d.Close() + if _, err = d.Readdir(1); err != io.EOF { + t.Errorf("TMPDIR is not empty after apptainer exited") + } +} diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index 951dc1e5d6..49029592d1 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -27,6 +27,7 @@ import ( "github.com/apptainer/apptainer/internal/pkg/cache" "github.com/apptainer/apptainer/internal/pkg/cgroups" "github.com/apptainer/apptainer/internal/pkg/runtime/launcher" + "github.com/apptainer/apptainer/internal/pkg/util/fs" "github.com/apptainer/apptainer/internal/pkg/util/fs/files" "github.com/apptainer/apptainer/pkg/ocibundle" "github.com/apptainer/apptainer/pkg/ocibundle/native" @@ -425,7 +426,7 @@ func (l *Launcher) Exec(ctx context.Context, image string, process string, args } defer func() { sylog.Debugf("Removing OCI bundle at: %s", bundleDir) - if err := os.RemoveAll(bundleDir); err != nil { + if err := fs.ForceRemoveAll(bundleDir); err != nil { sylog.Errorf("Couldn't remove OCI bundle %s: %v", bundleDir, err) } }() From d78657a45d4c3cf61b05409aee290b298927728f Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Tue, 2 May 2023 10:34:26 +0100 Subject: [PATCH 091/114] oci: Enable --writable-tmpfs behaviour by default The `--oci` mode intends to follow behaviour that the native runtime implements when run with `--compat`. One missing aspect is that `--compat` sets `--writable-tmpfs`, where the container rootfs is made writable with a tmpfs backed overlay. This PR: - Introduces a simple wrapping of the `oci run` sub-command as `oci run-wrapped`. This hidden command implements prep / cleanup steps that must take place in a userns for non-root `--oci` execution. - Switches the oci launcher to calling `oci run-wrapped` instead of `oci-run`. - Adds a tmpfs based overlay creation function for OCI bundles. - Includes the tmpfs overlay creation in the `oci run-wrapped` flow. - Copies the native runtime `--compat` e2e tests to OCI mode. Fixes sylabs/singularity#1621 Signed-off-by: Edita Kizinevic --- cmd/internal/cli/oci_linux.go | 22 ++++++- docs/content.go | 3 + e2e/actions/actions.go | 15 ++--- e2e/actions/oci.go | 58 +++++++++++++++++++ internal/app/apptainer/oci_linux.go | 9 +++ .../runtime/launcher/oci/launcher_linux.go | 8 +-- .../runtime/launcher/oci/oci_runc_linux.go | 41 ++++++++++++- .../pkg/runtime/launcher/oci/spec_linux.go | 4 +- pkg/ocibundle/tools/overlay_linux.go | 47 +++++++++++++-- 9 files changed, 186 insertions(+), 21 deletions(-) diff --git a/cmd/internal/cli/oci_linux.go b/cmd/internal/cli/oci_linux.go index 2d242ce5a5..8eef6e974b 100644 --- a/cmd/internal/cli/oci_linux.go +++ b/cmd/internal/cli/oci_linux.go @@ -111,6 +111,7 @@ func init() { cmdManager.RegisterSubCmd(OciCmd, OciStartCmd) cmdManager.RegisterSubCmd(OciCmd, OciCreateCmd) cmdManager.RegisterSubCmd(OciCmd, OciRunCmd) + cmdManager.RegisterSubCmd(OciCmd, OciRunWrappedCmd) cmdManager.RegisterSubCmd(OciCmd, OciDeleteCmd) cmdManager.RegisterSubCmd(OciCmd, OciKillCmd) cmdManager.RegisterSubCmd(OciCmd, OciStateCmd) @@ -122,7 +123,7 @@ func init() { cmdManager.RegisterSubCmd(OciCmd, OciMountCmd) cmdManager.RegisterSubCmd(OciCmd, OciUmountCmd) - cmdManager.SetCmdGroup("create_run", OciCreateCmd, OciRunCmd) + cmdManager.SetCmdGroup("create_run", OciCreateCmd, OciRunCmd, OciRunWrappedCmd) createRunCmd := cmdManager.GetCmdGroup("create_run") cmdManager.RegisterFlagForCmd(&ociBundleFlag, createRunCmd...) @@ -171,6 +172,25 @@ var OciRunCmd = &cobra.Command{ Example: docs.OciRunExample, } +// OciRunWrappedCmd is for internal OCI launcher use. +// Executes an oci run, wrapped with preparation / cleanup code. +var OciRunWrappedCmd = &cobra.Command{ + Args: cobra.ExactArgs(1), + DisableFlagsInUseLine: true, + PreRun: CheckRoot, + Run: func(cmd *cobra.Command, args []string) { + if err := apptainer.OciRunWrapped(cmd.Context(), args[0], &ociArgs); err != nil { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { + os.Exit(exitErr.ExitCode()) + } + sylog.Fatalf("%s", err) + } + }, + Use: docs.OciRunWrappedUse, + Hidden: true, +} + // OciStartCmd represents oci start command. var OciStartCmd = &cobra.Command{ Args: cobra.ExactArgs(1), diff --git a/docs/content.go b/docs/content.go index 8155d97579..9481dc50c1 100644 --- a/docs/content.go +++ b/docs/content.go @@ -1035,6 +1035,9 @@ Enterprise Performance Computing (EPC)` $ apptainer oci attach mycontainer $ apptainer oci delete mycontainer` + // Internal oci launcher use only - no user-facing docs + OciRunWrappedUse string = `run-wrapped -b [run options...] ` + OciUpdateUse string = `update [update options...] ` OciUpdateShort string = `Update container cgroups resources (root user only)` OciUpdateLong string = ` diff --git a/e2e/actions/actions.go b/e2e/actions/actions.go index 3d73090ba7..dc6db75a8a 100644 --- a/e2e/actions/actions.go +++ b/e2e/actions/actions.go @@ -2967,12 +2967,13 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { // // OCI Runtime Mode // - "ociRun": c.actionOciRun, // apptainer run --oci - "ociExec": c.actionOciExec, // apptainer exec --oci - "ociShell": c.actionOciShell, // apptainer shell --oci - "ociNetwork": c.actionOciNetwork, // apptainer exec --oci --net - "ociBinds": c.actionOciBinds, // apptainer exec --oci --bind / --mount - "ociCdi": c.actionOciCdi, // apptainer exec --oci --cdi - "ociIDMaps": c.actionOciIDMaps, // check uid/gid mapping on host for --oci as user / --fakeroot + "ociRun": c.actionOciRun, // apptainer run --oci + "ociExec": c.actionOciExec, // apptainer exec --oci + "ociShell": c.actionOciShell, // apptainer shell --oci + "ociNetwork": c.actionOciNetwork, // apptainer exec --oci --net + "ociBinds": c.actionOciBinds, // apptainer exec --oci --bind / --mount + "ociCdi": c.actionOciCdi, // apptainer exec --oci --cdi + "ociIDMaps": c.actionOciIDMaps, // check uid/gid mapping on host for --oci as user / --fakeroot + "ociCompat": np(c.actionOciCompat), // --oci equivalence to native mode --compat } } diff --git a/e2e/actions/oci.go b/e2e/actions/oci.go index f7d630d201..ed11903bbc 100644 --- a/e2e/actions/oci.go +++ b/e2e/actions/oci.go @@ -16,6 +16,7 @@ import ( "os/exec" "path/filepath" "strings" + "syscall" "testing" "text/template" @@ -894,3 +895,60 @@ func (c actionTests) actionOciIDMaps(t *testing.T) { }) } } + +// actionOCICompat checks that the --oci mode has the behavior that the native mode gains from the --compat flag. +// Must be run in sequential section as it modifies host process umask. +func (c actionTests) actionOciCompat(t *testing.T) { + e2e.EnsureOCIArchive(t, c.env) + imageRef := "oci-archive:" + c.env.OCIArchivePath + + type test struct { + name string + args []string + exitCode int + expect e2e.ApptainerCmdResultOp + } + + tests := []test{ + { + name: "containall", + args: []string{imageRef, "sh", "-c", "ls -lah $HOME"}, + exitCode: 0, + expect: e2e.ExpectOutput(e2e.ContainMatch, "total 0"), + }, + { + name: "writable-tmpfs", + args: []string{imageRef, "sh", "-c", "touch /test"}, + exitCode: 0, + }, + { + name: "no-init", + args: []string{imageRef, "sh", "-c", "ps"}, + exitCode: 0, + expect: e2e.ExpectOutput(e2e.UnwantedContainMatch, "sinit"), + }, + { + name: "no-umask", + args: []string{imageRef, "sh", "-c", "umask"}, + exitCode: 0, + expect: e2e.ExpectOutput(e2e.ContainMatch, "0022"), + }, + } + + oldUmask := syscall.Umask(0) + defer syscall.Umask(oldUmask) + + for _, tt := range tests { + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(e2e.OCIUserProfile), + e2e.WithCommand("exec"), + e2e.WithArgs(tt.args...), + e2e.ExpectExit( + tt.exitCode, + tt.expect, + ), + ) + } +} diff --git a/internal/app/apptainer/oci_linux.go b/internal/app/apptainer/oci_linux.go index faa803d730..25df12e4c2 100644 --- a/internal/app/apptainer/oci_linux.go +++ b/internal/app/apptainer/oci_linux.go @@ -46,6 +46,15 @@ func OciRun(ctx context.Context, containerID string, args *OciArgs) error { return oci.Run(ctx, containerID, args.BundlePath, args.PidFile, systemdCgroups) } +// OciRun runs a container via the OCI runtime, wrapped with prep / cleanup steps. +func OciRunWrapped(ctx context.Context, containerID string, args *OciArgs) error { + systemdCgroups, err := systemdCgroups() + if err != nil { + return err + } + return oci.RunWrapped(ctx, containerID, args.BundlePath, args.PidFile, systemdCgroups) +} + // OciCreate creates a container from an OCI bundle func OciCreate(containerID string, args *OciArgs) error { systemdCgroups, err := systemdCgroups() diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index 49029592d1..f8731ac515 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -83,7 +83,7 @@ func checkOpts(lo launcher.Options) error { badOpt = append(badOpt, "Writable") } if lo.WritableTmpfs { - badOpt = append(badOpt, "WritableTmpfs") + sylog.Infof("--oci mode uses --writable-tmpfs by default") } if len(lo.OverlayPaths) > 0 { badOpt = append(badOpt, "OverlayPaths") @@ -474,12 +474,12 @@ func (l *Launcher) Exec(ctx context.Context, image string, process string, args } if os.Getuid() == 0 { - // Direct execution of runc/crun run. - err = Run(ctx, id.String(), b.Path(), "", l.apptainerConf.SystemdCgroups) + // Execution of runc/crun run, wrapped with prep / cleanup. + err = RunWrapped(ctx, id.String(), b.Path(), "", l.apptainerConf.SystemdCgroups) } else { // Reexec apptainer oci run in a userns with mappings. // Note - the oci run command will pull out the SystemdCgroups setting from config. - err = RunNS(ctx, id.String(), b.Path(), "") + err = RunWrappedNS(ctx, id.String(), b.Path(), "") } var exitErr *exec.ExitError if errors.As(err, &exitErr) { diff --git a/internal/pkg/runtime/launcher/oci/oci_runc_linux.go b/internal/pkg/runtime/launcher/oci/oci_runc_linux.go index fa919dcc34..5a960fff4a 100644 --- a/internal/pkg/runtime/launcher/oci/oci_runc_linux.go +++ b/internal/pkg/runtime/launcher/oci/oci_runc_linux.go @@ -23,8 +23,10 @@ import ( "github.com/apptainer/apptainer/internal/pkg/buildcfg" fakerootConfig "github.com/apptainer/apptainer/internal/pkg/runtime/engine/fakeroot/config" "github.com/apptainer/apptainer/internal/pkg/util/starter" + "github.com/apptainer/apptainer/pkg/ocibundle/tools" "github.com/apptainer/apptainer/pkg/runtime/engine/config" "github.com/apptainer/apptainer/pkg/sylog" + "github.com/apptainer/apptainer/pkg/util/apptainerconf" ) // Delete deletes container resources @@ -223,8 +225,27 @@ func Run(ctx context.Context, containerID, bundlePath, pidFile string, systemdCg return cmd.Run() } -// RunNS reexecs apptainer in a user namespace, with supplied uid/gid mapping, calling oci run. -func RunNS(ctx context.Context, containerID, bundlePath, pidFile string) error { +// RunWrapped runs a container via the OCI runtime, wrapped with prep / cleanup steps. +func RunWrapped(ctx context.Context, containerID, bundlePath, pidFile string, systemdCgroups bool) error { + // TODO: --oci mode always emulating --compat, which uses --writable-tmpfs. + // Provide a way of disabling this, for a read only rootfs. + if err := prepareWriteableTmpfs(bundlePath); err != nil { + return err + } + + err := Run(ctx, containerID, bundlePath, pidFile, systemdCgroups) + + // Cleanup actions log errors, but don't return - so we get as much cleanup done as possible. + if err := cleanupWritableTmpfs(bundlePath); err != nil { + sylog.Errorf("While cleaning up writable tmpfs: %v", err) + } + + // Return any error from the actual container payload - preserve exit code. + return err +} + +// RunWrappedNS reexecs apptainer in a user namespace, with supplied uid/gid mapping, calling oci run. +func RunWrappedNS(ctx context.Context, containerID, bundlePath, pidFile string) error { absBundle, err := filepath.Abs(bundlePath) if err != nil { return fmt.Errorf("failed to determine bundle absolute path: %s", err) @@ -237,7 +258,7 @@ func RunNS(ctx context.Context, containerID, bundlePath, pidFile string) error { args := []string{ filepath.Join(buildcfg.BINDIR, "apptainer"), "oci", - "run", + "run-wrapped", "-b", absBundle, containerID, } @@ -346,3 +367,17 @@ func Update(containerID, cgFile string, systemdCgroups bool) error { sylog.Debugf("Calling %s with args %v", runtimeBin, runtimeArgs) return cmd.Run() } + +func prepareWriteableTmpfs(bundleDir string) error { + sylog.Debugf("Configuring writable tmpfs overlay for %s", bundleDir) + c := apptainerconf.GetCurrentConfig() + if c == nil { + return fmt.Errorf("apptainer configuration is not initialized") + } + return tools.CreateOverlayTmpfs(bundleDir, int(c.SessiondirMaxSize)) +} + +func cleanupWritableTmpfs(bundleDir string) error { + sylog.Debugf("Cleaning up writable tmpfs overlay for %s", bundleDir) + return tools.DeleteOverlay(bundleDir) +} diff --git a/internal/pkg/runtime/launcher/oci/spec_linux.go b/internal/pkg/runtime/launcher/oci/spec_linux.go index dac2ccdea4..42ba358898 100644 --- a/internal/pkg/runtime/launcher/oci/spec_linux.go +++ b/internal/pkg/runtime/launcher/oci/spec_linux.go @@ -40,8 +40,8 @@ func minimalSpec() specs.Spec { } config.Root = &specs.Root{ Path: "rootfs", - // TODO - support writable-tmpfs / writable - Readonly: true, + // TODO - support read-only. At present we always have a writable tmpfs overlay, like native runtime --compat. + Readonly: false, } config.Process = &specs.Process{ Terminal: true, diff --git a/pkg/ocibundle/tools/overlay_linux.go b/pkg/ocibundle/tools/overlay_linux.go index e21a40d98e..1c2b9b414f 100644 --- a/pkg/ocibundle/tools/overlay_linux.go +++ b/pkg/ocibundle/tools/overlay_linux.go @@ -16,7 +16,7 @@ import ( "syscall" ) -// CreateOverlay creates a writable overlay +// CreateOverlay creates a writable overlay based on a directory. func CreateOverlay(bundlePath string) error { var err error @@ -49,18 +49,57 @@ func CreateOverlay(bundlePath string) error { return fmt.Errorf("failed to remount %s: %s", overlayDir, err) } + err = prepareOverlay(bundlePath, overlayDir) + return err +} + +// CreateOverlay creates a writable overlay based on a tmpfs. +func CreateOverlayTmpfs(bundlePath string, sizeMiB int) error { + var err error + + oldumask := syscall.Umask(0) + defer syscall.Umask(oldumask) + + overlayDir := filepath.Join(bundlePath, "overlay") + if err = os.Mkdir(overlayDir, 0o700); err != nil { + return fmt.Errorf("failed to create %s: %s", overlayDir, err) + } + // delete overlay directory in case of error + defer func() { + if err != nil { + os.RemoveAll(overlayDir) + } + }() + + options := fmt.Sprintf("mode=1777,size=%dm", sizeMiB) + err = syscall.Mount("tmpfs", overlayDir, "tmpfs", syscall.MS_NODEV, options) + if err != nil { + return fmt.Errorf("failed to bind %s: %s", overlayDir, err) + } + // best effort to cleanup mount + defer func() { + if err != nil { + syscall.Unmount(overlayDir, syscall.MNT_DETACH) + } + }() + + err = prepareOverlay(bundlePath, overlayDir) + return err +} + +func prepareOverlay(bundlePath, overlayDir string) error { upperDir := filepath.Join(overlayDir, "upper") - if err = os.Mkdir(upperDir, 0o755); err != nil { + if err := os.Mkdir(upperDir, 0o755); err != nil { return fmt.Errorf("failed to create %s: %s", upperDir, err) } workDir := filepath.Join(overlayDir, "work") - if err = os.Mkdir(workDir, 0o700); err != nil { + if err := os.Mkdir(workDir, 0o700); err != nil { return fmt.Errorf("failed to create %s: %s", workDir, err) } rootFsDir := RootFs(bundlePath).Path() options := fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", rootFsDir, upperDir, workDir) - if err = syscall.Mount("overlay", rootFsDir, "overlay", 0, options); err != nil { + if err := syscall.Mount("overlay", rootFsDir, "overlay", 0, options); err != nil { return fmt.Errorf("failed to mount %s: %s", overlayDir, err) } return nil From aa691e5308be087d3313ba5b7f41efb8d070c19a Mon Sep 17 00:00:00 2001 From: preminger Date: Mon, 3 Apr 2023 09:55:38 -0400 Subject: [PATCH 092/114] 1481 support pwd in oci mode (sylabs/singularity#1496) * oci: support --pwd * replaced --pwd with --cwd (leaving former as synonym), adjusted code accordingly throughout * to-be-squashed: addressing DT's Mar 30 comments Signed-off-by: Edita Kizinevic --- e2e/actions/oci.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/e2e/actions/oci.go b/e2e/actions/oci.go index ed11903bbc..22d08d9faf 100644 --- a/e2e/actions/oci.go +++ b/e2e/actions/oci.go @@ -215,6 +215,14 @@ func (c actionTests) actionOciExec(t *testing.T) { e2e.ExpectOutput(e2e.ExactMatch, "/tmp"), }, }, + { + name: "Cwd", + argv: []string{"--cwd", "/tmp", imageRef, "pwd"}, + exit: 0, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ExactMatch, "/tmp"), + }, + }, { name: "ResolvConfGoogle", argv: []string{"--dns", "8.8.8.8,8.8.4.4", imageRef, "nslookup", "w3.org"}, From 712ed4ca19812047f3fb0d2220ed8e4226cb97bd Mon Sep 17 00:00:00 2001 From: Omer Preminger Date: Wed, 5 Apr 2023 13:33:53 -0400 Subject: [PATCH 093/114] oci: fix --cwd/--pwd and actionOciExec() e2e-test Signed-off-by: Edita Kizinevic --- e2e/actions/oci.go | 34 +++++++++++++------ e2e/suite.go | 8 +++-- .../pkg/runtime/launcher/oci/process_linux.go | 4 +++ 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/e2e/actions/oci.go b/e2e/actions/oci.go index 22d08d9faf..d4d2f0c030 100644 --- a/e2e/actions/oci.go +++ b/e2e/actions/oci.go @@ -125,10 +125,11 @@ func (c actionTests) actionOciExec(t *testing.T) { homePath := filepath.Join("/home", basename) tests := []struct { - name string - argv []string - exit int - wantOutputs []e2e.ApptainerCmdResultOp + name string + argv []string + exit int + wantOutputs []e2e.ApptainerCmdResultOp + skipProfiles map[string]bool }{ { name: "NoCommand", @@ -177,6 +178,10 @@ func (c actionTests) actionOciExec(t *testing.T) { e2e.ExpectOutput(e2e.RegexMatch, `\bHOME=/myhomeloc\b`), e2e.ExpectOutput(e2e.RegexMatch, `\btmpfs on /myhomeloc\b`), }, + skipProfiles: map[string]bool{ + e2e.OCIRootProfile.String(): true, + e2e.OCIFakerootProfile.String(): true, + }, exit: 0, }, { @@ -209,18 +214,18 @@ func (c actionTests) actionOciExec(t *testing.T) { }, { name: "Pwd", - argv: []string{"--pwd", "/tmp", imageRef, "pwd"}, + argv: []string{"--pwd", "/etc", imageRef, "pwd"}, exit: 0, wantOutputs: []e2e.ApptainerCmdResultOp{ - e2e.ExpectOutput(e2e.ExactMatch, "/tmp"), + e2e.ExpectOutput(e2e.ExactMatch, "/etc"), }, }, { name: "Cwd", - argv: []string{"--cwd", "/tmp", imageRef, "pwd"}, + argv: []string{"--cwd", "/etc", imageRef, "pwd"}, exit: 0, wantOutputs: []e2e.ApptainerCmdResultOp{ - e2e.ExpectOutput(e2e.ExactMatch, "/tmp"), + e2e.ExpectOutput(e2e.ExactMatch, "/etc"), }, }, { @@ -243,10 +248,15 @@ func (c actionTests) actionOciExec(t *testing.T) { for _, profile := range e2e.OCIProfiles { t.Run(profile.String(), func(t *testing.T) { for _, tt := range tests { + skip, ok := tt.skipProfiles[profile.String()] + if ok && skip { + continue + } + c.env.RunApptainer( t, e2e.AsSubtest(tt.name), - e2e.WithProfile(e2e.UserProfile), + e2e.WithProfile(profile), e2e.WithCommand("exec"), e2e.WithDir("/tmp"), e2e.WithArgs(tt.argv...), @@ -377,7 +387,11 @@ func (c actionTests) actionOciBinds(t *testing.T) { imageRef := "oci-archive:" + c.env.OCIArchivePath workspace, cleanup := e2e.MakeTempDir(t, c.env.TestDir, "bind-workspace-", "") - defer e2e.Privileged(cleanup) + t.Cleanup(func() { + if !t.Failed() { + e2e.Privileged(cleanup) + } + }) contCanaryDir := "/canary" hostCanaryDir := filepath.Join(workspace, "canary") diff --git a/e2e/suite.go b/e2e/suite.go index b28e22b3d2..303c4673d1 100644 --- a/e2e/suite.go +++ b/e2e/suite.go @@ -202,9 +202,11 @@ func Run(t *testing.T) { testenv.OrasTestImage = fmt.Sprintf("oras://%s/oras_test_sif:latest", testenv.TestRegistry) t.Cleanup(func() { - os.Remove(imagePath) - os.Remove(ociArchivePath) - os.Remove(dockerArchivePath) + if !t.Failed() { + os.Remove(imagePath) + os.Remove(ociArchivePath) + os.Remove(dockerArchivePath) + } }) suite := testhelper.NewSuite(t, testenv) diff --git a/internal/pkg/runtime/launcher/oci/process_linux.go b/internal/pkg/runtime/launcher/oci/process_linux.go index 78402e320f..eae0d66050 100644 --- a/internal/pkg/runtime/launcher/oci/process_linux.go +++ b/internal/pkg/runtime/launcher/oci/process_linux.go @@ -99,6 +99,10 @@ func getProcessArgs(imageSpec imgspecv1.Image, process string, args []string) [] // Currently this is the user's tmpfs home directory (see --containall). // Because this is called after mounts have already been computed, we can count on l.cfg.HomeDir containing the right value, incorporating any custom home dir overrides (i.e., --home). func (l *Launcher) getProcessCwd() (dir string, err error) { + if len(l.cfg.CwdPath) > 0 { + return l.cfg.CwdPath, nil + } + return l.cfg.HomeDir, nil } From 58be7e55b56e048027f57bbd47f8c72e88826c44 Mon Sep 17 00:00:00 2001 From: preminger Date: Mon, 10 Apr 2023 09:34:21 -0400 Subject: [PATCH 094/114] oci: fix --home when running as root or fakeroot (sylabs/singularity#1530) Signed-off-by: Edita Kizinevic --- e2e/actions/oci.go | 4 ---- internal/pkg/runtime/launcher/oci/process_linux.go | 3 +++ 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/e2e/actions/oci.go b/e2e/actions/oci.go index d4d2f0c030..47a5f9ab9f 100644 --- a/e2e/actions/oci.go +++ b/e2e/actions/oci.go @@ -178,10 +178,6 @@ func (c actionTests) actionOciExec(t *testing.T) { e2e.ExpectOutput(e2e.RegexMatch, `\bHOME=/myhomeloc\b`), e2e.ExpectOutput(e2e.RegexMatch, `\btmpfs on /myhomeloc\b`), }, - skipProfiles: map[string]bool{ - e2e.OCIRootProfile.String(): true, - e2e.OCIFakerootProfile.String(): true, - }, exit: 0, }, { diff --git a/internal/pkg/runtime/launcher/oci/process_linux.go b/internal/pkg/runtime/launcher/oci/process_linux.go index eae0d66050..b7dc574e4a 100644 --- a/internal/pkg/runtime/launcher/oci/process_linux.go +++ b/internal/pkg/runtime/launcher/oci/process_linux.go @@ -51,6 +51,9 @@ func (l *Launcher) getProcess(ctx context.Context, imgSpec imgspecv1.Image, imag // --env flag can override --env-file and APPTAINERENV_ rtEnv = mergeMap(rtEnv, l.cfg.Env) + // Ensure HOME points to the required home directory, even if it is a custom one. + rtEnv["HOME"] = l.cfg.HomeDir + cwd, err := l.getProcessCwd() if err != nil { return nil, err From f033d46f6c63d3220d3c07ec2b31080b5d9ef547 Mon Sep 17 00:00:00 2001 From: Omer Preminger Date: Mon, 24 Apr 2023 13:58:05 -0400 Subject: [PATCH 095/114] oci: fix to not override HOME when container specifies USER (+ e2e test of this) Signed-off-by: Edita Kizinevic --- e2e/docker/docker.go | 89 ++++++++++++++----- .../pkg/runtime/launcher/oci/process_linux.go | 6 +- 2 files changed, 73 insertions(+), 22 deletions(-) diff --git a/e2e/docker/docker.go b/e2e/docker/docker.go index 14aa50dee9..0642d180ad 100644 --- a/e2e/docker/docker.go +++ b/e2e/docker/docker.go @@ -886,35 +886,86 @@ func (c ctx) testDockerCMDENTRYPOINT(t *testing.T) { // Check that the USER in a docker container is honored under --oci mode func (c ctx) testDockerUSER(t *testing.T) { + dockerURI := "docker://ghcr.io/apptainer/docker-user" tests := []struct { - name string - expectOutput string - profile e2e.Profile + name string + cmd string + args []string + expectOutputs []e2e.ApptainerCmdResultOp + profile e2e.Profile + expectExit int }{ // Sanity check apptainer native engine... no support for USER { name: "default", + cmd: "run", profile: e2e.UserProfile, - expectOutput: fmt.Sprintf("uid=%d(%s) gid=%d", - e2e.UserProfile.ContainerUser(t).UID, - e2e.UserProfile.ContainerUser(t).Name, - e2e.UserProfile.ContainerUser(t).GID), + args: []string{dockerURI}, + expectOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ContainMatch, fmt.Sprintf("uid=%d(%s) gid=%d", + e2e.UserProfile.ContainerUser(t).UID, + e2e.UserProfile.ContainerUser(t).Name, + e2e.UserProfile.ContainerUser(t).GID, + )), + }, }, // `--oci` modes (USER honored by default) { - name: "OCIUser", - profile: e2e.OCIUserProfile, - expectOutput: `uid=2000(testuser) gid=2000(testgroup)`, + name: "OCIUser", + cmd: "run", + profile: e2e.OCIUserProfile, + args: []string{dockerURI}, + expectOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ContainMatch, `uid=2000(testuser) gid=2000(testgroup)`), + }, }, { - name: "OCIFakeroot", - profile: e2e.OCIFakerootProfile, - expectOutput: `uid=0(root) gid=0(root)`, + name: "OCIFakeroot", + profile: e2e.OCIFakerootProfile, + args: []string{dockerURI}, + expectOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ContainMatch, `uid=0(root) gid=0(root)`), + }, }, { - name: "OCIRoot", - profile: e2e.OCIRootProfile, - expectOutput: `uid=2000(testuser) gid=2000(testgroup)`, + name: "OCIRoot", + cmd: "run", + profile: e2e.OCIRootProfile, + args: []string{dockerURI}, + expectOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ContainMatch, `uid=2000(testuser) gid=2000(testgroup)`), + }, + }, + // `--oci` modes: check that we don't override container-user's home directory + { + name: "OrigHomeOCIUser", + cmd: "exec", + profile: e2e.OCIUserProfile, + args: []string{dockerURI, "env"}, + expectOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.RegexMatch, `\bHOME=/home/testuser\b`), + }, + expectExit: 0, + }, + { + name: "OrigHomeOCIFakeroot", + cmd: "exec", + profile: e2e.OCIFakerootProfile, + args: []string{dockerURI, "env"}, + expectOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.RegexMatch, `\bHOME=/root\b`), + }, + expectExit: 0, + }, + { + name: "OrigHomeOCIRoot", + cmd: "exec", + profile: e2e.OCIRootProfile, + args: []string{dockerURI, "env"}, + expectOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.RegexMatch, `\bHOME=/home/testuser\b`), + }, + expectExit: 0, }, } @@ -924,10 +975,8 @@ func (c ctx) testDockerUSER(t *testing.T) { e2e.AsSubtest(tt.name), e2e.WithProfile(tt.profile), e2e.WithCommand("run"), - e2e.WithArgs("docker://ghcr.io/apptainer/docker-user"), - e2e.ExpectExit(0, - e2e.ExpectOutput(e2e.ContainMatch, tt.expectOutput), - ), + e2e.WithArgs(tt.args...), + e2e.ExpectExit(tt.expectExit, tt.expectOutputs...), ) } } diff --git a/internal/pkg/runtime/launcher/oci/process_linux.go b/internal/pkg/runtime/launcher/oci/process_linux.go index b7dc574e4a..f9b36b5600 100644 --- a/internal/pkg/runtime/launcher/oci/process_linux.go +++ b/internal/pkg/runtime/launcher/oci/process_linux.go @@ -51,8 +51,10 @@ func (l *Launcher) getProcess(ctx context.Context, imgSpec imgspecv1.Image, imag // --env flag can override --env-file and APPTAINERENV_ rtEnv = mergeMap(rtEnv, l.cfg.Env) - // Ensure HOME points to the required home directory, even if it is a custom one. - rtEnv["HOME"] = l.cfg.HomeDir + // Ensure HOME points to the required home directory, even if it is a custom one, unless the container explicitly specifies its USER, in which case we don't want to touch HOME. + if imgSpec.Config.User == "" { + rtEnv["HOME"] = l.cfg.HomeDir + } cwd, err := l.getProcessCwd() if err != nil { From 05a9efc9548dee9eb265fc896db87025f1572449 Mon Sep 17 00:00:00 2001 From: preminger Date: Thu, 11 May 2023 09:59:40 -0400 Subject: [PATCH 096/114] oci: support --overlay (sylabs/singularity#1659) * oci: support --overlay * support for multiple overlays, other revisions * fixup: deduplicate RunWrapped logic * lots of refactoring and cleanup * remove leftover debug-related panic call * cleanup comments, fix small issues w/erroring --------- Co-authored-by: David Trudgian Signed-off-by: Edita Kizinevic --- CHANGELOG.md | 4 + cmd/internal/cli/oci_linux.go | 12 + docs/content.go | 2 +- e2e/actions/actions.go | 1 + e2e/actions/oci.go | 97 ++++++- e2e/docker/docker.go | 12 +- e2e/docker/regressions.go | 2 +- e2e/imgbuild/imgbuild.go | 2 +- e2e/suite.go | 6 +- internal/app/apptainer/oci_linux.go | 4 +- .../runtime/launcher/oci/launcher_linux.go | 7 +- .../pkg/runtime/launcher/oci/oci_overlay.go | 112 ++++++++ .../runtime/launcher/oci/oci_runc_linux.go | 57 ++-- internal/pkg/runtime/launcher/options.go | 2 +- pkg/ocibundle/tools/overlay_linux.go | 246 +++++++++++++++--- 15 files changed, 466 insertions(+), 100 deletions(-) create mode 100644 internal/pkg/runtime/launcher/oci/oci_overlay.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 185aece478..af4f0a706d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,6 +70,10 @@ For older changes see the [archived Singularity change log](https://github.com/a `--dns` flag can be used to pass a comma-separated list of DNS servers that will be used in the container; if this flag is not used, the container will use the same `resolv.conf` settings as the host. +- OCI-mode now supports `--overlay ` flag, allowing writes to the + filesystem to persist across runs of the OCI container. If specified dir does + not exist, Apptainer will attempt to create it. Multiple overlays can be + specified, but all but one must be read-only (`--overlay :ro`). ### New Features & Functionality diff --git a/cmd/internal/cli/oci_linux.go b/cmd/internal/cli/oci_linux.go index 8eef6e974b..3d8afe5971 100644 --- a/cmd/internal/cli/oci_linux.go +++ b/cmd/internal/cli/oci_linux.go @@ -37,6 +37,17 @@ var ociBundleFlag = cmdline.Flag{ EnvKeys: []string{"BUNDLE"}, } +// -o|--overlay +var ociOverlayFlag = cmdline.Flag{ + ID: "ociOverlayFlag", + Value: &ociArgs.OverlayPaths, + DefaultValue: []string{}, + Name: "overlay", + ShortHand: "o", + Usage: "specify an overlay dir to use in lieu of a writable tmpfs", + Tag: "", +} + // -l|--log-path var ociLogPathFlag = cmdline.Flag{ ID: "ociLogPathFlag", @@ -130,6 +141,7 @@ func init() { cmdManager.RegisterFlagForCmd(&ociLogPathFlag, createRunCmd...) cmdManager.RegisterFlagForCmd(&ociLogFormatFlag, createRunCmd...) cmdManager.RegisterFlagForCmd(&ociPidFileFlag, createRunCmd...) + cmdManager.RegisterFlagForCmd(&ociOverlayFlag, OciRunWrappedCmd) cmdManager.RegisterFlagForCmd(&ociKillForceFlag, OciKillCmd) cmdManager.RegisterFlagForCmd(&ociKillSignalFlag, OciKillCmd) cmdManager.RegisterFlagForCmd(&ociUpdateFromFileFlag, OciUpdateCmd) diff --git a/docs/content.go b/docs/content.go index 9481dc50c1..1354270a23 100644 --- a/docs/content.go +++ b/docs/content.go @@ -1036,7 +1036,7 @@ Enterprise Performance Computing (EPC)` $ apptainer oci delete mycontainer` // Internal oci launcher use only - no user-facing docs - OciRunWrappedUse string = `run-wrapped -b [run options...] ` + OciRunWrappedUse string = `run-wrapped -b [-o ] [run options...] ` OciUpdateUse string = `update [update options...] ` OciUpdateShort string = `Update container cgroups resources (root user only)` diff --git a/e2e/actions/actions.go b/e2e/actions/actions.go index dc6db75a8a..9d1c1ffe67 100644 --- a/e2e/actions/actions.go +++ b/e2e/actions/actions.go @@ -2975,5 +2975,6 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { "ociCdi": c.actionOciCdi, // apptainer exec --oci --cdi "ociIDMaps": c.actionOciIDMaps, // check uid/gid mapping on host for --oci as user / --fakeroot "ociCompat": np(c.actionOciCompat), // --oci equivalence to native mode --compat + "ociOverlay": (c.actionOciOverlay), // --overlay in OCI mode } } diff --git a/e2e/actions/oci.go b/e2e/actions/oci.go index 47a5f9ab9f..171a4f6007 100644 --- a/e2e/actions/oci.go +++ b/e2e/actions/oci.go @@ -807,7 +807,7 @@ func (c actionTests) actionOciCdi(t *testing.T) { // Generate the command to be executed in the container // Start by printing all environment variables, to test using e2e.ContainMatch conditions later - execCmd := "/bin/env" + execCmd := "/usr/bin/env" // Add commands to test the presence of mapped devices. for _, d := range tt.DeviceNodes { @@ -970,3 +970,98 @@ func (c actionTests) actionOciCompat(t *testing.T) { ) } } + +// actionOciOverlay checks that --overlay functions correctly in OCI mode. +func (c actionTests) actionOciOverlay(t *testing.T) { + e2e.EnsureOCIArchive(t, c.env) + imageRef := "oci-archive:" + c.env.OCIArchivePath + + for _, profile := range []e2e.Profile{e2e.OCIRootProfile, e2e.OCIFakerootProfile} { + testDir, err := fs.MakeTmpDir(c.env.TestDir, "overlaytestdir", 0o755) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if !t.Failed() { + os.RemoveAll(testDir) + } + }) + + // Create a few read-only overlay subdirs under testDir + for i := 0; i < 3; i++ { + dirName := fmt.Sprintf("my_ro_ol_dir%d", i) + fullPath := filepath.Join(testDir, dirName) + if err = os.Mkdir(fullPath, 0o755); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if !t.Failed() { + os.RemoveAll(fullPath) + } + }) + if err = os.WriteFile( + filepath.Join(fullPath, fmt.Sprintf("testfile.%d", i)), + []byte(fmt.Sprintf("test_string_%d\n", i)), + 0o644); err != nil { + t.Fatal(err) + } + if err = os.WriteFile( + filepath.Join(fullPath, "maskable_testfile"), + []byte(fmt.Sprintf("maskable_string_%d\n", i)), + 0o644); err != nil { + t.Fatal(err) + } + } + + tests := []struct { + name string + args []string + exitCode int + wantOutputs []e2e.ApptainerCmdResultOp + }{ + { + name: "NewWritable", + args: []string{"--overlay", filepath.Join(testDir, "my_rw_ol_dir"), imageRef, "sh", "-c", "echo my_test_string > /my_test_file"}, + exitCode: 0, + }, + { + name: "ExistWritable", + args: []string{"--overlay", filepath.Join(testDir, "my_rw_ol_dir"), imageRef, "cat", "/my_test_file"}, + exitCode: 0, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ExactMatch, "my_test_string"), + }, + }, + { + name: "NonExistReadonly", + args: []string{"--overlay", filepath.Join(testDir, "my_ro_ol_dir_nonexistent:ro"), imageRef, "echo", "hi"}, + exitCode: 255, + }, + { + name: "SeveralReadonly", + args: []string{"--overlay", filepath.Join(testDir, "my_ro_ol_dir2:ro"), "--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"), imageRef, "cat", "/testfile.1", "/maskable_testfile"}, + exitCode: 0, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ContainMatch, "test_string_1"), + e2e.ExpectOutput(e2e.ContainMatch, "maskable_string_2"), + }, + }, + } + + t.Run(profile.String(), func(t *testing.T) { + for _, tt := range tests { + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(profile), + e2e.WithCommand("exec"), + e2e.WithArgs(tt.args...), + e2e.ExpectExit( + tt.exitCode, + tt.wantOutputs..., + ), + ) + } + }) + } +} diff --git a/e2e/docker/docker.go b/e2e/docker/docker.go index 0642d180ad..7dcb2a2d1f 100644 --- a/e2e/docker/docker.go +++ b/e2e/docker/docker.go @@ -492,28 +492,28 @@ func (c ctx) testDockerRegistry(t *testing.T) { dfd e2e.DefFileDetails }{ { - name: "BusyBox", + name: "Alpine", exit: 0, dfd: e2e.DefFileDetails{ Bootstrap: "docker", - From: fmt.Sprintf("%s/my-busybox", c.env.TestRegistry), + From: fmt.Sprintf("%s/my-alpine", c.env.TestRegistry), }, }, { - name: "BusyBoxRegistry", + name: "AlpineRegistry", exit: 0, dfd: e2e.DefFileDetails{ Bootstrap: "docker", - From: "my-busybox", + From: "my-alpine", Registry: c.env.TestRegistry, }, }, { - name: "BusyBoxNamespace", + name: "AlpineNamespace", exit: 255, dfd: e2e.DefFileDetails{ Bootstrap: "docker", - From: "my-busybox", + From: "my-alpine", Registry: c.env.TestRegistry, Namespace: "not-a-namespace", }, diff --git a/e2e/docker/regressions.go b/e2e/docker/regressions.go index bd7d153cb7..39170ea0bf 100644 --- a/e2e/docker/regressions.go +++ b/e2e/docker/regressions.go @@ -84,7 +84,7 @@ func (c ctx) issue4943(t *testing.T) { func (c ctx) issue5172(t *testing.T) { // create $HOME/.config/containers/registries.conf - regImage := fmt.Sprintf("docker://%s/my-busybox", c.env.TestRegistry) + regImage := fmt.Sprintf("docker://%s/my-alpine", c.env.TestRegistry) imagePath := filepath.Join(c.env.TestDir, "issue-5172") c.env.RunApptainer( diff --git a/e2e/imgbuild/imgbuild.go b/e2e/imgbuild/imgbuild.go index 3e881b0852..8032de8efd 100644 --- a/e2e/imgbuild/imgbuild.go +++ b/e2e/imgbuild/imgbuild.go @@ -1811,7 +1811,7 @@ func (c imgBuildTests) buildLibraryHost(t *testing.T) { ) } -// testWritableTmpfs checks that we can run the build using a writeable tmpfs in the %test step +// testWritableTmpfs checks that we can run the build using a writable tmpfs in the %test step func (c imgBuildTests) testWritableTmpfs(t *testing.T) { e2e.EnsureImage(t, c.env) diff --git a/e2e/suite.go b/e2e/suite.go index 303c4673d1..42328d7f91 100644 --- a/e2e/suite.go +++ b/e2e/suite.go @@ -170,9 +170,9 @@ func Run(t *testing.T) { testenv.OrasTestImage = fmt.Sprintf("oras://%s/oras_test_sif:latest", testenv.TestRegistry) // Provision local registry - testenv.TestRegistryImage = fmt.Sprintf("docker://%s/my-busybox:latest", testenv.TestRegistry) + testenv.TestRegistryImage = fmt.Sprintf("docker://%s/my-alpine:latest", testenv.TestRegistry) - // Copy small test image (busybox:latest) into local registry from DockerHub + // Copy small test image (alpine:latest) into local registry from DockerHub insecureSource := false insecureValue := os.Getenv("E2E_DOCKER_MIRROR_INSECURE") if insecureValue != "" { @@ -181,7 +181,7 @@ func Run(t *testing.T) { t.Fatalf("could not convert E2E_DOCKER_MIRROR_INSECURE=%s: %s", insecureValue, err) } } - e2e.CopyImage(t, "docker://busybox:latest", testenv.TestRegistryImage, insecureSource, true) + e2e.CopyImage(t, "docker://alpine:latest", testenv.TestRegistryImage, insecureSource, true) // SIF base test path, built on demand by e2e.EnsureImage imagePath := path.Join(name, "test.sif") diff --git a/internal/app/apptainer/oci_linux.go b/internal/app/apptainer/oci_linux.go index 25df12e4c2..9c654a21c0 100644 --- a/internal/app/apptainer/oci_linux.go +++ b/internal/app/apptainer/oci_linux.go @@ -27,6 +27,7 @@ import ( // OciArgs contains CLI arguments type OciArgs struct { BundlePath string + OverlayPaths []string LogPath string LogFormat string PidFile string @@ -52,7 +53,8 @@ func OciRunWrapped(ctx context.Context, containerID string, args *OciArgs) error if err != nil { return err } - return oci.RunWrapped(ctx, containerID, args.BundlePath, args.PidFile, systemdCgroups) + + return oci.RunWrapped(ctx, containerID, args.BundlePath, args.PidFile, args.OverlayPaths, systemdCgroups) } // OciCreate creates a container from an OCI bundle diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index f8731ac515..5ed0c3ff85 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -85,9 +85,6 @@ func checkOpts(lo launcher.Options) error { if lo.WritableTmpfs { sylog.Infof("--oci mode uses --writable-tmpfs by default") } - if len(lo.OverlayPaths) > 0 { - badOpt = append(badOpt, "OverlayPaths") - } if lo.WorkDir != "" { badOpt = append(badOpt, "WorkDir") } @@ -475,11 +472,11 @@ func (l *Launcher) Exec(ctx context.Context, image string, process string, args if os.Getuid() == 0 { // Execution of runc/crun run, wrapped with prep / cleanup. - err = RunWrapped(ctx, id.String(), b.Path(), "", l.apptainerConf.SystemdCgroups) + err = RunWrapped(ctx, id.String(), b.Path(), "", l.cfg.OverlayPaths, l.apptainerConf.SystemdCgroups) } else { // Reexec apptainer oci run in a userns with mappings. // Note - the oci run command will pull out the SystemdCgroups setting from config. - err = RunWrappedNS(ctx, id.String(), b.Path(), "") + err = RunWrappedNS(ctx, id.String(), b.Path(), l.cfg.OverlayPaths) } var exitErr *exec.ExitError if errors.As(err, &exitErr) { diff --git a/internal/pkg/runtime/launcher/oci/oci_overlay.go b/internal/pkg/runtime/launcher/oci/oci_overlay.go new file mode 100644 index 0000000000..7e0cacb5e2 --- /dev/null +++ b/internal/pkg/runtime/launcher/oci/oci_overlay.go @@ -0,0 +1,112 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2018-2023, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package oci + +import ( + "fmt" + "path/filepath" + "strings" + + "github.com/apptainer/apptainer/pkg/ocibundle/tools" + "github.com/apptainer/apptainer/pkg/sylog" + "github.com/apptainer/apptainer/pkg/util/apptainerconf" +) + +// WrapWithWritableTmpFs runs a function wrapped with prep / cleanup steps for a writable tmpfs. +func WrapWithWritableTmpFs(f func() error, bundleDir string) error { + // TODO: --oci mode always emulating --compat, which uses --writable-tmpfs. + // Provide a way of disabling this, for a read only rootfs. + overlayDir, err := prepareWritableTmpfs(bundleDir) + if err != nil { + return err + } + + err = f() + + // Cleanup actions log errors, but don't return - so we get as much cleanup done as possible. + if cleanupErr := cleanupWritableTmpfs(bundleDir, overlayDir); cleanupErr != nil { + sylog.Errorf("While cleaning up writable tmpfs: %v", cleanupErr) + } + + // Return any error from the actual container payload - preserve exit code. + return err +} + +// WrapWithOverlays runs a function wrapped with prep / cleanup steps for overlays. +func WrapWithOverlays(f func() error, bundleDir string, overlayPaths []string) error { + writableOverlayFound := false + ovs := tools.OverlaySet{} + for _, p := range overlayPaths { + writable := true + splitted := strings.SplitN(p, ":", 2) + barePath := splitted[0] + if len(splitted) > 1 { + if splitted[1] == "ro" { + writable = false + } + } + + if writable && writableOverlayFound { + return fmt.Errorf("you can't specify more than one writable overlay; %#v has already been specified as a writable overlay; use '--overlay %s:ro' instead", ovs.WritableLoc, barePath) + } + if writable { + writableOverlayFound = true + ovs.WritableLoc = barePath + } else { + ovs.ReadonlyLocs = append(ovs.ReadonlyLocs, barePath) + } + } + + rootFsDir := tools.RootFs(bundleDir).Path() + err := tools.ApplyOverlay(rootFsDir, ovs) + if err != nil { + return err + } + + err = f() + + // Cleanup actions log errors, but don't return - so we get as much cleanup done as possible. + if cleanupErr := tools.UnmountOverlay(rootFsDir); cleanupErr != nil { + sylog.Errorf("While unmounting rootfs overlay: %v", cleanupErr) + } + + // Return any error from the actual container payload - preserve exit code. + return err +} + +func prepareWritableTmpfs(bundleDir string) (string, error) { + sylog.Debugf("Configuring writable tmpfs overlay for %s", bundleDir) + c := apptainerconf.GetCurrentConfig() + if c == nil { + return "", fmt.Errorf("apptainer configuration is not initialized") + } + return tools.CreateOverlayTmpfs(bundleDir, int(c.SessiondirMaxSize)) +} + +func cleanupWritableTmpfs(bundleDir, overlayDir string) error { + sylog.Debugf("Cleaning up writable tmpfs overlay for %s", bundleDir) + return tools.DeleteOverlayTmpfs(bundleDir, overlayDir) +} + +// absOverlay takes an overlay description string (a path, optionally followed by a colon with an option string, like ":ro" or ":rw"), and replaces any relative path in the description string with an absolute one. +func absOverlay(desc string) (string, error) { + splitted := strings.SplitN(desc, ":", 2) + barePath := splitted[0] + absBarePath, err := filepath.Abs(barePath) + if err != nil { + return "", err + } + absDesc := absBarePath + if len(splitted) > 1 { + absDesc += ":" + splitted[1] + } + + return absDesc, nil +} diff --git a/internal/pkg/runtime/launcher/oci/oci_runc_linux.go b/internal/pkg/runtime/launcher/oci/oci_runc_linux.go index 5a960fff4a..264a2193de 100644 --- a/internal/pkg/runtime/launcher/oci/oci_runc_linux.go +++ b/internal/pkg/runtime/launcher/oci/oci_runc_linux.go @@ -2,7 +2,7 @@ // Apptainer a Series of LF Projects LLC. // For website terms of use, trademark policy, privacy policy and other // project policies see https://lfprojects.org/policies -// Copyright (c) 2018-2022, Sylabs Inc. All rights reserved. +// Copyright (c) 2018-2023, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -23,10 +23,8 @@ import ( "github.com/apptainer/apptainer/internal/pkg/buildcfg" fakerootConfig "github.com/apptainer/apptainer/internal/pkg/runtime/engine/fakeroot/config" "github.com/apptainer/apptainer/internal/pkg/util/starter" - "github.com/apptainer/apptainer/pkg/ocibundle/tools" "github.com/apptainer/apptainer/pkg/runtime/engine/config" "github.com/apptainer/apptainer/pkg/sylog" - "github.com/apptainer/apptainer/pkg/util/apptainerconf" ) // Delete deletes container resources @@ -226,44 +224,43 @@ func Run(ctx context.Context, containerID, bundlePath, pidFile string, systemdCg } // RunWrapped runs a container via the OCI runtime, wrapped with prep / cleanup steps. -func RunWrapped(ctx context.Context, containerID, bundlePath, pidFile string, systemdCgroups bool) error { - // TODO: --oci mode always emulating --compat, which uses --writable-tmpfs. - // Provide a way of disabling this, for a read only rootfs. - if err := prepareWriteableTmpfs(bundlePath); err != nil { - return err +func RunWrapped(ctx context.Context, containerID, bundlePath, pidFile string, overlayPaths []string, systemdCgroups bool) error { + runFunc := func() error { + return Run(ctx, containerID, bundlePath, "", systemdCgroups) } - err := Run(ctx, containerID, bundlePath, pidFile, systemdCgroups) - - // Cleanup actions log errors, but don't return - so we get as much cleanup done as possible. - if err := cleanupWritableTmpfs(bundlePath); err != nil { - sylog.Errorf("While cleaning up writable tmpfs: %v", err) + if len(overlayPaths) > 0 { + return WrapWithOverlays(runFunc, bundlePath, overlayPaths) } - // Return any error from the actual container payload - preserve exit code. - return err + return WrapWithWritableTmpFs(runFunc, bundlePath) } // RunWrappedNS reexecs apptainer in a user namespace, with supplied uid/gid mapping, calling oci run. -func RunWrappedNS(ctx context.Context, containerID, bundlePath, pidFile string) error { +func RunWrappedNS(ctx context.Context, containerID, bundlePath string, overlayPaths []string) error { absBundle, err := filepath.Abs(bundlePath) if err != nil { return fmt.Errorf("failed to determine bundle absolute path: %s", err) } - if err := os.Chdir(absBundle); err != nil { - return fmt.Errorf("failed to change directory to %s: %s", absBundle, err) - } - args := []string{ filepath.Join(buildcfg.BINDIR, "apptainer"), "oci", "run-wrapped", "-b", absBundle, - containerID, } - if pidFile != "" { - args = append(args, "--pid-file="+pidFile) + for _, p := range overlayPaths { + absPath, err := absOverlay(p) + if err != nil { + return fmt.Errorf("could not convert %q to absolute path: %w", p, err) + } + + args = append(args, "--overlay", absPath) + } + args = append(args, containerID) + + if err := os.Chdir(absBundle); err != nil { + return fmt.Errorf("failed to change directory to %s: %s", absBundle, err) } sylog.Debugf("Calling fakeroot engine to execute %q", strings.Join(args, " ")) @@ -367,17 +364,3 @@ func Update(containerID, cgFile string, systemdCgroups bool) error { sylog.Debugf("Calling %s with args %v", runtimeBin, runtimeArgs) return cmd.Run() } - -func prepareWriteableTmpfs(bundleDir string) error { - sylog.Debugf("Configuring writable tmpfs overlay for %s", bundleDir) - c := apptainerconf.GetCurrentConfig() - if c == nil { - return fmt.Errorf("apptainer configuration is not initialized") - } - return tools.CreateOverlayTmpfs(bundleDir, int(c.SessiondirMaxSize)) -} - -func cleanupWritableTmpfs(bundleDir string) error { - sylog.Debugf("Cleaning up writable tmpfs overlay for %s", bundleDir) - return tools.DeleteOverlay(bundleDir) -} diff --git a/internal/pkg/runtime/launcher/options.go b/internal/pkg/runtime/launcher/options.go index d9a51fb3b0..6ed9016a1b 100644 --- a/internal/pkg/runtime/launcher/options.go +++ b/internal/pkg/runtime/launcher/options.go @@ -29,7 +29,7 @@ type Namespaces struct { type Options struct { // Writable marks the container image itself as writable. Writable bool - // WriteableTmpfs applies an ephemeral writable overlay to the container. + // WritableTmpfs applies an ephemeral writable overlay to the container. WritableTmpfs bool // OverlayPaths holds paths to image or directory overlays to be applied. OverlayPaths []string diff --git a/pkg/ocibundle/tools/overlay_linux.go b/pkg/ocibundle/tools/overlay_linux.go index 1c2b9b414f..3b0a01cc29 100644 --- a/pkg/ocibundle/tools/overlay_linux.go +++ b/pkg/ocibundle/tools/overlay_linux.go @@ -13,18 +13,41 @@ import ( "fmt" "os" "path/filepath" + "strings" "syscall" ) -// CreateOverlay creates a writable overlay based on a directory. -func CreateOverlay(bundlePath string) error { - var err error +// OverlaySet represents a set of overlay directories which will be overlain on +// top of some filesystem mount point. The actual mount point atop which these +// directories will be overlain is not specified in the OverlaySet; it is left +// implicit, to be chosen by whichever function consumes an OverlaySet. An +// OverlaySet contains two types of directories: zero or more directories which +// will be mounted as read-only overlays atop the (implicit) mount point, and +// one directory which will be mounted as a writable overlay atop all the rest. +// An empty WritableLoc field indicates that no writable overlay is to be +// mounted. +type OverlaySet struct { + // ReadonlyLocs is a list of directories to be mounted as read-only + // overlays. The mount point atop which these will be mounted is left + // implicit, to be chosen by whichever function consumes the OverlaySet. + ReadonlyLocs []string + + // WritableLoc is the directory to be mounted as a writable overlay. The + // mount point atop which this will be mounted is left implicit, to be + // chosen by whichever function consumes the OverlaySet. Empty value + // indicates no writable overlay is to be mounted. + WritableLoc string +} +// CreateOverlay creates a writable overlay using a directory inside the OCI +// bundle. +func CreateOverlay(bundlePath string) error { oldumask := syscall.Umask(0) defer syscall.Umask(oldumask) overlayDir := filepath.Join(bundlePath, "overlay") - if err = os.Mkdir(overlayDir, 0o700); err != nil { + var err error + if err = ensureOverlayDir(overlayDir, true, 0o700); err != nil { return fmt.Errorf("failed to create %s: %s", overlayDir, err) } // delete overlay directory in case of error @@ -34,35 +57,31 @@ func CreateOverlay(bundlePath string) error { } }() - err = syscall.Mount(overlayDir, overlayDir, "", syscall.MS_BIND, "") - if err != nil { - return fmt.Errorf("failed to bind %s: %s", overlayDir, err) - } - // best effort to cleanup mount - defer func() { - if err != nil { - syscall.Unmount(overlayDir, syscall.MNT_DETACH) - } - }() - - if err = syscall.Mount("", overlayDir, "", syscall.MS_REMOUNT|syscall.MS_BIND, ""); err != nil { - return fmt.Errorf("failed to remount %s: %s", overlayDir, err) - } + return ApplyOverlay( + RootFs(bundlePath).Path(), + OverlaySet{WritableLoc: overlayDir}, + ) +} - err = prepareOverlay(bundlePath, overlayDir) - return err +// DeleteOverlay deletes an overlay previously created using a directory inside +// the OCI bundle. +func DeleteOverlay(bundlePath string) error { + overlayDir := filepath.Join(bundlePath, "overlay") + rootFsDir := RootFs(bundlePath).Path() + return unmountAndDeleteOverlay(rootFsDir, overlayDir) } -// CreateOverlay creates a writable overlay based on a tmpfs. -func CreateOverlayTmpfs(bundlePath string, sizeMiB int) error { +// CreateOverlay creates a writable overlay using tmpfs. +func CreateOverlayTmpfs(bundlePath string, sizeMiB int) (string, error) { var err error oldumask := syscall.Umask(0) defer syscall.Umask(oldumask) overlayDir := filepath.Join(bundlePath, "overlay") - if err = os.Mkdir(overlayDir, 0o700); err != nil { - return fmt.Errorf("failed to create %s: %s", overlayDir, err) + err = ensureOverlayDir(overlayDir, true, 0o700) + if err != nil { + return "", fmt.Errorf("failed to create %s: %s", overlayDir, err) } // delete overlay directory in case of error defer func() { @@ -74,7 +93,7 @@ func CreateOverlayTmpfs(bundlePath string, sizeMiB int) error { options := fmt.Sprintf("mode=1777,size=%dm", sizeMiB) err = syscall.Mount("tmpfs", overlayDir, "tmpfs", syscall.MS_NODEV, options) if err != nil { - return fmt.Errorf("failed to bind %s: %s", overlayDir, err) + return "", fmt.Errorf("failed to bind %s: %s", overlayDir, err) } // best effort to cleanup mount defer func() { @@ -83,41 +102,182 @@ func CreateOverlayTmpfs(bundlePath string, sizeMiB int) error { } }() - err = prepareOverlay(bundlePath, overlayDir) - return err + err = ApplyOverlay( + RootFs(bundlePath).Path(), + OverlaySet{WritableLoc: overlayDir}, + ) + if err != nil { + return "", err + } + + return overlayDir, nil +} + +// DeleteOverlayTmpfs deletes an overlay previously created using tmpfs. +func DeleteOverlayTmpfs(bundlePath, overlayDir string) error { + rootFsDir := RootFs(bundlePath).Path() + return unmountAndDeleteOverlay(rootFsDir, overlayDir) } -func prepareOverlay(bundlePath, overlayDir string) error { - upperDir := filepath.Join(overlayDir, "upper") - if err := os.Mkdir(upperDir, 0o755); err != nil { - return fmt.Errorf("failed to create %s: %s", upperDir, err) +// ApplyOverlay prepares and mounts the specified overlay +func ApplyOverlay(rootFsDir string, ovs OverlaySet) error { + // Prepare internal structure of writable overlay dir, if necessary + if len(ovs.WritableLoc) > 0 { + if err := ensureOverlayDir(ovs.WritableLoc, true, 0o755); err != nil { + return err + } + if err := prepareWritableOverlay(ovs.WritableLoc); err != nil { + return err + } } - workDir := filepath.Join(overlayDir, "work") - if err := os.Mkdir(workDir, 0o700); err != nil { - return fmt.Errorf("failed to create %s: %s", workDir, err) + + // Perform identity mounts for this OverlaySet + if err := performIdentityMounts(ovs); err != nil { + return err } - rootFsDir := RootFs(bundlePath).Path() - options := fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", rootFsDir, upperDir, workDir) + // Perform actual overlay mount + return performOverlayMount(rootFsDir, overlayOptions(rootFsDir, ovs)) +} + +// UnmountOverlay umounts an overlay +func UnmountOverlay(rootFsDir string) error { + if err := syscall.Unmount(rootFsDir, syscall.MNT_DETACH); err != nil { + return fmt.Errorf("failed to unmount %s: %s", rootFsDir, err) + } + + return nil +} + +// prepareWritableOverlay ensures that the upper and work subdirs of a writable +// overlay dir exist, and if not, creates them. +func prepareWritableOverlay(dir string) error { + if err := ensureOverlayDir(upperSubdirOf(dir), true, 0o755); err != nil { + return fmt.Errorf("err encountered while preparing upper subdir of overlay dir %q: %w", upperSubdirOf(dir), err) + } + if err := ensureOverlayDir(workSubdirOf(dir), true, 0o700); err != nil { + return fmt.Errorf("err encountered while preparing work subdir of overlay dir %q: %w", workSubdirOf(dir), err) + } + + return nil +} + +// performIdentityMounts creates the writable OverlaySet directory if it does +// not exist, and performs a bind mount & remount of every OverlaySet dir onto +// itself. The pattern of bind mount followed by remount allows application of +// more restrictive mount flags than are in force on the underlying filesystem. +func performIdentityMounts(ovs OverlaySet) error { + var err error + + locsToBind := ovs.ReadonlyLocs + if len(ovs.WritableLoc) > 0 { + // Check if writable overlay dir already exists; if it doesn't, try to + // create it. + if err = ensureOverlayDir(ovs.WritableLoc, true, 0o755); err != nil { + return err + } + + locsToBind = append(locsToBind, ovs.WritableLoc) + } + + // Try to do initial bind-mounts + for _, d := range locsToBind { + if err = ensureOverlayDir(d, false, 0); err != nil { + return fmt.Errorf("error accessing directory %s: %s", d, err) + } + + if err = syscall.Mount(d, d, "", syscall.MS_BIND, ""); err != nil { + return fmt.Errorf("failed to bind %s: %s", d, err) + } + + // best effort to cleanup mount + defer func() { + if err != nil { + syscall.Unmount(d, syscall.MNT_DETACH) + } + }() + + // Try to perform remount + if err = syscall.Mount("", d, "", syscall.MS_REMOUNT|syscall.MS_BIND, ""); err != nil { + return fmt.Errorf("failed to remount %s: %s", d, err) + } + } + + return err +} + +// overlayOptions creates the options string to be used in an overlay mount +func overlayOptions(rootFsDir string, ovs OverlaySet) string { + // Create lowerdir argument of options string + lowerDirJoined := strings.Join(append(ovs.ReadonlyLocs, rootFsDir), ":") + + if len(ovs.WritableLoc) > 0 { + return fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", lowerDirJoined, upperSubdirOf(ovs.WritableLoc), workSubdirOf(ovs.WritableLoc)) + } + + return fmt.Sprintf("lowerdir=%s", lowerDirJoined) +} + +// performOverlayMount mounts an overlay atop a given rootfs directory +func performOverlayMount(rootFsDir, options string) error { + // Try to perform actual mount if err := syscall.Mount("overlay", rootFsDir, "overlay", 0, options); err != nil { - return fmt.Errorf("failed to mount %s: %s", overlayDir, err) + return fmt.Errorf("failed to mount %s: %s", rootFsDir, err) + } + + return nil +} + +// ensureOverlayDir checks if a directory already exists; if it doesn't, and +// createIfMissing is true, it attempts to create it with the specified +// permissions. +func ensureOverlayDir(dir string, createIfMissing bool, createPerm os.FileMode) error { + if len(dir) == 0 { + return fmt.Errorf("internal error: ensureOverlayDir() called with empty dir name") + } + + _, err := os.Stat(dir) + if err == nil { + return nil + } + + if !os.IsNotExist(err) { + return err + } + + if !createIfMissing { + return fmt.Errorf("missing overlay dir %q", dir) + } + + // Create the requested dir + if err := os.Mkdir(dir, createPerm); err != nil { + return fmt.Errorf("failed to create %q: %s", dir, err) } + return nil } -// DeleteOverlay deletes overlay -func DeleteOverlay(bundlePath string) error { - overlayDir := filepath.Join(bundlePath, "overlay") - rootFsDir := RootFs(bundlePath).Path() +func upperSubdirOf(overlayDir string) string { + return filepath.Join(overlayDir, "upper") +} - if err := syscall.Unmount(rootFsDir, syscall.MNT_DETACH); err != nil { - return fmt.Errorf("failed to unmount %s: %s", rootFsDir, err) +func workSubdirOf(overlayDir string) string { + return filepath.Join(overlayDir, "work") +} + +// unmountAndDeleteOverlay unmounts and deletes a previously-created overlay. +func unmountAndDeleteOverlay(rootFsDir, overlayDir string) error { + if err := UnmountOverlay(rootFsDir); err != nil { + return err } + if err := syscall.Unmount(overlayDir, syscall.MNT_DETACH); err != nil { return fmt.Errorf("failed to unmount %s: %s", overlayDir, err) } + if err := os.RemoveAll(overlayDir); err != nil { return fmt.Errorf("failed to remove %s: %s", overlayDir, err) } + return nil } From 061b653228deef757db1a93394a21c9252d8e00e Mon Sep 17 00:00:00 2001 From: Omer Preminger Date: Thu, 11 May 2023 14:10:06 -0400 Subject: [PATCH 097/114] wrap in writabletmpfs overlay if there's only read-only overlay(s) Signed-off-by: Edita Kizinevic --- .../pkg/runtime/launcher/oci/oci_overlay.go | 7 ++- .../runtime/launcher/oci/oci_runc_linux.go | 2 +- pkg/ocibundle/tools/overlay_linux.go | 56 ++++++++++++++----- 3 files changed, 49 insertions(+), 16 deletions(-) diff --git a/internal/pkg/runtime/launcher/oci/oci_overlay.go b/internal/pkg/runtime/launcher/oci/oci_overlay.go index 7e0cacb5e2..2451b16cc9 100644 --- a/internal/pkg/runtime/launcher/oci/oci_overlay.go +++ b/internal/pkg/runtime/launcher/oci/oci_overlay.go @@ -24,6 +24,7 @@ func WrapWithWritableTmpFs(f func() error, bundleDir string) error { // TODO: --oci mode always emulating --compat, which uses --writable-tmpfs. // Provide a way of disabling this, for a read only rootfs. overlayDir, err := prepareWritableTmpfs(bundleDir) + sylog.Debugf("Done with prepareWritableTmpfs; overlayDir is: %q", overlayDir) if err != nil { return err } @@ -70,7 +71,11 @@ func WrapWithOverlays(f func() error, bundleDir string, overlayPaths []string) e return err } - err = f() + if writableOverlayFound { + err = f() + } else { + err = WrapWithWritableTmpFs(f, bundleDir) + } // Cleanup actions log errors, but don't return - so we get as much cleanup done as possible. if cleanupErr := tools.UnmountOverlay(rootFsDir); cleanupErr != nil { diff --git a/internal/pkg/runtime/launcher/oci/oci_runc_linux.go b/internal/pkg/runtime/launcher/oci/oci_runc_linux.go index 264a2193de..bfa18b41f5 100644 --- a/internal/pkg/runtime/launcher/oci/oci_runc_linux.go +++ b/internal/pkg/runtime/launcher/oci/oci_runc_linux.go @@ -226,7 +226,7 @@ func Run(ctx context.Context, containerID, bundlePath, pidFile string, systemdCg // RunWrapped runs a container via the OCI runtime, wrapped with prep / cleanup steps. func RunWrapped(ctx context.Context, containerID, bundlePath, pidFile string, overlayPaths []string, systemdCgroups bool) error { runFunc := func() error { - return Run(ctx, containerID, bundlePath, "", systemdCgroups) + return Run(ctx, containerID, bundlePath, pidFile, systemdCgroups) } if len(overlayPaths) > 0 { diff --git a/pkg/ocibundle/tools/overlay_linux.go b/pkg/ocibundle/tools/overlay_linux.go index 3b0a01cc29..eea6502eae 100644 --- a/pkg/ocibundle/tools/overlay_linux.go +++ b/pkg/ocibundle/tools/overlay_linux.go @@ -2,7 +2,7 @@ // Apptainer a Series of LF Projects LLC. // For website terms of use, trademark policy, privacy policy and other // project policies see https://lfprojects.org/policies -// Copyright (c) 2019, Sylabs Inc. All rights reserved. +// Copyright (c) 2019-2023, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -15,6 +15,8 @@ import ( "path/filepath" "strings" "syscall" + + "github.com/apptainer/apptainer/pkg/sylog" ) // OverlaySet represents a set of overlay directories which will be overlain on @@ -53,6 +55,7 @@ func CreateOverlay(bundlePath string) error { // delete overlay directory in case of error defer func() { if err != nil { + sylog.Debugf("Encountered error in CreateOverlay; attempting to remove overlayDir %q", overlayDir) os.RemoveAll(overlayDir) } }() @@ -68,7 +71,12 @@ func CreateOverlay(bundlePath string) error { func DeleteOverlay(bundlePath string) error { overlayDir := filepath.Join(bundlePath, "overlay") rootFsDir := RootFs(bundlePath).Path() - return unmountAndDeleteOverlay(rootFsDir, overlayDir) + + if err := detachMount(rootFsDir); err != nil { + return err + } + + return detachAndDelete(overlayDir) } // CreateOverlay creates a writable overlay using tmpfs. @@ -86,6 +94,7 @@ func CreateOverlayTmpfs(bundlePath string, sizeMiB int) (string, error) { // delete overlay directory in case of error defer func() { if err != nil { + sylog.Debugf("Encountered error in CreateOverlay; attempting to remove overlayDir %q", overlayDir) os.RemoveAll(overlayDir) } }() @@ -98,6 +107,7 @@ func CreateOverlayTmpfs(bundlePath string, sizeMiB int) (string, error) { // best effort to cleanup mount defer func() { if err != nil { + sylog.Debugf("Encountered error in CreateOverlayTmpfs; attempting to detach overlayDir %q", overlayDir) syscall.Unmount(overlayDir, syscall.MNT_DETACH) } }() @@ -116,7 +126,18 @@ func CreateOverlayTmpfs(bundlePath string, sizeMiB int) (string, error) { // DeleteOverlayTmpfs deletes an overlay previously created using tmpfs. func DeleteOverlayTmpfs(bundlePath, overlayDir string) error { rootFsDir := RootFs(bundlePath).Path() - return unmountAndDeleteOverlay(rootFsDir, overlayDir) + + if err := detachMount(rootFsDir); err != nil { + return err + } + + // Because CreateOverlayTmpfs() mounts the tmpfs on overlayDir, and then + // calls ApplyOverlay(), there needs to be an extra unmount in the this case + if err := detachMount(overlayDir); err != nil { + return err + } + + return detachAndDelete(overlayDir) } // ApplyOverlay prepares and mounts the specified overlay @@ -142,19 +163,17 @@ func ApplyOverlay(rootFsDir string, ovs OverlaySet) error { // UnmountOverlay umounts an overlay func UnmountOverlay(rootFsDir string) error { - if err := syscall.Unmount(rootFsDir, syscall.MNT_DETACH); err != nil { - return fmt.Errorf("failed to unmount %s: %s", rootFsDir, err) - } - - return nil + return detachMount(rootFsDir) } // prepareWritableOverlay ensures that the upper and work subdirs of a writable // overlay dir exist, and if not, creates them. func prepareWritableOverlay(dir string) error { + sylog.Debugf("Ensuring %q exists", upperSubdirOf(dir)) if err := ensureOverlayDir(upperSubdirOf(dir), true, 0o755); err != nil { return fmt.Errorf("err encountered while preparing upper subdir of overlay dir %q: %w", upperSubdirOf(dir), err) } + sylog.Debugf("Ensuring %q exists", workSubdirOf(dir)) if err := ensureOverlayDir(workSubdirOf(dir), true, 0o700); err != nil { return fmt.Errorf("err encountered while preparing work subdir of overlay dir %q: %w", workSubdirOf(dir), err) } @@ -186,6 +205,7 @@ func performIdentityMounts(ovs OverlaySet) error { return fmt.Errorf("error accessing directory %s: %s", d, err) } + sylog.Debugf("Performing identity bind-mount of %q", d) if err = syscall.Mount(d, d, "", syscall.MS_BIND, ""); err != nil { return fmt.Errorf("failed to bind %s: %s", d, err) } @@ -193,11 +213,13 @@ func performIdentityMounts(ovs OverlaySet) error { // best effort to cleanup mount defer func() { if err != nil { + sylog.Debugf("Encountered error with current OverlaySet; attempting to unmount %q", d) syscall.Unmount(d, syscall.MNT_DETACH) } }() // Try to perform remount + sylog.Debugf("Performing remount of %q", d) if err = syscall.Mount("", d, "", syscall.MS_REMOUNT|syscall.MS_BIND, ""); err != nil { return fmt.Errorf("failed to remount %s: %s", d, err) } @@ -221,6 +243,7 @@ func overlayOptions(rootFsDir string, ovs OverlaySet) string { // performOverlayMount mounts an overlay atop a given rootfs directory func performOverlayMount(rootFsDir, options string) error { // Try to perform actual mount + sylog.Debugf("Mounting overlay with rootFsDir %q, options: %q", rootFsDir, options) if err := syscall.Mount("overlay", rootFsDir, "overlay", 0, options); err != nil { return fmt.Errorf("failed to mount %s: %s", rootFsDir, err) } @@ -265,19 +288,24 @@ func workSubdirOf(overlayDir string) string { return filepath.Join(overlayDir, "work") } -// unmountAndDeleteOverlay unmounts and deletes a previously-created overlay. -func unmountAndDeleteOverlay(rootFsDir, overlayDir string) error { - if err := UnmountOverlay(rootFsDir); err != nil { - return err - } - +func detachAndDelete(overlayDir string) error { + sylog.Debugf("Detaching overlayDir %q", overlayDir) if err := syscall.Unmount(overlayDir, syscall.MNT_DETACH); err != nil { return fmt.Errorf("failed to unmount %s: %s", overlayDir, err) } + sylog.Debugf("Removing overlayDir %q", overlayDir) if err := os.RemoveAll(overlayDir); err != nil { return fmt.Errorf("failed to remove %s: %s", overlayDir, err) } + return nil +} + +func detachMount(dir string) error { + sylog.Debugf("Calling syscall.Unmount() to detach %q", dir) + if err := syscall.Unmount(dir, syscall.MNT_DETACH); err != nil { + return fmt.Errorf("failed to detach %s: %s", dir, err) + } return nil } From f96e620304b7c39242ecabe52cf8c00a303dbfdf Mon Sep 17 00:00:00 2001 From: Omer Preminger Date: Fri, 12 May 2023 10:07:34 -0400 Subject: [PATCH 098/114] improved unmounting of user-supplied overlays Signed-off-by: Edita Kizinevic --- .../pkg/runtime/launcher/oci/oci_overlay.go | 2 +- pkg/ocibundle/tools/overlay_linux.go | 34 +++++++++++++++++-- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/internal/pkg/runtime/launcher/oci/oci_overlay.go b/internal/pkg/runtime/launcher/oci/oci_overlay.go index 2451b16cc9..15ddcb0dcb 100644 --- a/internal/pkg/runtime/launcher/oci/oci_overlay.go +++ b/internal/pkg/runtime/launcher/oci/oci_overlay.go @@ -78,7 +78,7 @@ func WrapWithOverlays(f func() error, bundleDir string, overlayPaths []string) e } // Cleanup actions log errors, but don't return - so we get as much cleanup done as possible. - if cleanupErr := tools.UnmountOverlay(rootFsDir); cleanupErr != nil { + if cleanupErr := tools.UnmountOverlay(rootFsDir, ovs); cleanupErr != nil { sylog.Errorf("While unmounting rootfs overlay: %v", cleanupErr) } diff --git a/pkg/ocibundle/tools/overlay_linux.go b/pkg/ocibundle/tools/overlay_linux.go index eea6502eae..d212f2c514 100644 --- a/pkg/ocibundle/tools/overlay_linux.go +++ b/pkg/ocibundle/tools/overlay_linux.go @@ -162,8 +162,12 @@ func ApplyOverlay(rootFsDir string, ovs OverlaySet) error { } // UnmountOverlay umounts an overlay -func UnmountOverlay(rootFsDir string) error { - return detachMount(rootFsDir) +func UnmountOverlay(rootFsDir string, ovs OverlaySet) error { + if err := detachMount(rootFsDir); err != nil { + return err + } + + return detachIdentityMounts(ovs) } // prepareWritableOverlay ensures that the upper and work subdirs of a writable @@ -228,6 +232,32 @@ func performIdentityMounts(ovs OverlaySet) error { return err } +// detachIdentityMounts detaches mounts created by the bind-mount & remount +// pattern (as implemented in performIdentityMounts()) +func detachIdentityMounts(ovs OverlaySet) error { + locsToDetach := ovs.ReadonlyLocs + if len(ovs.WritableLoc) > 0 { + locsToDetach = append(locsToDetach, ovs.WritableLoc) + } + + // Don't stop on the first error; try to clean up as much as possible, and + // then return the first error encountered. + errors := []error{} + for _, d := range locsToDetach { + err := detachMount(d) + if err != nil { + sylog.Errorf("Error encountered trying to detach identity mount %s: %s", d, err) + errors = append(errors, err) + } + } + + if len(errors) > 0 { + return errors[0] + } + + return nil +} + // overlayOptions creates the options string to be used in an overlay mount func overlayOptions(rootFsDir string, ovs OverlaySet) string { // Create lowerdir argument of options string From 002535d6407302fef2d9c87474ddfef40cd2e58e Mon Sep 17 00:00:00 2001 From: Omer Preminger Date: Fri, 12 May 2023 07:03:13 -0400 Subject: [PATCH 099/114] added e2e testing for 1670 regression Signed-off-by: Edita Kizinevic --- e2e/actions/oci.go | 7 ++++++- e2e/docker/docker.go | 1 + e2e/docker/regressions.go | 23 +++++++++++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/e2e/actions/oci.go b/e2e/actions/oci.go index 171a4f6007..387f230c18 100644 --- a/e2e/actions/oci.go +++ b/e2e/actions/oci.go @@ -976,7 +976,7 @@ func (c actionTests) actionOciOverlay(t *testing.T) { e2e.EnsureOCIArchive(t, c.env) imageRef := "oci-archive:" + c.env.OCIArchivePath - for _, profile := range []e2e.Profile{e2e.OCIRootProfile, e2e.OCIFakerootProfile} { + for _, profile := range e2e.OCIProfiles { testDir, err := fs.MakeTmpDir(c.env.TestDir, "overlaytestdir", 0o755) if err != nil { t.Fatal(err) @@ -1037,6 +1037,11 @@ func (c actionTests) actionOciOverlay(t *testing.T) { args: []string{"--overlay", filepath.Join(testDir, "my_ro_ol_dir_nonexistent:ro"), imageRef, "echo", "hi"}, exitCode: 255, }, + { + name: "ReadonlyAddsTmpfs", + args: []string{"--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"), imageRef, "sh", "-c", "echo this_should_disappear > /my_test_file"}, + exitCode: 0, + }, { name: "SeveralReadonly", args: []string{"--overlay", filepath.Join(testDir, "my_ro_ol_dir2:ro"), "--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"), imageRef, "cat", "/testfile.1", "/maskable_testfile"}, diff --git a/e2e/docker/docker.go b/e2e/docker/docker.go index 7dcb2a2d1f..1d5d0195c5 100644 --- a/e2e/docker/docker.go +++ b/e2e/docker/docker.go @@ -1008,6 +1008,7 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { t.Run("issue 1286", c.issue1286) t.Run("issue 1528", c.issue1528) t.Run("issue 1586", c.issue1586) + t.Run("issue 1670", c.issue1670) }, // Tests that are especially slow, or run against a local docker // registry, can be run in parallel, with `--disable-cache` used within diff --git a/e2e/docker/regressions.go b/e2e/docker/regressions.go index 39170ea0bf..c3dcdeb932 100644 --- a/e2e/docker/regressions.go +++ b/e2e/docker/regressions.go @@ -318,3 +318,26 @@ func (c ctx) issue1586(t *testing.T) { t.Errorf("TMPDIR is not empty after apptainer exited") } } + +// https://github.com/sylabs/singularity/issues/1670 +// Check that runc/crun can add directories the rootfs before entering the +// container, by running a container based on busybox that lacks, e.g., /proc +func (c ctx) issue1670(t *testing.T) { + for _, profile := range e2e.OCIProfiles { + tmpDir, cleanup := e2e.MakeTempDir(t, c.env.TestDir, fmt.Sprintf("issue1670-%s-", profile.String()), "") + t.Cleanup(func() { + if !t.Failed() { + cleanup(t) + } + }) + + c.env.RunApptainer( + t, + e2e.AsSubtest(profile.String()), + e2e.WithProfile(profile), + e2e.WithCommand("exec"), + e2e.WithArgs("--overlay", fmt.Sprintf("%s:ro", tmpDir), "docker://busybox", "echo", "hi"), + e2e.ExpectExit(0), + ) + } +} From 3bcbf6838e0daa8ae674b88bc21d0b2a3dd89cf5 Mon Sep 17 00:00:00 2001 From: preminger Date: Fri, 12 May 2023 12:45:56 -0400 Subject: [PATCH 100/114] oci: add e2e OCIRoot test to check proper unmounting of overlays (sylabs/singularity#1678) Signed-off-by: Edita Kizinevic --- e2e/actions/actions.go | 19 ++++++++------- e2e/actions/oci.go | 55 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 9 deletions(-) diff --git a/e2e/actions/actions.go b/e2e/actions/actions.go index 9d1c1ffe67..0f75f082df 100644 --- a/e2e/actions/actions.go +++ b/e2e/actions/actions.go @@ -2967,14 +2967,15 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { // // OCI Runtime Mode // - "ociRun": c.actionOciRun, // apptainer run --oci - "ociExec": c.actionOciExec, // apptainer exec --oci - "ociShell": c.actionOciShell, // apptainer shell --oci - "ociNetwork": c.actionOciNetwork, // apptainer exec --oci --net - "ociBinds": c.actionOciBinds, // apptainer exec --oci --bind / --mount - "ociCdi": c.actionOciCdi, // apptainer exec --oci --cdi - "ociIDMaps": c.actionOciIDMaps, // check uid/gid mapping on host for --oci as user / --fakeroot - "ociCompat": np(c.actionOciCompat), // --oci equivalence to native mode --compat - "ociOverlay": (c.actionOciOverlay), // --overlay in OCI mode + "ociRun": c.actionOciRun, // apptainer run --oci + "ociExec": c.actionOciExec, // apptainer exec --oci + "ociShell": c.actionOciShell, // apptainer shell --oci + "ociNetwork": c.actionOciNetwork, // apptainer exec --oci --net + "ociBinds": c.actionOciBinds, // apptainer exec --oci --bind / --mount + "ociCdi": c.actionOciCdi, // apptainer exec --oci --cdi + "ociIDMaps": c.actionOciIDMaps, // check uid/gid mapping on host for --oci as user / --fakeroot + "ociCompat": np(c.actionOciCompat), // --oci equivalence to native mode --compat + "ociOverlay": (c.actionOciOverlay), // --overlay in OCI mode + "ociOverlayTeardown": np(c.actionOciOverlayTeardown), // proper overlay unmounting in OCI mode } } diff --git a/e2e/actions/oci.go b/e2e/actions/oci.go index 387f230c18..044687ef8f 100644 --- a/e2e/actions/oci.go +++ b/e2e/actions/oci.go @@ -10,6 +10,7 @@ package actions import ( + "bufio" "encoding/json" "fmt" "os" @@ -23,6 +24,7 @@ import ( "github.com/apptainer/apptainer/e2e/internal/e2e" "github.com/apptainer/apptainer/internal/pkg/util/fs" cdispecs "github.com/container-orchestrated-devices/container-device-interface/specs-go" + "gotest.tools/v3/assert" ) func (c actionTests) actionOciRun(t *testing.T) { @@ -1070,3 +1072,56 @@ func (c actionTests) actionOciOverlay(t *testing.T) { }) } } + +// actionOciOverlayTeardown checks that OCI-mode overlays are correctly +// unmounted even in root mode (i.e., when user namespaces are not involved). +func (c actionTests) actionOciOverlayTeardown(t *testing.T) { + e2e.EnsureOCIArchive(t, c.env) + imageRef := "oci-archive:" + c.env.OCIArchivePath + + const mountInfoPath string = "/proc/self/mountinfo" + numMountLinesPre, err := countLines(mountInfoPath) + if err != nil { + t.Fatal(err) + } + + tmpDir, cleanup := e2e.MakeTempDir(t, c.env.TestDir, "oci_overlay_teardown-", "") + t.Cleanup(func() { + if !t.Failed() { + cleanup(t) + } + }) + + c.env.RunApptainer( + t, + e2e.WithProfile(e2e.OCIRootProfile), + e2e.WithCommand("exec"), + e2e.WithArgs("--overlay", tmpDir+":ro", imageRef, "/bin/true"), + e2e.ExpectExit(0), + ) + + numMountLinesPost, err := countLines(mountInfoPath) + if err != nil { + t.Fatal(err) + } + + assert.Equal( + t, numMountLinesPost, numMountLinesPre, + "Number of mounts after running in OCI-mode with overlays (%d) does not match the number before the run (%d)", numMountLinesPost, numMountLinesPre) +} + +func countLines(path string) (int, error) { + file, err := os.Open(path) + if err != nil { + return -1, err + } + defer file.Close() + scanner := bufio.NewScanner(file) + scanner.Split(bufio.ScanLines) + lines := 0 + for scanner.Scan() { + lines++ + } + + return lines, nil +} From c8508ef6e1a6dec576b4eba60ebca7ed5743d92e Mon Sep 17 00:00:00 2001 From: Omer Preminger Date: Wed, 17 May 2023 10:49:43 -0400 Subject: [PATCH 101/114] chg --workdir usage msg: remove mention of $HOME Signed-off-by: Edita Kizinevic --- cmd/internal/cli/action_flags.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/internal/cli/action_flags.go b/cmd/internal/cli/action_flags.go index 8bf5d8a674..ac7c3453dd 100644 --- a/cmd/internal/cli/action_flags.go +++ b/cmd/internal/cli/action_flags.go @@ -179,7 +179,7 @@ var actionWorkdirFlag = cmdline.Flag{ DefaultValue: "", Name: "workdir", ShortHand: "W", - Usage: "working directory to be used for /tmp, /var/tmp and $HOME (if -c/--contain was also used)", + Usage: "working directory to be used for /tmp and /var/tmp (if -c/--contain was also used)", EnvKeys: []string{"WORKDIR"}, Tag: "", } From 11992df3a27cf851d1b186493fce5dcbcaf60157 Mon Sep 17 00:00:00 2001 From: Omer Preminger Date: Wed, 17 May 2023 16:01:18 -0400 Subject: [PATCH 102/114] oci: support --workdir Signed-off-by: Edita Kizinevic --- CHANGELOG.md | 6 + e2e/actions/actions.go | 36 +---- e2e/actions/common.go | 54 ++++++++ e2e/actions/oci.go | 98 +++++++++---- .../runtime/launcher/oci/launcher_linux.go | 4 - .../pkg/runtime/launcher/oci/mounts_linux.go | 131 +++++++++++++++--- 6 files changed, 247 insertions(+), 82 deletions(-) create mode 100644 e2e/actions/common.go diff --git a/CHANGELOG.md b/CHANGELOG.md index af4f0a706d..5ac710a64e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,6 +74,12 @@ For older changes see the [archived Singularity change log](https://github.com/a filesystem to persist across runs of the OCI container. If specified dir does not exist, Apptainer will attempt to create it. Multiple overlays can be specified, but all but one must be read-only (`--overlay :ro`). +- OCI-mode now supports the `--workdir ` option. If this option is + specified, `/tmp` and `/var/tmp` will be mapped, respectively, to + `/tmp` and `/var_tmp` on the host, rather than to tmpfs + storage. If `--scratch ` is used in conjunction with `--workdir`, + scratch directories will be mapped to subdirectories nested under + `/scratch` on the host, rather than to tmpfs storage. ### New Features & Functionality diff --git a/e2e/actions/actions.go b/e2e/actions/actions.go index 0f75f082df..122e53dd74 100644 --- a/e2e/actions/actions.go +++ b/e2e/actions/actions.go @@ -2,7 +2,7 @@ // Apptainer a Series of LF Projects LLC. // For website terms of use, trademark policy, privacy policy and other // project policies see https://lfprojects.org/policies -// Copyright (c) 2019-2022, Sylabs Inc. All rights reserved. +// Copyright (c) 2019-2023, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -1132,39 +1132,7 @@ func (c actionTests) actionBinds(t *testing.T) { hostWorkDir := filepath.Join(workspace, "workdir") createWorkspaceDirs := func(t *testing.T) { - e2e.Privileged(func(t *testing.T) { - if err := os.RemoveAll(hostCanaryDir); err != nil && !os.IsNotExist(err) { - t.Fatalf("failed to delete canary_dir: %s", err) - } - if err := os.RemoveAll(hostHomeDir); err != nil && !os.IsNotExist(err) { - t.Fatalf("failed to delete workspace home: %s", err) - } - if err := os.RemoveAll(hostWorkDir); err != nil && !os.IsNotExist(err) { - t.Fatalf("failed to delete workspace work: %s", err) - } - })(t) - - if err := fs.Mkdir(hostCanaryDir, 0o777); err != nil { - t.Fatalf("failed to create canary_dir: %s", err) - } - if err := fs.Touch(hostCanaryFile); err != nil { - t.Fatalf("failed to create canary_file: %s", err) - } - if err := fs.Touch(hostCanaryFileWithComma); err != nil { - t.Fatalf("failed to create canary_file_comma: %s", err) - } - if err := fs.Touch(hostCanaryFileWithColon); err != nil { - t.Fatalf("failed to create canary_file_colon: %s", err) - } - if err := os.Chmod(hostCanaryFile, 0o777); err != nil { - t.Fatalf("failed to apply permissions on canary_file: %s", err) - } - if err := fs.Mkdir(hostHomeDir, 0o777); err != nil { - t.Fatalf("failed to create workspace home directory: %s", err) - } - if err := fs.Mkdir(hostWorkDir, 0o777); err != nil { - t.Fatalf("failed to create workspace work directory: %s", err) - } + workspaceDirsGenerator(t, hostCanaryDir, hostHomeDir, hostWorkDir, hostCanaryFile, hostCanaryFileWithComma, hostCanaryFileWithColon) } // convert test image to sandbox diff --git a/e2e/actions/common.go b/e2e/actions/common.go new file mode 100644 index 0000000000..228e6bf7e3 --- /dev/null +++ b/e2e/actions/common.go @@ -0,0 +1,54 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2023, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package actions + +import ( + "os" + "testing" + + "github.com/apptainer/apptainer/e2e/internal/e2e" + "github.com/apptainer/apptainer/internal/pkg/util/fs" +) + +func workspaceDirsGenerator(t *testing.T, hostCanaryDir, hostHomeDir, hostWorkDir, hostCanaryFile, hostCanaryFileWithComma, hostCanaryFileWithColon string) { + e2e.Privileged(func(t *testing.T) { + if err := os.RemoveAll(hostCanaryDir); err != nil && !os.IsNotExist(err) { + t.Fatalf("failed to delete canary_dir: %s", err) + } + if err := os.RemoveAll(hostHomeDir); err != nil && !os.IsNotExist(err) { + t.Fatalf("failed to delete workspace home: %s", err) + } + if err := os.RemoveAll(hostWorkDir); err != nil && !os.IsNotExist(err) { + t.Fatalf("failed to delete workspace home: %s", err) + } + })(t) + + if err := fs.Mkdir(hostCanaryDir, 0o777); err != nil { + t.Fatalf("failed to create canary_dir: %s", err) + } + if err := fs.Touch(hostCanaryFile); err != nil { + t.Fatalf("failed to create canary_file: %s", err) + } + if err := fs.Touch(hostCanaryFileWithComma); err != nil { + t.Fatalf("failed to create canary_file_comma: %s", err) + } + if err := fs.Touch(hostCanaryFileWithColon); err != nil { + t.Fatalf("failed to create canary_file_colon: %s", err) + } + if err := os.Chmod(hostCanaryFile, 0o777); err != nil { + t.Fatalf("failed to apply permissions on canary_file: %s", err) + } + if err := fs.Mkdir(hostHomeDir, 0o777); err != nil { + t.Fatalf("failed to create workspace home directory: %s", err) + } + if err := fs.Mkdir(hostWorkDir, 0o777); err != nil { + t.Fatalf("failed to create workspace home directory: %s", err) + } +} diff --git a/e2e/actions/oci.go b/e2e/actions/oci.go index 044687ef8f..e6caa65bba 100644 --- a/e2e/actions/oci.go +++ b/e2e/actions/oci.go @@ -2,7 +2,7 @@ // Apptainer a Series of LF Projects LLC. // For website terms of use, trademark policy, privacy policy and other // project policies see https://lfprojects.org/policies -// Copyright (c) 2022, Sylabs Inc. All rights reserved. +// Copyright (c) 2022-2023, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -124,6 +124,7 @@ func (c actionTests) actionOciExec(t *testing.T) { tmpfile.Close() basename := filepath.Base(tmpfile.Name()) + tmpfilePath := filepath.Join("/tmp", basename) homePath := filepath.Join("/home", basename) tests := []struct { @@ -210,6 +211,11 @@ func (c actionTests) actionOciExec(t *testing.T) { e2e.ExpectOutput(e2e.ExactMatch, "whats-in-an-oci-name"), }, }, + { + name: "Workdir", + argv: []string{"--workdir", testdata, imageRef, "test", "-f", tmpfilePath}, + exit: 0, + }, { name: "Pwd", argv: []string{"--pwd", "/etc", imageRef, "pwd"}, @@ -396,6 +402,8 @@ func (c actionTests) actionOciBinds(t *testing.T) { contCanaryFile := "/canary/file" hostCanaryFile := filepath.Join(hostCanaryDir, "file") + hostCanaryFileWithComma := filepath.Join(hostCanaryDir, "file,comma") + hostCanaryFileWithColon := filepath.Join(hostCanaryDir, "file:colon") canaryFileBind := hostCanaryFile + ":" + contCanaryFile canaryFileMount := "type=bind,source=" + hostCanaryFile + ",destination=" + contCanaryFile @@ -403,29 +411,10 @@ func (c actionTests) actionOciBinds(t *testing.T) { canaryDirMount := "type=bind,source=" + hostCanaryDir + ",destination=" + contCanaryDir hostHomeDir := filepath.Join(workspace, "home") + hostWorkDir := filepath.Join(workspace, "workdir") createWorkspaceDirs := func(t *testing.T) { - e2e.Privileged(func(t *testing.T) { - if err := os.RemoveAll(hostCanaryDir); err != nil && !os.IsNotExist(err) { - t.Fatalf("failed to delete canary_dir: %s", err) - } - if err := os.RemoveAll(hostHomeDir); err != nil && !os.IsNotExist(err) { - t.Fatalf("failed to delete workspace home: %s", err) - } - })(t) - - if err := fs.Mkdir(hostCanaryDir, 0o777); err != nil { - t.Fatalf("failed to create canary_dir: %s", err) - } - if err := fs.Touch(hostCanaryFile); err != nil { - t.Fatalf("failed to create canary_file: %s", err) - } - if err := os.Chmod(hostCanaryFile, 0o777); err != nil { - t.Fatalf("failed to apply permissions on canary_file: %s", err) - } - if err := fs.Mkdir(hostHomeDir, 0o777); err != nil { - t.Fatalf("failed to create workspace home directory: %s", err) - } + workspaceDirsGenerator(t, hostCanaryDir, hostHomeDir, hostWorkDir, hostCanaryFile, hostCanaryFileWithComma, hostCanaryFileWithColon) } checkHostFn := func(path string, fn func(string) bool) func(*testing.T) { @@ -436,9 +425,23 @@ func (c actionTests) actionOciBinds(t *testing.T) { if !fn(path) { t.Errorf("%s not found on host", path) } - if err := os.RemoveAll(path); err != nil { - t.Errorf("failed to delete %s: %s", path, err) - } + // This part needs to be in privileged mode because of the following + // case. Suppose X1 on the host is mounted as Y1 in-container; and + // you bind mount X2 on host to Y1/Z/Y2 in-container. This creates a + // situation where Y1/Z needs to be mkdir'd. Apparently, runc/crun + // mkdirs it with the uid and gid of the in-container user, leading + // to a dir whose owner & group on host are not those of the host + // user, but rather a uid and gid that's shifted according to the + // /etc/subuid and /etc/subgid specifications. That means that the + // host user can't then os.RemoveAll() the contents without root + // privileges. + // This scenario is precisely what arises with a test like + // WorkdirTmpBind, below. + e2e.Privileged(func(t *testing.T) { + if err := os.RemoveAll(path); err != nil { + t.Errorf("failed to delete %s: %s", path, err) + } + })(t) } } checkHostFile := func(path string) func(*testing.T) { @@ -552,6 +555,39 @@ func (c actionTests) actionOciBinds(t *testing.T) { postRun: checkHostFile(filepath.Join(hostCanaryDir, "nested")), exit: 0, }, + { + name: "WorkdirTmpBind", + args: []string{ + "--workdir", hostWorkDir, + "--bind", hostCanaryDir + ":/tmp/canary/dir", + imageRef, + "test", "-f", "/tmp/canary/dir/file", + }, + postRun: checkHostDir(filepath.Join(hostWorkDir, "tmp", "canary/dir")), + exit: 0, + }, + { + name: "WorkdirVarTmpBind", + args: []string{ + "--workdir", hostWorkDir, + "--bind", hostCanaryDir + ":/var/tmp/canary/dir", + imageRef, + "test", "-f", "/var/tmp/canary/dir/file", + }, + postRun: checkHostDir(filepath.Join(hostWorkDir, "var_tmp", "canary/dir")), + exit: 0, + }, + { + name: "WorkdirVarTmpBindWritable", + args: []string{ + "--workdir", hostWorkDir, + "--bind", hostCanaryDir + ":/var/tmp/canary/dir", + imageRef, + "test", "-f", "/var/tmp/canary/dir/file", + }, + postRun: checkHostDir(filepath.Join(hostWorkDir, "var_tmp", "canary/dir")), + exit: 0, + }, { name: "IsScratchTmpfs", args: []string{ @@ -584,6 +620,18 @@ func (c actionTests) actionOciBinds(t *testing.T) { }, exit: 0, }, + { + name: "ScratchWorkdirBind", + args: []string{ + "--workdir", hostWorkDir, + "--scratch", "/scratch", + "--bind", hostCanaryDir + ":/scratch/dir", + imageRef, + "test", "-f", "/scratch/dir/file", + }, + postRun: checkHostDir(filepath.Join(hostWorkDir, "scratch/scratch", "dir")), + exit: 0, + }, { name: "CustomHomeOneToOne", args: []string{ diff --git a/internal/pkg/runtime/launcher/oci/launcher_linux.go b/internal/pkg/runtime/launcher/oci/launcher_linux.go index 5ed0c3ff85..5b2ebd23e9 100644 --- a/internal/pkg/runtime/launcher/oci/launcher_linux.go +++ b/internal/pkg/runtime/launcher/oci/launcher_linux.go @@ -85,10 +85,6 @@ func checkOpts(lo launcher.Options) error { if lo.WritableTmpfs { sylog.Infof("--oci mode uses --writable-tmpfs by default") } - if lo.WorkDir != "" { - badOpt = append(badOpt, "WorkDir") - } - if lo.NoHome { badOpt = append(badOpt, "NoHome") } diff --git a/internal/pkg/runtime/launcher/oci/mounts_linux.go b/internal/pkg/runtime/launcher/oci/mounts_linux.go index 93a2932348..3bd0c3d555 100644 --- a/internal/pkg/runtime/launcher/oci/mounts_linux.go +++ b/internal/pkg/runtime/launcher/oci/mounts_linux.go @@ -19,6 +19,7 @@ import ( "strings" "github.com/apptainer/apptainer/internal/pkg/buildcfg" + "github.com/apptainer/apptainer/internal/pkg/util/fs" "github.com/apptainer/apptainer/internal/pkg/util/gpu" "github.com/apptainer/apptainer/internal/pkg/util/user" "github.com/apptainer/apptainer/pkg/sylog" @@ -36,7 +37,9 @@ func (l *Launcher) getMounts() ([]specs.Mount, error) { if err := l.addDevMounts(mounts); err != nil { return nil, fmt.Errorf("while configuring devpts mount: %w", err) } - l.addTmpMounts(mounts) + if err := l.addTmpMounts(mounts); err != nil { + return nil, fmt.Errorf("while configuring tmp mounts: %w", err) + } if err := l.addHomeMount(mounts); err != nil { return nil, fmt.Errorf("while configuring home mount: %w", err) } @@ -61,16 +64,73 @@ func (l *Launcher) getMounts() ([]specs.Mount, error) { } // addTmpMounts adds tmpfs mounts for /tmp and /var/tmp in the container. -func (l *Launcher) addTmpMounts(mounts *[]specs.Mount) { +func (l *Launcher) addTmpMounts(mounts *[]specs.Mount) error { + const ( + tmpDest = "/tmp" + vartmpDest = "/var/tmp" + ) + if !l.apptainerConf.MountTmp { sylog.Debugf("Skipping mount of /tmp due to apptainer.conf") - return + return nil + } + + if len(l.cfg.WorkDir) > 0 { + sylog.Debugf("WorkDir specification provided: %s", l.cfg.WorkDir) + const ( + tmpSrcSubdir = "tmp" + vartmpSrcSubdir = "var_tmp" + ) + + workdir, err := filepath.Abs(filepath.Clean(l.cfg.WorkDir)) + if err != nil { + sylog.Warningf("Can't determine absolute path of workdir %s", l.cfg.WorkDir) + } + + tmpSrc := filepath.Join(workdir, tmpSrcSubdir) + vartmpSrc := filepath.Join(workdir, vartmpSrcSubdir) + + if err := fs.Mkdir(tmpSrc, os.ModeSticky|0o777); err != nil && !os.IsExist(err) { + return fmt.Errorf("failed to create %s: %s", tmpSrc, err) + } + if err := fs.Mkdir(vartmpSrc, os.ModeSticky|0o777); err != nil && !os.IsExist(err) { + return fmt.Errorf("failed to create %s: %s", vartmpSrc, err) + } + + *mounts = append(*mounts, + + specs.Mount{ + Destination: tmpDest, + Type: "none", + Source: tmpSrc, + Options: []string{ + "rbind", + "nosuid", + "relatime", + "mode=777", + }, + }, + specs.Mount{ + Destination: vartmpDest, + Type: "none", + Source: vartmpSrc, + Options: []string{ + "rbind", + "nosuid", + "relatime", + "mode=777", + }, + }, + ) + + return nil } + sylog.Debugf(("No workdir specification provided. Proceeding with tmpfs mounts for /tmp and /var/tmp")) *mounts = append(*mounts, specs.Mount{ - Destination: "/tmp", + Destination: tmpDest, Type: "tmpfs", Source: "tmpfs", Options: []string{ @@ -81,7 +141,7 @@ func (l *Launcher) addTmpMounts(mounts *[]specs.Mount) { }, }, specs.Mount{ - Destination: "/var/tmp", + Destination: vartmpDest, Type: "tmpfs", Source: "tmpfs", Options: []string{ @@ -90,7 +150,10 @@ func (l *Launcher) addTmpMounts(mounts *[]specs.Mount) { "mode=777", fmt.Sprintf("size=%dm", l.apptainerConf.SessiondirMaxSize), }, - }) + }, + ) + + return nil } // addDevMounts adds mounts to assemble a minimal /dev in the container. @@ -268,20 +331,50 @@ func (l *Launcher) addHomeMount(mounts *[]specs.Mount) error { // addScratchMounts adds tmpfs mounts for scratch directories in the container. func (l *Launcher) addScratchMounts(mounts *[]specs.Mount) error { - for _, s := range l.cfg.ScratchDirs { - *mounts = append(*mounts, - specs.Mount{ - Destination: s, - Type: "tmpfs", - Source: "tmpfs", - Options: []string{ - "nosuid", - "relatime", - "nodev", - fmt.Sprintf("size=%dm", l.apptainerConf.SessiondirMaxSize), + const scratchContainerDirName = "/scratch" + + if len(l.cfg.WorkDir) > 0 { + scratchContainerDirPath := filepath.Join(l.cfg.WorkDir, scratchContainerDirName) + if err := fs.Mkdir(scratchContainerDirPath, os.ModeSticky|0o777); err != nil && !os.IsExist(err) { + return fmt.Errorf("failed to create %s: %s", scratchContainerDirPath, err) + } + + for _, s := range l.cfg.ScratchDirs { + scratchDirPath := filepath.Join(scratchContainerDirPath, s) + if err := fs.Mkdir(scratchDirPath, os.ModeSticky|0o777); err != nil && !os.IsExist(err) { + return fmt.Errorf("failed to create %s: %s", scratchDirPath, err) + } + + *mounts = append(*mounts, + specs.Mount{ + Destination: s, + Type: "", + Source: scratchDirPath, + Options: []string{ + "rbind", + "nosuid", + "relatime", + "nodev", + }, }, - }, - ) + ) + } + } else { + for _, s := range l.cfg.ScratchDirs { + *mounts = append(*mounts, + specs.Mount{ + Destination: s, + Type: "tmpfs", + Source: "tmpfs", + Options: []string{ + "nosuid", + "relatime", + "nodev", + fmt.Sprintf("size=%dm", l.apptainerConf.SessiondirMaxSize), + }, + }, + ) + } } return nil From 166b35bb927a6c27cffc54c010f0dfc21e1c67b3 Mon Sep 17 00:00:00 2001 From: Omer Preminger Date: Thu, 18 May 2023 09:52:52 -0400 Subject: [PATCH 103/114] rel. path --workdir/--scratch fix; code style improvements Signed-off-by: Edita Kizinevic --- e2e/actions/actions.go | 2 +- e2e/actions/common.go | 2 +- e2e/actions/oci.go | 20 +++++++------------ .../pkg/runtime/launcher/oci/mounts_linux.go | 6 +++++- 4 files changed, 14 insertions(+), 16 deletions(-) diff --git a/e2e/actions/actions.go b/e2e/actions/actions.go index 122e53dd74..02d9033b1d 100644 --- a/e2e/actions/actions.go +++ b/e2e/actions/actions.go @@ -1132,7 +1132,7 @@ func (c actionTests) actionBinds(t *testing.T) { hostWorkDir := filepath.Join(workspace, "workdir") createWorkspaceDirs := func(t *testing.T) { - workspaceDirsGenerator(t, hostCanaryDir, hostHomeDir, hostWorkDir, hostCanaryFile, hostCanaryFileWithComma, hostCanaryFileWithColon) + mkWorkspaceDirs(t, hostCanaryDir, hostHomeDir, hostWorkDir, hostCanaryFile, hostCanaryFileWithComma, hostCanaryFileWithColon) } // convert test image to sandbox diff --git a/e2e/actions/common.go b/e2e/actions/common.go index 228e6bf7e3..05fddff251 100644 --- a/e2e/actions/common.go +++ b/e2e/actions/common.go @@ -17,7 +17,7 @@ import ( "github.com/apptainer/apptainer/internal/pkg/util/fs" ) -func workspaceDirsGenerator(t *testing.T, hostCanaryDir, hostHomeDir, hostWorkDir, hostCanaryFile, hostCanaryFileWithComma, hostCanaryFileWithColon string) { +func mkWorkspaceDirs(t *testing.T, hostCanaryDir, hostHomeDir, hostWorkDir, hostCanaryFile, hostCanaryFileWithComma, hostCanaryFileWithColon string) { e2e.Privileged(func(t *testing.T) { if err := os.RemoveAll(hostCanaryDir); err != nil && !os.IsNotExist(err) { t.Fatalf("failed to delete canary_dir: %s", err) diff --git a/e2e/actions/oci.go b/e2e/actions/oci.go index e6caa65bba..4d77e026ec 100644 --- a/e2e/actions/oci.go +++ b/e2e/actions/oci.go @@ -414,7 +414,7 @@ func (c actionTests) actionOciBinds(t *testing.T) { hostWorkDir := filepath.Join(workspace, "workdir") createWorkspaceDirs := func(t *testing.T) { - workspaceDirsGenerator(t, hostCanaryDir, hostHomeDir, hostWorkDir, hostCanaryFile, hostCanaryFileWithComma, hostCanaryFileWithColon) + mkWorkspaceDirs(t, hostCanaryDir, hostHomeDir, hostWorkDir, hostCanaryFile, hostCanaryFileWithComma, hostCanaryFileWithColon) } checkHostFn := func(path string, fn func(string) bool) func(*testing.T) { @@ -425,18 +425,12 @@ func (c actionTests) actionOciBinds(t *testing.T) { if !fn(path) { t.Errorf("%s not found on host", path) } - // This part needs to be in privileged mode because of the following - // case. Suppose X1 on the host is mounted as Y1 in-container; and - // you bind mount X2 on host to Y1/Z/Y2 in-container. This creates a - // situation where Y1/Z needs to be mkdir'd. Apparently, runc/crun - // mkdirs it with the uid and gid of the in-container user, leading - // to a dir whose owner & group on host are not those of the host - // user, but rather a uid and gid that's shifted according to the - // /etc/subuid and /etc/subgid specifications. That means that the - // host user can't then os.RemoveAll() the contents without root - // privileges. - // This scenario is precisely what arises with a test like - // WorkdirTmpBind, below. + // When a nested bind is performed under workdir, the bind + // destination will be created (if necessary) by runc/crun inside + // workdir on the host. The bind destination will be created with + // subuid:subgid ownership. This requires privilege, or a userns + + // id mapping, to remove. (Relevant to tests like WorkdirTmpBind, + // below.) e2e.Privileged(func(t *testing.T) { if err := os.RemoveAll(path); err != nil { t.Errorf("failed to delete %s: %s", path, err) diff --git a/internal/pkg/runtime/launcher/oci/mounts_linux.go b/internal/pkg/runtime/launcher/oci/mounts_linux.go index 3bd0c3d555..edcb8da550 100644 --- a/internal/pkg/runtime/launcher/oci/mounts_linux.go +++ b/internal/pkg/runtime/launcher/oci/mounts_linux.go @@ -334,7 +334,11 @@ func (l *Launcher) addScratchMounts(mounts *[]specs.Mount) error { const scratchContainerDirName = "/scratch" if len(l.cfg.WorkDir) > 0 { - scratchContainerDirPath := filepath.Join(l.cfg.WorkDir, scratchContainerDirName) + workdir, err := filepath.Abs(filepath.Clean(l.cfg.WorkDir)) + if err != nil { + sylog.Warningf("Can't determine absolute path of workdir %s", l.cfg.WorkDir) + } + scratchContainerDirPath := filepath.Join(workdir, scratchContainerDirName) if err := fs.Mkdir(scratchContainerDirPath, os.ModeSticky|0o777); err != nil && !os.IsExist(err) { return fmt.Errorf("failed to create %s: %s", scratchContainerDirPath, err) } From 56364e4082f70b1fe657b61e053c86f6e568e4e6 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Tue, 30 May 2023 15:45:28 +0100 Subject: [PATCH 104/114] fix: correct Stdin wire-up in runc/crun execs When runc/crun is called from oci_runc_linux.go, Stdin was incorrectly connected for the various runc/crun operations. * Non-interactive operations such as resume / kill don't need Stdin. * Interactive operations (run/exec) had cmd.Stdin incorrectly set to os.Stdout. This prevented OCI containers from receiving input from pipes, redirection, etc. Fixes sylabs/singularity#1712 Signed-off-by: Edita Kizinevic --- internal/pkg/runtime/launcher/oci/oci_runc_linux.go | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/internal/pkg/runtime/launcher/oci/oci_runc_linux.go b/internal/pkg/runtime/launcher/oci/oci_runc_linux.go index bfa18b41f5..f091904178 100644 --- a/internal/pkg/runtime/launcher/oci/oci_runc_linux.go +++ b/internal/pkg/runtime/launcher/oci/oci_runc_linux.go @@ -98,7 +98,7 @@ func Exec(containerID string, cmdArgs []string, systemdCgroups bool) error { cmd := exec.Command(runtimeBin, runtimeArgs...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr - cmd.Stdin = os.Stdout + cmd.Stdin = os.Stdin sylog.Debugf("Calling %s with args %v", runtimeBin, runtimeArgs) return cmd.Run() } @@ -124,7 +124,6 @@ func Kill(containerID string, killSignal string, systemdCgroups bool) error { cmd := exec.Command(runtimeBin, runtimeArgs...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr - cmd.Stdin = os.Stdout sylog.Debugf("Calling %s with args %v", runtimeBin, runtimeArgs) return cmd.Run() } @@ -151,12 +150,11 @@ func Pause(containerID string, systemdCgroups bool) error { cmd := exec.Command(runtimeBin, runtimeArgs...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr - cmd.Stdin = os.Stdout sylog.Debugf("Calling %s with args %v", runtimeBin, runtimeArgs) return cmd.Run() } -// Resume pauses processes in a container +// Resume un-pauses processes in a container func Resume(containerID string, systemdCgroups bool) error { runtimeBin, err := runtime() if err != nil { @@ -178,7 +176,6 @@ func Resume(containerID string, systemdCgroups bool) error { cmd := exec.Command(runtimeBin, runtimeArgs...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr - cmd.Stdin = os.Stdout sylog.Debugf("Calling %s with args %v", runtimeBin, runtimeArgs) return cmd.Run() } @@ -218,7 +215,7 @@ func Run(ctx context.Context, containerID, bundlePath, pidFile string, systemdCg cmd := exec.Command(runtimeBin, runtimeArgs...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr - cmd.Stdin = os.Stdout + cmd.Stdin = os.Stdin sylog.Debugf("Calling %s with args %v", runtimeBin, runtimeArgs) return cmd.Run() } @@ -306,7 +303,6 @@ func Start(containerID string, systemdCgroups bool) error { cmd := exec.Command(runtimeBin, runtimeArgs...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr - cmd.Stdin = os.Stdout sylog.Debugf("Calling %s with args %v", runtimeBin, runtimeArgs) return cmd.Run() } @@ -333,7 +329,6 @@ func State(containerID string, systemdCgroups bool) error { cmd := exec.Command(runtimeBin, runtimeArgs...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr - cmd.Stdin = os.Stdout sylog.Debugf("Calling %s with args %v", runtimeBin, runtimeArgs) return cmd.Run() } @@ -360,7 +355,6 @@ func Update(containerID, cgFile string, systemdCgroups bool) error { cmd := exec.Command(runtimeBin, runtimeArgs...) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr - cmd.Stdin = os.Stdout sylog.Debugf("Calling %s with args %v", runtimeBin, runtimeArgs) return cmd.Run() } From 7e357b012cb1c32efdcdd37c40c12bdebfaa9aa1 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Tue, 30 May 2023 15:53:39 +0100 Subject: [PATCH 105/114] e2e: port action STDPIPE tests to --oci mode These were missed, resulting in sylabs/singularity#1712 not being caught by the e2e suite. Signed-off-by: Edita Kizinevic --- e2e/actions/actions.go | 1 + e2e/actions/oci.go | 92 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+) diff --git a/e2e/actions/actions.go b/e2e/actions/actions.go index 02d9033b1d..632184d6d3 100644 --- a/e2e/actions/actions.go +++ b/e2e/actions/actions.go @@ -2938,6 +2938,7 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { "ociRun": c.actionOciRun, // apptainer run --oci "ociExec": c.actionOciExec, // apptainer exec --oci "ociShell": c.actionOciShell, // apptainer shell --oci + "ociSTDPIPE": c.ociSTDPipe, // stdin/stdout pipe --oci "ociNetwork": c.actionOciNetwork, // apptainer exec --oci --net "ociBinds": c.actionOciBinds, // apptainer exec --oci --bind / --mount "ociCdi": c.actionOciCdi, // apptainer exec --oci --cdi diff --git a/e2e/actions/oci.go b/e2e/actions/oci.go index 4d77e026ec..2e2439f4d8 100644 --- a/e2e/actions/oci.go +++ b/e2e/actions/oci.go @@ -11,6 +11,7 @@ package actions import ( "bufio" + "bytes" "encoding/json" "fmt" "os" @@ -1167,3 +1168,94 @@ func countLines(path string) (int, error) { return lines, nil } + +// ociSTDPipe tests pipe stdin/stdout to apptainer actions cmd +func (c actionTests) ociSTDPipe(t *testing.T) { + e2e.EnsureOCIArchive(t, c.env) + imageRef := "oci-archive:" + c.env.OCIArchivePath + + stdinTests := []struct { + name string + command string + argv []string + input string + exit int + }{ + { + name: "TrueSTDIN", + command: "exec", + argv: []string{imageRef, "grep", "hi"}, + input: "hi", + exit: 0, + }, + { + name: "FalseSTDIN", + command: "exec", + argv: []string{imageRef, "grep", "hi"}, + input: "bye", + exit: 1, + }, + } + + var input bytes.Buffer + + for _, tt := range stdinTests { + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(e2e.OCIUserProfile), + e2e.WithCommand(tt.command), + e2e.WithArgs(tt.argv...), + e2e.WithStdin(&input), + e2e.PreRun(func(t *testing.T) { + input.WriteString(tt.input) + }), + e2e.ExpectExit(tt.exit), + ) + input.Reset() + } + + user := e2e.CurrentUser(t) + stdoutTests := []struct { + name string + command string + argv []string + output string + exit int + }{ + { + name: "CwdPath", + command: "exec", + argv: []string{"--cwd", "/etc", imageRef, "pwd"}, + output: "/etc", + exit: 0, + }, + { + name: "PwdPath", + command: "exec", + argv: []string{"--pwd", "/etc", imageRef, "pwd"}, + output: "/etc", + exit: 0, + }, + { + name: "id", + command: "exec", + argv: []string{imageRef, "id", "-un"}, + output: user.Name, + exit: 0, + }, + } + for _, tt := range stdoutTests { + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(e2e.OCIUserProfile), + e2e.WithCommand(tt.command), + e2e.WithArgs(tt.argv...), + e2e.ExpectExit( + tt.exit, + e2e.ExpectOutput(e2e.ExactMatch, tt.output), + ), + ) + } +} From 8691b8e4e5ffacfdc59eaf7b3561d09f5f686bad Mon Sep 17 00:00:00 2001 From: preminger Date: Tue, 30 May 2023 04:53:14 -0400 Subject: [PATCH 106/114] oci: support --overlay of bare images (sylabs/singularity#1699) OCI-mode: support mounting of squashfs images (read-only) and extfs images (read-write or read-only) with --overlay, alongside the already existing functionality of directory-based overlays. Squashed merge of: * prepare data structures for non-dir overlays * fix e2e test (no more "auto-create overlay dir" functionality) * switch to using image.Init() to analyze image files * added caching mechanism to FindBin * first working version with squashfs support * refactor funcs into methods of OverlayItem where appropriate * standardized naming + added lots of comments * addressing first round of review comments * refactor: move non-OCI-dep. code to internal/pkg/util/fs/overlay * initial support for extfs overlay (readonly) * resurrect testing of write to persistent overlay * fix bug in writable overlay, introduced in course of refactor * removed caching mechanism from FindBin * addressing second round of review comments, except tests * move calling of prepareWritableOverlay() into Item.Mount() * removed redundant error-string wrapping * overlay.Item unit-test * overlay.Set unit-test * e2e tests for oci image overlays, misc. testing improvements Signed-off-by: Edita Kizinevic --- CHANGELOG.md | 9 +- e2e/actions/oci.go | 181 +++++++++- .../pkg/runtime/launcher/oci/oci_overlay.go | 29 +- internal/pkg/util/bin/bin.go | 4 +- .../pkg/util/fs/overlay/overlay_item_linux.go | 316 ++++++++++++++++++ .../fs/overlay/overlay_item_linux_test.go | 232 +++++++++++++ internal/pkg/util/fs/overlay/overlay_linux.go | 54 +++ .../pkg/util/fs/overlay/overlay_set_linux.go | 137 ++++++++ .../util/fs/overlay/overlay_set_linux_test.go | 176 ++++++++++ .../pkg/util/fs/overlay/testdata/extfs.img | Bin 0 -> 2097152 bytes .../pkg/util/fs/overlay/testdata/squashfs.img | Bin 0 -> 4096 bytes pkg/ocibundle/tools/overlay_linux.go | 293 +++------------- 12 files changed, 1144 insertions(+), 287 deletions(-) create mode 100644 internal/pkg/util/fs/overlay/overlay_item_linux.go create mode 100644 internal/pkg/util/fs/overlay/overlay_item_linux_test.go create mode 100644 internal/pkg/util/fs/overlay/overlay_set_linux.go create mode 100644 internal/pkg/util/fs/overlay/overlay_set_linux_test.go create mode 100644 internal/pkg/util/fs/overlay/testdata/extfs.img create mode 100644 internal/pkg/util/fs/overlay/testdata/squashfs.img diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ac710a64e..5035c9cdb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,10 +70,11 @@ For older changes see the [archived Singularity change log](https://github.com/a `--dns` flag can be used to pass a comma-separated list of DNS servers that will be used in the container; if this flag is not used, the container will use the same `resolv.conf` settings as the host. -- OCI-mode now supports `--overlay ` flag, allowing writes to the - filesystem to persist across runs of the OCI container. If specified dir does - not exist, Apptainer will attempt to create it. Multiple overlays can be - specified, but all but one must be read-only (`--overlay :ro`). +- OCI-mode now supports an `--overlay ` flag. `` can be a writable + directory, in which case changes to the filesystem will persist across runs of + the OCI container. Alternatively, `` can be `:ro` or the path of a + squashfs or extfs image, to be mounted as a read-only overlay. Multiple + overlays can be specified, but all but one must be read-only. - OCI-mode now supports the `--workdir ` option. If this option is specified, `/tmp` and `/var/tmp` will be mapped, respectively, to `/tmp` and `/var_tmp` on the host, rather than to tmpfs diff --git a/e2e/actions/oci.go b/e2e/actions/oci.go index 2e2439f4d8..745862d42a 100644 --- a/e2e/actions/oci.go +++ b/e2e/actions/oci.go @@ -28,6 +28,18 @@ import ( "gotest.tools/v3/assert" ) +const ( + imgTestFilePath string = "file-for-testing" + squashfsTestString string = "squashfs-test-string" + extfsTestString string = "extfs-test-string" +) + +var ( + imgsPath = filepath.Join("..", "internal", "pkg", "util", "fs", "overlay", "testdata") + squashfsImgPath = filepath.Join(imgsPath, "squashfs.img") + extfsImgPath = filepath.Join(imgsPath, "extfs.img") +) + func (c actionTests) actionOciRun(t *testing.T) { e2e.EnsureOCIArchive(t, c.env) e2e.EnsureDockerArchive(t, c.env) @@ -1017,6 +1029,8 @@ func (c actionTests) actionOciCompat(t *testing.T) { } // actionOciOverlay checks that --overlay functions correctly in OCI mode. +// +//nolint:maintidx func (c actionTests) actionOciOverlay(t *testing.T) { e2e.EnsureOCIArchive(t, c.env) imageRef := "oci-archive:" + c.env.OCIArchivePath @@ -1032,6 +1046,20 @@ func (c actionTests) actionOciOverlay(t *testing.T) { } }) + // Create a few read-only overlay subdirs under testDir + for i := 0; i < 3; i++ { + dirName := fmt.Sprintf("my_rw_ol_dir%d", i) + fullPath := filepath.Join(testDir, dirName) + if err = os.Mkdir(fullPath, 0o755); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if !t.Failed() { + os.RemoveAll(fullPath) + } + }) + } + // Create a few read-only overlay subdirs under testDir for i := 0; i < 3; i++ { dirName := fmt.Sprintf("my_ro_ol_dir%d", i) @@ -1059,47 +1087,164 @@ func (c actionTests) actionOciOverlay(t *testing.T) { } tests := []struct { - name string - args []string - exitCode int - wantOutputs []e2e.ApptainerCmdResultOp + name string + args []string + exitCode int + requiredCmds []string + wantOutputs []e2e.ApptainerCmdResultOp }{ { - name: "NewWritable", - args: []string{"--overlay", filepath.Join(testDir, "my_rw_ol_dir"), imageRef, "sh", "-c", "echo my_test_string > /my_test_file"}, + name: "ExistRWDir", + args: []string{"--overlay", filepath.Join(testDir, "my_rw_ol_dir0"), imageRef, "sh", "-c", "echo my_test_string > /my_test_file"}, exitCode: 0, }, { - name: "ExistWritable", - args: []string{"--overlay", filepath.Join(testDir, "my_rw_ol_dir"), imageRef, "cat", "/my_test_file"}, + name: "ExistRWDirRevisit", + args: []string{"--overlay", filepath.Join(testDir, "my_rw_ol_dir0"), imageRef, "cat", "/my_test_file"}, exitCode: 0, wantOutputs: []e2e.ApptainerCmdResultOp{ e2e.ExpectOutput(e2e.ExactMatch, "my_test_string"), }, }, { - name: "NonExistReadonly", - args: []string{"--overlay", filepath.Join(testDir, "my_ro_ol_dir_nonexistent:ro"), imageRef, "echo", "hi"}, + name: "RWOverlayMissing", + args: []string{"--overlay", filepath.Join(testDir, "something_nonexistent"), imageRef, "echo", "hi"}, + exitCode: 255, + }, + { + name: "ROOverlayMissing", + args: []string{"--overlay", filepath.Join(testDir, "something_nonexistent:ro"), imageRef, "echo", "hi"}, exitCode: 255, }, { - name: "ReadonlyAddsTmpfs", + name: "AutoAddTmpfs", args: []string{"--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"), imageRef, "sh", "-c", "echo this_should_disappear > /my_test_file"}, exitCode: 0, }, { - name: "SeveralReadonly", - args: []string{"--overlay", filepath.Join(testDir, "my_ro_ol_dir2:ro"), "--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"), imageRef, "cat", "/testfile.1", "/maskable_testfile"}, + name: "SeveralRODirs", + args: []string{ + "--overlay", filepath.Join(testDir, "my_ro_ol_dir2:ro"), + "--overlay", filepath.Join(testDir, "my_ro_ol_dir0:ro"), + "--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"), + imageRef, "cat", "/testfile.1", "/maskable_testfile", + }, exitCode: 0, wantOutputs: []e2e.ApptainerCmdResultOp{ e2e.ExpectOutput(e2e.ContainMatch, "test_string_1"), e2e.ExpectOutput(e2e.ContainMatch, "maskable_string_2"), }, }, + { + name: "AllTypesAtOnce", + args: []string{ + "--overlay", filepath.Join(testDir, "my_ro_ol_dir2:ro"), + "--overlay", extfsImgPath + ":ro", + "--overlay", squashfsImgPath, + "--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"), + "--overlay", filepath.Join(testDir, "my_rw_ol_dir0"), + imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath), + }, + requiredCmds: []string{"squashfuse", "fuse2fs"}, + exitCode: 0, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ContainMatch, "test_string_1"), + e2e.ExpectOutput(e2e.ContainMatch, "maskable_string_2"), + e2e.ExpectOutput(e2e.ContainMatch, extfsTestString), + }, + }, + { + name: "SquashfsAndDirs", + args: []string{ + "--overlay", filepath.Join(testDir, "my_ro_ol_dir2:ro"), + "--overlay", squashfsImgPath, + "--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"), + "--overlay", filepath.Join(testDir, "my_rw_ol_dir0"), + imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath), + }, + requiredCmds: []string{"squashfuse"}, + exitCode: 0, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ContainMatch, "test_string_1"), + e2e.ExpectOutput(e2e.ContainMatch, "maskable_string_2"), + e2e.ExpectOutput(e2e.ContainMatch, squashfsTestString), + }, + }, + { + name: "ExtfsAndDirs", + args: []string{ + "--overlay", filepath.Join(testDir, "my_ro_ol_dir2:ro"), + "--overlay", extfsImgPath + ":ro", + "--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"), + "--overlay", filepath.Join(testDir, "my_rw_ol_dir0"), + imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath), + }, + requiredCmds: []string{"fuse2fs"}, + exitCode: 0, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ContainMatch, "test_string_1"), + e2e.ExpectOutput(e2e.ContainMatch, "maskable_string_2"), + e2e.ExpectOutput(e2e.ContainMatch, extfsTestString), + }, + }, + { + name: "SquashfsAndDirsAndMissingRO", + args: []string{ + "--overlay", filepath.Join(testDir, "my_ro_ol_dir2:ro"), + "--overlay", squashfsImgPath, + "--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"), + "--overlay", filepath.Join(testDir, "something_nonexistent:ro"), + "--overlay", filepath.Join(testDir, "my_rw_ol_dir0"), + imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath), + }, + requiredCmds: []string{"squashfuse"}, + exitCode: 255, + }, + { + name: "SquashfsAndDirsAndMissingRW", + args: []string{ + "--overlay", filepath.Join(testDir, "my_ro_ol_dir2:ro"), + "--overlay", squashfsImgPath, + "--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"), + "--overlay", filepath.Join(testDir, "something_nonexistent"), + imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath), + }, + requiredCmds: []string{"squashfuse"}, + exitCode: 255, + }, + { + name: "TwoWritables", + args: []string{ + "--overlay", filepath.Join(testDir, "my_ro_ol_dir2:ro"), + "--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"), + "--overlay", filepath.Join(testDir, "my_rw_ol_dir1"), + "--overlay", filepath.Join(testDir, "my_rw_ol_dir0"), + imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath), + }, + requiredCmds: []string{"squashfuse"}, + exitCode: 255, + }, + { + name: "ThreeWritables", + args: []string{ + "--overlay", filepath.Join(testDir, "my_ro_ol_dir2:ro"), + "--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"), + "--overlay", filepath.Join(testDir, "my_rw_ol_dir1"), + "--overlay", filepath.Join(testDir, "my_rw_ol_dir0"), + "--overlay", filepath.Join(testDir, "my_rw_ol_dir2"), + imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath), + }, + requiredCmds: []string{"squashfuse"}, + exitCode: 255, + }, } t.Run(profile.String(), func(t *testing.T) { for _, tt := range tests { + if !haveAllCommands(t, tt.requiredCmds) { + continue + } + c.env.RunApptainer( t, e2e.AsSubtest(tt.name), @@ -1169,6 +1314,16 @@ func countLines(path string) (int, error) { return lines, nil } +func haveAllCommands(t *testing.T, cmds []string) bool { + for _, c := range cmds { + if _, err := exec.LookPath(c); err != nil { + return false + } + } + + return true +} + // ociSTDPipe tests pipe stdin/stdout to apptainer actions cmd func (c actionTests) ociSTDPipe(t *testing.T) { e2e.EnsureOCIArchive(t, c.env) diff --git a/internal/pkg/runtime/launcher/oci/oci_overlay.go b/internal/pkg/runtime/launcher/oci/oci_overlay.go index 15ddcb0dcb..518d83c94d 100644 --- a/internal/pkg/runtime/launcher/oci/oci_overlay.go +++ b/internal/pkg/runtime/launcher/oci/oci_overlay.go @@ -14,6 +14,7 @@ import ( "path/filepath" "strings" + "github.com/apptainer/apptainer/internal/pkg/util/fs/overlay" "github.com/apptainer/apptainer/pkg/ocibundle/tools" "github.com/apptainer/apptainer/pkg/sylog" "github.com/apptainer/apptainer/pkg/util/apptainerconf" @@ -43,30 +44,28 @@ func WrapWithWritableTmpFs(f func() error, bundleDir string) error { // WrapWithOverlays runs a function wrapped with prep / cleanup steps for overlays. func WrapWithOverlays(f func() error, bundleDir string, overlayPaths []string) error { writableOverlayFound := false - ovs := tools.OverlaySet{} + s := overlay.Set{} for _, p := range overlayPaths { - writable := true - splitted := strings.SplitN(p, ":", 2) - barePath := splitted[0] - if len(splitted) > 1 { - if splitted[1] == "ro" { - writable = false - } + item, err := overlay.NewItemFromString(p) + if err != nil { + return err } - if writable && writableOverlayFound { - return fmt.Errorf("you can't specify more than one writable overlay; %#v has already been specified as a writable overlay; use '--overlay %s:ro' instead", ovs.WritableLoc, barePath) + item.SetParentDir(bundleDir) + + if item.Writable && writableOverlayFound { + return fmt.Errorf("you can't specify more than one writable overlay; %#v has already been specified as a writable overlay; use '--overlay %s:ro' instead", s.WritableOverlay, item.SourcePath) } - if writable { + if item.Writable { writableOverlayFound = true - ovs.WritableLoc = barePath + s.WritableOverlay = item } else { - ovs.ReadonlyLocs = append(ovs.ReadonlyLocs, barePath) + s.ReadonlyOverlays = append(s.ReadonlyOverlays, item) } } rootFsDir := tools.RootFs(bundleDir).Path() - err := tools.ApplyOverlay(rootFsDir, ovs) + err := s.Mount(rootFsDir) if err != nil { return err } @@ -78,7 +77,7 @@ func WrapWithOverlays(f func() error, bundleDir string, overlayPaths []string) e } // Cleanup actions log errors, but don't return - so we get as much cleanup done as possible. - if cleanupErr := tools.UnmountOverlay(rootFsDir, ovs); cleanupErr != nil { + if cleanupErr := s.Unmount(rootFsDir); cleanupErr != nil { sylog.Errorf("While unmounting rootfs overlay: %v", cleanupErr) } diff --git a/internal/pkg/util/bin/bin.go b/internal/pkg/util/bin/bin.go index 5b36df5820..8268826534 100644 --- a/internal/pkg/util/bin/bin.go +++ b/internal/pkg/util/bin/bin.go @@ -56,6 +56,7 @@ func FindBin(name string) (path string, err error) { "fakeroot-sysv", "fuse-overlayfs", "fuse2fs", + "fusermount", "go", "ldconfig", "mksquashfs", @@ -74,8 +75,9 @@ func FindBin(name string) (path string, err error) { "zypper", "gocryptfs": return findOnPath(name, false) + default: + return "", fmt.Errorf("unknown executable name %q", name) } - return "", fmt.Errorf("unknown executable name %q", name) } // findOnPath performs a search on the configurated binary path for the diff --git a/internal/pkg/util/fs/overlay/overlay_item_linux.go b/internal/pkg/util/fs/overlay/overlay_item_linux.go new file mode 100644 index 0000000000..766e49004e --- /dev/null +++ b/internal/pkg/util/fs/overlay/overlay_item_linux.go @@ -0,0 +1,316 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2019-2023, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package overlay + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "syscall" + + "github.com/apptainer/apptainer/internal/pkg/util/bin" + "github.com/apptainer/apptainer/pkg/image" + "github.com/apptainer/apptainer/pkg/sylog" +) + +// Item represents information about a single overlay item (as specified, +// for example, in a single --overlay argument) +type Item struct { + // Type represents what type of overlay this is (from among the values in + // pkg/image) + Type int + + // Writable represents whether this is a writable overlay + Writable bool + + // SourcePath is the path of the overlay item stripped of any colon-prefixed + // options (like ":ro") + SourcePath string + + // StagingDir is the directory on which this overlay item is staged, to be + // used as a source for an overlayfs mount as part of an overlay.Set + StagingDir string + + // parentDir is the (optional) location of a secure parent-directory in + // which to create mount directories if needed. If empty, one will be + // created with os.MkdirTemp() + parentDir string +} + +// NewItemFromString takes a string argument, as passed to --overlay, and +// returns an Item struct describing the requested overlay. +func NewItemFromString(overlayString string) (*Item, error) { + item := Item{Writable: true} + + splitted := strings.SplitN(overlayString, ":", 2) + item.SourcePath = splitted[0] + if len(splitted) > 1 { + if splitted[1] == "ro" { + item.Writable = false + } + } + + s, err := os.Stat(item.SourcePath) + if (err != nil) && os.IsNotExist(err) { + return nil, fmt.Errorf("specified overlay %q does not exist", item.SourcePath) + } + if err != nil { + return nil, err + } + + if s.IsDir() { + item.Type = image.SANDBOX + } else if err := item.analyzeImageFile(); err != nil { + return nil, fmt.Errorf("error encountered while examining image file %s: %s", item.SourcePath, err) + } + + return &item, nil +} + +// analyzeImageFile attempts to determine the format of an image file based on +// its header +func (i *Item) analyzeImageFile() error { + img, err := image.Init(i.SourcePath, false) + if err != nil { + return err + } + + switch img.Type { + case image.SQUASHFS: + i.Type = image.SQUASHFS + // squashfs image must be readonly + i.Writable = false + case image.EXT3: + i.Type = image.EXT3 + default: + return fmt.Errorf("image %s is of a type that is not currently supported as overlay", i.SourcePath) + } + + return nil +} + +// SetParentDir sets the parent-dir in which to create overlay-specific mount +// directories. +func (i *Item) SetParentDir(d string) { + i.parentDir = d +} + +// GetParentDir gets a parent-dir in which to create overlay-specific mount +// directories. If one has not been set using SetParentDir(), one will be +// created using os.MkdirTemp(). +func (i *Item) GetParentDir() (string, error) { + // Check if we've already been given a parentDir value; if not, create + // one using os.MkdirTemp() + if len(i.parentDir) > 0 { + return i.parentDir, nil + } + + d, err := os.MkdirTemp("", "overlay-parent-") + if err != nil { + return d, err + } + + i.parentDir = d + return i.parentDir, nil +} + +// Mount performs the necessary steps to mount an individual Item. Note that +// this method does not mount the assembled overlay itself. That happens in +// Set.Mount(). +func (i *Item) Mount() error { + if i.Writable { + if err := i.prepareWritableOverlay(); err != nil { + return err + } + } + + switch i.Type { + case image.SANDBOX: + return i.mountDir() + case image.SQUASHFS: + return i.mountWithFuse("squashfuse") + case image.EXT3: + if i.Writable { + return fmt.Errorf("mounting writable extfs images is not currently supported, please use :ro suffix on image specification for read-only mode") + } + return i.mountWithFuse("fuse2fs", "-o", "ro") + default: + return fmt.Errorf("internal error: unrecognized image type in overlay.Item.Mount() (type: %v)", i.Type) + } +} + +// mountDir mounts directory-based Items. This involves bind-mounting followed +// by remounting of the directory onto itself. This pattern of bind-mount +// followed by remount allows application of more restrictive mount flags than +// are in force on the underlying filesystem. +func (i *Item) mountDir() error { + var err error + if len(i.StagingDir) < 1 { + i.StagingDir = i.SourcePath + } + + if err = EnsureOverlayDir(i.StagingDir, false, 0); err != nil { + return fmt.Errorf("error accessing directory %s: %s", i.StagingDir, err) + } + + sylog.Debugf("Performing identity bind-mount of %q", i.StagingDir) + if err = syscall.Mount(i.StagingDir, i.StagingDir, "", syscall.MS_BIND, ""); err != nil { + return fmt.Errorf("failed to bind %s: %s", i.StagingDir, err) + } + + // Best effort to cleanup mount + defer func() { + if err != nil { + sylog.Debugf("Encountered error with current overlay set; attempting to unmount %q", i.StagingDir) + syscall.Unmount(i.StagingDir, syscall.MNT_DETACH) + } + }() + + // Try to perform remount + sylog.Debugf("Performing remount of %q", i.StagingDir) + if err = syscall.Mount("", i.StagingDir, "", syscall.MS_REMOUNT|syscall.MS_BIND, ""); err != nil { + return fmt.Errorf("failed to remount %s: %s", i.StagingDir, err) + } + + return nil +} + +// mountWithFuse mounts an image to a temporary directory using a specified fuse +// tool. It also verifies that fusermount is present before performing the +// mount. +func (i *Item) mountWithFuse(fuseMountTool string, additionalArgs ...string) error { + var err error + fuseMountCmd, err := bin.FindBin(fuseMountTool) + if err != nil { + return fmt.Errorf("use of image %q as overlay requires %s to be installed: %s", i.SourcePath, fuseMountTool, err) + } + + // Even though fusermount is not needed for this step, we shouldn't perform + // the mount unless we have the necessary tools to eventually unmount it + _, err = bin.FindBin("fusermount") + if err != nil { + return fmt.Errorf("use of image %q as overlay requires fusermount to be installed: %s", i.SourcePath, err) + } + + // Obtain parent directory in which to create overlay-related mount + // directories. See https://github.com/apptainer/singularity/pull/5575 for + // related discussion. + parentDir, err := i.GetParentDir() + if err != nil { + return fmt.Errorf("error while trying to create parent dir for overlay %q: %s", i.SourcePath, err) + } + fuseMountDir, err := os.MkdirTemp(parentDir, "overlay-mountpoint-") + if err != nil { + return fmt.Errorf("failed to create temporary dir %q for overlay %q: %s", fuseMountDir, i.SourcePath, err) + } + + // Best effort to cleanup temporary dir + defer func() { + if err != nil { + sylog.Debugf("Encountered error with current overlay set; attempting to remove %q", fuseMountDir) + os.Remove(fuseMountDir) + } + }() + + args := make([]string, 0, len(additionalArgs)+2) + args = append(args, i.SourcePath) + args = append(args, fuseMountDir) + args = append(args, additionalArgs...) + sylog.Debugf("Executing FUSE mount command: %s %s", fuseMountCmd, strings.Join(args, " ")) + execCmd := exec.Command(fuseMountCmd, args...) + execCmd.Stderr = os.Stderr + _, err = execCmd.Output() + if err != nil { + return fmt.Errorf("encountered error while trying to mount image %q as overlay at %s: %s", i.SourcePath, fuseMountDir, err) + } + i.StagingDir = fuseMountDir + + return nil +} + +// Unmount performs the necessary steps to unmount an individual Item. Note that +// this method does not unmount the overlay itself. That happens in +// Set.Unmount(). +func (i Item) Unmount() error { + switch i.Type { + case image.SANDBOX: + return i.unmountDir() + + case image.SQUASHFS: + fallthrough + case image.EXT3: + return i.unmountFuse() + + default: + return fmt.Errorf("internal error: unrecognized image type in overlay.Item.Unmount() (type: %v)", i.Type) + } +} + +// unmountDir unmounts directory-based Items. +func (i Item) unmountDir() error { + return DetachMount(i.StagingDir) +} + +// unmountFuse unmounts FUSE-based Items. +func (i Item) unmountFuse() error { + defer os.Remove(i.StagingDir) + fusermountCmd, innerErr := bin.FindBin("fusermount") + if innerErr != nil { + // The code in performIndividualMounts() should not have created + // a FUSE-based overlay without fusermount in place + return fmt.Errorf("internal error: FUSE-based mount created without fusermount installed: %s", innerErr) + } + sylog.Debugf("Executing FUSE unmount command: %s -u %s", fusermountCmd, i.StagingDir) + execCmd := exec.Command(fusermountCmd, "-u", i.StagingDir) + execCmd.Stderr = os.Stderr + _, innerErr = execCmd.Output() + if innerErr != nil { + return fmt.Errorf("error while trying to unmount image %q from %s: %s", i.SourcePath, i.StagingDir, innerErr) + } + return nil +} + +// PrepareWritableOverlay ensures that the upper and work subdirs of a writable +// overlay dir exist, and if not, creates them. +func (i *Item) prepareWritableOverlay() error { + switch i.Type { + case image.SANDBOX: + i.StagingDir = i.SourcePath + if err := EnsureOverlayDir(i.StagingDir, true, 0o755); err != nil { + return err + } + sylog.Debugf("Ensuring %q exists", i.Upper()) + if err := EnsureOverlayDir(i.Upper(), true, 0o755); err != nil { + return fmt.Errorf("err encountered while preparing upper subdir of overlay dir %q: %w", i.Upper(), err) + } + sylog.Debugf("Ensuring %q exists", i.Work()) + if err := EnsureOverlayDir(i.Work(), true, 0o700); err != nil { + return fmt.Errorf("err encountered while preparing work subdir of overlay dir %q: %w", i.Work(), err) + } + default: + return fmt.Errorf("unsupported image type in prepareWritableOverlay() (type: %v)", i.Type) + } + + return nil +} + +// Upper returns the "upper"-subdir of the Item's DirToMount field. +// Useful for computing options strings for overlay-related mount system calls. +func (i Item) Upper() string { + return filepath.Join(i.StagingDir, "upper") +} + +// Work returns the "work"-subdir of the Item's DirToMount field. Useful +// for computing options strings for overlay-related mount system calls. +func (i Item) Work() string { + return filepath.Join(i.StagingDir, "work") +} diff --git a/internal/pkg/util/fs/overlay/overlay_item_linux_test.go b/internal/pkg/util/fs/overlay/overlay_item_linux_test.go new file mode 100644 index 0000000000..d11721a3a1 --- /dev/null +++ b/internal/pkg/util/fs/overlay/overlay_item_linux_test.go @@ -0,0 +1,232 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2019-2023, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package overlay + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/apptainer/apptainer/internal/pkg/test/tool/require" + "github.com/apptainer/apptainer/pkg/image" +) + +const ( + testFilePath string = "file-for-testing" + squashfsTestString string = "squashfs-test-string\n" + extfsTestString string = "extfs-test-string\n" +) + +func mkTempDirOrFatal(t *testing.T) string { + tmpDir, err := os.MkdirTemp("", "testoverlayitem-") + if err != nil { + t.Fatalf("failed to create temporary dir: %s", err) + } + t.Cleanup(func() { + if !t.Failed() { + os.RemoveAll(tmpDir) + } + }) + + return tmpDir +} + +func TestItemWritableField(t *testing.T) { + tmpDir := mkTempDirOrFatal(t) + rwOverlayStr := tmpDir + roOverlayStr := tmpDir + ":ro" + + rwItem, err := NewItemFromString(rwOverlayStr) + if err != nil { + t.Fatalf("unexpected error while initializing rwItem from string %q: %s", rwOverlayStr, err) + } + roItem, err := NewItemFromString(roOverlayStr) + if err != nil { + t.Fatalf("unexpected error while initializing roItem from string %q: %s", roOverlayStr, err) + } + + if !rwItem.Writable { + t.Errorf("Writable field of overlay.Item initialized with string %q should be true but is false", rwOverlayStr) + } + + if roItem.Writable { + t.Errorf("Writable field of overlay.Item initialized with string %q should be false but is true", roOverlayStr) + } +} + +func TestItemMissing(t *testing.T) { + const dir string = "/testoverlayitem-this_should_be_missing" + rwOverlayStr := dir + roOverlayStr := dir + ":ro" + + if _, err := NewItemFromString(rwOverlayStr); err == nil { + t.Errorf("unexpected success: initializing overlay.Item with missing file/dir (%q) should have failed", rwOverlayStr) + } + if _, err := NewItemFromString(roOverlayStr); err == nil { + t.Errorf("unexpected success: initializing overlay.Item with missing file/dir (%q) should have failed", roOverlayStr) + } +} + +func verifyAutoParentDir(t *testing.T, item *Item) { + const autoParentDirStr string = "overlay-parent-" + if parentDir, err := item.GetParentDir(); err != nil { + t.Fatalf("unexpected error while calling Item.GetParentDir(): %s", err) + } else if !strings.Contains(parentDir, autoParentDirStr) { + t.Errorf("auto-generated parent dir %q does not contain expected identifier string %q", parentDir, autoParentDirStr) + } else if !strings.HasPrefix(parentDir, "/tmp/") { + t.Errorf("auto-generated parent dir %q is not in expected location", parentDir) + } +} + +func TestAutofillParentDir(t *testing.T) { + tmpDir := mkTempDirOrFatal(t) + rwOverlayStr := tmpDir + roOverlayStr := tmpDir + ":ro" + + rwItem, err := NewItemFromString(rwOverlayStr) + if err != nil { + t.Fatalf("unexpected error while initializing rwItem from string %q: %s", rwOverlayStr, err) + } + roItem, err := NewItemFromString(roOverlayStr) + if err != nil { + t.Fatalf("unexpected error while initializing roItem from string %q: %s", roOverlayStr, err) + } + + verifyAutoParentDir(t, rwItem) + verifyAutoParentDir(t, roItem) +} + +func verifyExplicitParentDir(t *testing.T, item *Item, dir string) { + item.SetParentDir(dir) + if parentDir, err := item.GetParentDir(); err != nil { + t.Fatalf("unexpected error while calling Item.GetParentDir(): %s", err) + } else if parentDir != dir { + t.Errorf("item returned parent dir %q (expected: %q)", parentDir, dir) + } +} + +func TestExplicitParentDir(t *testing.T) { + tmpDir := mkTempDirOrFatal(t) + rwOverlayStr := tmpDir + roOverlayStr := tmpDir + ":ro" + + rwItem, err := NewItemFromString(rwOverlayStr) + if err != nil { + t.Fatalf("unexpected error while initializing rwItem from string %q: %s", rwOverlayStr, err) + } + roItem, err := NewItemFromString(roOverlayStr) + if err != nil { + t.Fatalf("unexpected error while initializing roItem from string %q: %s", roOverlayStr, err) + } + + verifyExplicitParentDir(t, rwItem, "/my-special-directory") + verifyExplicitParentDir(t, roItem, "/my-other-special-directory") +} + +func verifyDirExistsAndWritable(t *testing.T, dir string) { + s, err := os.Stat(dir) + if err != nil { + if os.IsNotExist(err) { + t.Errorf("expected directory %q not found", dir) + } else { + t.Fatalf("unexpected error while looking for directory %q: %s", dir, err) + } + return + } + + if !s.IsDir() { + t.Fatalf("expected %q to be a directory but it is not", dir) + return + } + + file, err := os.CreateTemp(dir, "attempt-to-write-a-file") + if err != nil { + t.Fatalf("could not create a file inside %q, which should have been writable: %s", dir, err) + } + path := file.Name() + file.Close() + if err := os.Remove(path); err != nil { + t.Fatalf("unexpected error while trying to remove temporary file %q: %s", path, err) + } +} + +func TestUpperAndWorkCreation(t *testing.T) { + tmpDir := mkTempDirOrFatal(t) + + item, err := NewItemFromString(tmpDir) + if err != nil { + t.Fatalf("unexpected error while initializing rwItem from string %q: %s", tmpDir, err) + } + + if err := item.prepareWritableOverlay(); err != nil { + t.Fatalf("unexpected error while calling prepareWritableOverlay(): %s", err) + } + + verifyDirExistsAndWritable(t, tmpDir+"/upper") + verifyDirExistsAndWritable(t, tmpDir+"/work") +} + +func dirMountUnmount(t *testing.T, olStr string) { + item, err := NewItemFromString(olStr) + if err != nil { + t.Fatalf("unexpected error while initializing overlay item from string %q: %s", olStr, err) + } + + if err := item.Mount(); err != nil { + t.Fatalf("error encountered while trying to mount dir %q: %s", olStr, err) + } + if err := item.Unmount(); err != nil { + t.Errorf("error encountered while trying to unmount dir %q: %s", olStr, err) + } +} + +func TestDirMounts(t *testing.T) { + dirMountUnmount(t, mkTempDirOrFatal(t)+":ro") + dirMountUnmount(t, mkTempDirOrFatal(t)) +} + +func tryImageRO(t *testing.T, olStr string, typeCode int, typeStr, expectStr string) { + item, err := NewItemFromString(olStr) + if err != nil { + t.Fatalf("failed to mount %s image at %q: %s", typeStr, olStr, err) + } + + if item.Type != typeCode { + t.Errorf("item.Type is %v (should be %v)", item.Type, typeStr) + } + + if err := item.Mount(); err != nil { + t.Fatalf("unable to mount %s image for reading: %s", typeStr, err) + } + t.Cleanup(func() { + item.Unmount() + }) + + testFileStagedPath := filepath.Join(item.StagingDir, testFilePath) + data, err := os.ReadFile(testFileStagedPath) + if err != nil { + t.Fatalf("error while trying to read from file %q: %s", testFileStagedPath, err) + } + foundStr := string(data) + if foundStr != expectStr { + t.Errorf("incorrect file contents in %s img: expected %q, but found: %q", typeStr, expectStr, foundStr) + } +} + +func TestSquashfsRO(t *testing.T) { + require.Command(t, "squashfuse") + tryImageRO(t, filepath.Join(".", "testdata", "squashfs.img"), image.SQUASHFS, "squashfs", squashfsTestString) +} + +func TestExtfsRO(t *testing.T) { + require.Command(t, "fuse2fs") + tryImageRO(t, filepath.Join(".", "testdata", "extfs.img")+":ro", image.EXT3, "extfs", extfsTestString) +} diff --git a/internal/pkg/util/fs/overlay/overlay_linux.go b/internal/pkg/util/fs/overlay/overlay_linux.go index d30c9ffcc4..221ad8c87c 100644 --- a/internal/pkg/util/fs/overlay/overlay_linux.go +++ b/internal/pkg/util/fs/overlay/overlay_linux.go @@ -190,3 +190,57 @@ func CheckRootless() error { sylog.Debugf("Rootless overlay appears supported on this system.") return nil } + +// ensureOverlayDir checks if a directory already exists; if it doesn't, and +// createIfMissing is true, it attempts to create it with the specified +// permissions. +func EnsureOverlayDir(dir string, createIfMissing bool, createPerm os.FileMode) error { + if len(dir) == 0 { + return fmt.Errorf("internal error: ensureOverlayDir() called with empty dir name") + } + + _, err := os.Stat(dir) + if err == nil { + return nil + } + + if !os.IsNotExist(err) { + return err + } + + if !createIfMissing { + return fmt.Errorf("missing overlay dir %q", dir) + } + + // Create the requested dir + if err := os.Mkdir(dir, createPerm); err != nil { + return fmt.Errorf("failed to create %q: %s", dir, err) + } + + return nil +} + +// detachAndDelete performs an unmount system call on the specified directory, +// followed by deletion of the directory and all of its contents. +func DetachAndDelete(overlayDir string) error { + sylog.Debugf("Detaching overlayDir %q", overlayDir) + if err := syscall.Unmount(overlayDir, syscall.MNT_DETACH); err != nil { + return fmt.Errorf("failed to unmount %s: %s", overlayDir, err) + } + + sylog.Debugf("Removing overlayDir %q", overlayDir) + if err := os.RemoveAll(overlayDir); err != nil { + return fmt.Errorf("failed to remove %s: %s", overlayDir, err) + } + return nil +} + +// detachMount performs an unmount system call on the specified directory. +func DetachMount(dir string) error { + sylog.Debugf("Calling syscall.Unmount() to detach %q", dir) + if err := syscall.Unmount(dir, syscall.MNT_DETACH); err != nil { + return fmt.Errorf("failed to detach %s: %s", dir, err) + } + + return nil +} diff --git a/internal/pkg/util/fs/overlay/overlay_set_linux.go b/internal/pkg/util/fs/overlay/overlay_set_linux.go new file mode 100644 index 0000000000..160c17d688 --- /dev/null +++ b/internal/pkg/util/fs/overlay/overlay_set_linux.go @@ -0,0 +1,137 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2019-2023, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package overlay + +import ( + "fmt" + "strings" + "syscall" + + "github.com/apptainer/apptainer/pkg/sylog" + "github.com/samber/lo" +) + +// Set represents a set of overlay directories which will be overlain on top of +// some filesystem mount point. The actual mount point atop which these +// directories will be overlain is not specified in the Set; it is left +// implicit, to be chosen by whichever function consumes a Set. A Set contains +// two types of directories: zero or more directories which will be mounted as +// read-only overlays atop the (implicit) mount point, and one directory which +// will be mounted as a writable overlay atop all the rest. An empty WritableLoc +// field indicates that no writable overlay is to be mounted. +type Set struct { + // ReadonlyOverlays is a list of directories to be mounted as read-only + // overlays. The mount point atop which these will be mounted is left + // implicit, to be chosen by whichever function consumes the Set. + ReadonlyOverlays []*Item + + // WritableOverlay is the directory to be mounted as a writable overlay. The + // mount point atop which this will be mounted is left implicit, to be + // chosen by whichever function consumes the Set. Empty value indicates no + // writable overlay is to be mounted. + WritableOverlay *Item +} + +// Mount prepares and mounts the entire Set onto the specified rootfs +// directory. +func (s Set) Mount(rootFsDir string) error { + // Perform identity mounts for this Set + if err := s.performIndividualMounts(); err != nil { + return err + } + + // Perform actual overlay mount + return s.performFinalMount(rootFsDir) +} + +// UnmountOverlay ummounts a Set from a specified rootfs directory. +func (s Set) Unmount(rootFsDir string) error { + if err := DetachMount(rootFsDir); err != nil { + return err + } + + return s.detachIndividualMounts() +} + +// performIndividualMounts creates the mounts that furnish the individual +// elements of the Set. +func (s Set) performIndividualMounts() error { + overlaysToBind := s.ReadonlyOverlays + if s.WritableOverlay != nil { + overlaysToBind = append(overlaysToBind, s.WritableOverlay) + } + + // Try to do initial bind-mounts + for _, o := range overlaysToBind { + if err := o.Mount(); err != nil { + return err + } + } + + return nil +} + +// performFinalMount performs the final step in mounting a Set, namely mounting +// of the overlay with its full-fledged options string, representing all the +// individual Items (writable and read-only) that comprise the Set. +func (s Set) performFinalMount(rootFsDir string) error { + // Try to perform actual mount + options := s.options(rootFsDir) + sylog.Debugf("Mounting overlay with rootFsDir %q, options: %q", rootFsDir, options) + if err := syscall.Mount("overlay", rootFsDir, "overlay", syscall.MS_NODEV, options); err != nil { + return fmt.Errorf("failed to mount %s: %s", rootFsDir, err) + } + + return nil +} + +// options creates an options string to be used in an overlay mount, +// representing all the individual Items (writable and read-only) that comprise +// the Set. +func (s Set) options(rootFsDir string) string { + // Create lowerdir argument of options string + lowerDirs := lo.Map(s.ReadonlyOverlays, func(o *Item, _ int) string { + return o.StagingDir + }) + lowerDirJoined := strings.Join(append(lowerDirs, rootFsDir), ":") + + if s.WritableOverlay == nil { + return fmt.Sprintf("lowerdir=%s", lowerDirJoined) + } + + return fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", + lowerDirJoined, s.WritableOverlay.Upper(), s.WritableOverlay.Work()) +} + +// detachIndividualMounts detaches the bind mounts & remounts created by +// performIndividualMounts, above. +func (s Set) detachIndividualMounts() error { + overlaysToDetach := s.ReadonlyOverlays + if s.WritableOverlay != nil { + overlaysToDetach = append(overlaysToDetach, s.WritableOverlay) + } + + // Don't stop on the first error; try to clean up as much as possible, and + // then return the first error encountered. + errors := []error{} + for _, overlay := range overlaysToDetach { + err := overlay.Unmount() + if err != nil { + sylog.Errorf("Error encountered trying to detach identity mount %s: %s", overlay.StagingDir, err) + errors = append(errors, err) + } + } + + if len(errors) > 0 { + return errors[0] + } + + return nil +} diff --git a/internal/pkg/util/fs/overlay/overlay_set_linux_test.go b/internal/pkg/util/fs/overlay/overlay_set_linux_test.go new file mode 100644 index 0000000000..0a9e6f940f --- /dev/null +++ b/internal/pkg/util/fs/overlay/overlay_set_linux_test.go @@ -0,0 +1,176 @@ +// Copyright (c) Contributors to the Apptainer project, established as +// Apptainer a Series of LF Projects LLC. +// For website terms of use, trademark policy, privacy policy and other +// project policies see https://lfprojects.org/policies +// Copyright (c) 2023, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package overlay + +import ( + "os" + "os/exec" + "path/filepath" + "testing" +) + +func addROItemOrFatal(t *testing.T, s *Set, olStr string) *Item { + i, err := NewItemFromString(olStr) + if err != nil { + t.Fatalf("could not initialize overlay item from string %q: %s", olStr, err) + } + s.ReadonlyOverlays = append(s.ReadonlyOverlays, i) + + return i +} + +func TestAllTypesAtOnce(t *testing.T) { + s := Set{} + + tmpRODir := mkTempDirOrFatal(t) + addROItemOrFatal(t, &s, tmpRODir+":ro") + + squashfsSupported := false + if _, err := exec.LookPath("squashfs"); err == nil { + squashfsSupported = true + addROItemOrFatal(t, &s, filepath.Join(".", "testdata", "squashfs.img")) + } + + extfsSupported := false + if _, err := exec.LookPath("fuse2fs"); err == nil { + extfsSupported = true + addROItemOrFatal(t, &s, filepath.Join(".", "testdata", "extfs.img")+":ro") + } + + tmpRWDir := mkTempDirOrFatal(t) + i, err := NewItemFromString(tmpRWDir) + if err != nil { + t.Fatalf("failed to create writable-dir overlay item (%q): %s", tmpRWDir, err) + } + s.WritableOverlay = i + + rootfsDir := mkTempDirOrFatal(t) + if err := s.Mount(rootfsDir); err != nil { + t.Fatalf("failed to mount overlay set: %s", err) + } + t.Cleanup(func() { + s.Unmount(rootfsDir) + }) + + var expectStr string + if extfsSupported { + expectStr = extfsTestString + } else if squashfsSupported { + expectStr = squashfsTestString + } + + if squashfsSupported || extfsSupported { + testFileMountedPath := filepath.Join(rootfsDir, testFilePath) + data, err := os.ReadFile(testFileMountedPath) + if err != nil { + t.Fatalf("error while trying to read from file %q: %s", testFileMountedPath, err) + } + foundStr := string(data) + if foundStr != expectStr { + t.Errorf("incorrect file contents in mounted overlay set: expected %q, but found: %q", expectStr, foundStr) + } + } + + if err := s.Unmount(rootfsDir); err != nil { + t.Errorf("error encountered while trying to unmount overlay set: %s", err) + } +} + +func TestPersistentWriteToDir(t *testing.T) { + tmpRWDir := mkTempDirOrFatal(t) + i, err := NewItemFromString(tmpRWDir) + if err != nil { + t.Fatalf("failed to create writable-dir overlay item (%q): %s", tmpRWDir, err) + } + s := Set{WritableOverlay: i} + + rootfsDir := mkTempDirOrFatal(t) + + // This cleanup will serve adequately for both iterations of the overlay-set + // mounting, below. If it happens to get called while the set is not + // mounted, it should fail silently. + t.Cleanup(func() { + s.Unmount(rootfsDir) + }) + + // Mount the overlay set, write a string to a file, and unmount. + if err := s.Mount(rootfsDir); err != nil { + t.Fatalf("failed to mount overlay set: %s", err) + } + expectStr := "my_test_string" + bytes := []byte(expectStr) + testFilePath := "my_test_file" + testFileMountedPath := filepath.Join(rootfsDir, testFilePath) + if err := os.WriteFile(testFileMountedPath, bytes, 0o644); err != nil { + t.Fatalf("error encountered while trying to write file inside mounted overlay-set: %s", err) + } + + if err := s.Unmount(rootfsDir); err != nil { + t.Fatalf("error encountered while trying to unmount overlay set: %s", err) + } + + // Mount the same set again, and check that we see the file with the + // expected contents. + if err := s.Mount(rootfsDir); err != nil { + t.Fatalf("failed to mount overlay set: %s", err) + } + data, err := os.ReadFile(testFileMountedPath) + if err != nil { + t.Fatalf("error while trying to read from file %q: %s", testFileMountedPath, err) + } + foundStr := string(data) + if foundStr != expectStr { + t.Errorf("incorrect file contents in mounted overlay set: expected %q, but found: %q", expectStr, foundStr) + } + if err := s.Unmount(rootfsDir); err != nil { + t.Errorf("error encountered while trying to unmount overlay set: %s", err) + } +} + +func TestDuplicateItemsInSet(t *testing.T) { + var rootfsDir string + var rwI *Item + var err error + + s := Set{} + + // First, test mounting of an overlay set with only readonly items, one of + // which is a duplicate of another. + addROItemOrFatal(t, &s, mkTempDirOrFatal(t)+":ro") + roI2 := addROItemOrFatal(t, &s, mkTempDirOrFatal(t)+":ro") + addROItemOrFatal(t, &s, mkTempDirOrFatal(t)+":ro") + addROItemOrFatal(t, &s, roI2.SourcePath+":ro") + addROItemOrFatal(t, &s, mkTempDirOrFatal(t)+":ro") + + rootfsDir = mkTempDirOrFatal(t) + if err := s.Mount(rootfsDir); err == nil { + t.Errorf("unexpected success: Mounting overlay.Set with duplicate (%q) should have failed", roI2.SourcePath) + if err := s.Unmount(rootfsDir); err != nil { + t.Fatalf("could not unmount erroneous successful mount of overlay set: %s", err) + } + } + + // Next, test mounting of an overlay set with a writable item as well as + // several readonly items, one of which is a duplicate of another. + tmpRWDir := mkTempDirOrFatal(t) + rwI, err = NewItemFromString(tmpRWDir) + if err != nil { + t.Fatalf("failed to create writable-dir overlay item (%q): %s", tmpRWDir, err) + } + s.WritableOverlay = rwI + + rootfsDir = mkTempDirOrFatal(t) + if err := s.Mount(rootfsDir); err == nil { + t.Errorf("unexpected success: Mounting overlay.Set with duplicate file/dir (%q) should have failed", roI2.SourcePath) + if err := s.Unmount(rootfsDir); err != nil { + t.Fatalf("could not unmount erroneous successful mount of overlay set: %s", err) + } + } +} diff --git a/internal/pkg/util/fs/overlay/testdata/extfs.img b/internal/pkg/util/fs/overlay/testdata/extfs.img new file mode 100644 index 0000000000000000000000000000000000000000..7edacbe7a7faed0f4ac00e70c32427528abe3731 GIT binary patch literal 2097152 zcmeI*dDLxl!NBq74rWDCDaoZmbd8az3?VYlGZmMV`Cb_!Q==)G2Z;=&%tI+MWXMzq zks)MOGL!lJK6KT(wcgcxTc!8$*=v2Dz4tkLKWG0w=lPv`{y1mdfdLBv0P}bJ21=AXJ7uz3M-60^v**rn&Ot*m+hv1!!p_5{mzE=(z4=(M?A9pJWFqQ@)`Rc zzVvgWFC5l2PW_jA2gk8N+Z)zKwQ+4yo7M;GL-pa>tOf@>rUuQn96#u`2L`5n;pg}t zqXtI3F#hrTLwUW@$e}#Cp*yxI8`|J}9h?V;k9hPI^Y=P*pHU+tK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB=D4qQLY2JmO&Fv4z4z=s`vQXv zKE&X446ZE%2oU&p3+%Obk1+4Zk>g+c|N8nfY=Uny{NJx+{eR+=Q4<{V*T_(1{`K#M{Nn+Pn&9UD^!O)uc)PDKfpZ7n|IGX1`;PH*;6;Mr12g=M z#ma53QmfW#wR){lYt~w|cCAzE)_V2NE~6$m@IU?ce^jr3({&Bw-}?=&U%aGYFTMX8 z{0`fu|HAj1j$L)@`z9FXOS_}mI(X4V78vY~r)OVa@bL%NFCAcm009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!Cu@ zC@{Roo_b)|@VQ&RxIKLM-0e)&{`3P=4&QpavE%02V!NHT-F)!4fq`pAOgn6gt+pOB zdW-FL96fH#*l}BJyJa_92oU&xC@?tB|91XAI^xCo|Nn3(E|~yA}}Nb%>Np9y|A)lWXb=P@O2GV|C^jb%7!m^J|B!eZ4T8W#37G#A#pWaeLqT9fgBW}X zn4~7H$!hYNqAUam3|)b97hB}(jqWe+00vhtYz}K#sya+E^`h4wI`_}<=U>#InsDtZ}`eJ>lzFc3auh!RU z@CyvzsBhM{>d^Xj9ai6|!|RCpZXH=i)zNiKeXqV>$JP((hjm;XUnkUwbyA&Nr_`x+ zTAf}$svp-G^^^K(ompqq&+6gRQST~HU+MfHoixGt$n>$1AMep$b&E9%O+ zs(xKp*EMx*{id#~>+835L)}<6)y?(0x}|Qd+v@jqd)-lY)?IaX-Bb70ef5X>W8GhW zst4-9dZ-?*N9xgftRAl?>d*CLJylQFGxcmeSI^hrH$;Zj@ETDgYm%C@CacM7ikh;f zs;TQ$_3E0Yrmff1boJVrzFt?auQ${T^~QQr%~)@)nd&VybB(H5YSwydy{+C}v(@Z1 zN4=xwthwr)HM-`md1~I8uja1>>Rq*9EmZHWg=>*ov=*zy>pk_}TB4S$rE2MVUoBJ1 z)^hd!TE14O57dgaQmtI8)T*^wtzK)?nzdG~UF+1kwO*}X8`OriQEgnC)TZ^p`cQqi zHml8ROl?sX0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfWU+T9~m|ofJ)`dod!_N)EtfI6@asxQ>Rbx3`& zzEoeXuhduTYxVW|Mt!rsRfpEM>#+Jx9bQM&ck9SHs*bK>>U;J5I<|gLKdj^G_&T9Z ztdr{GI;Bpn)9Up4QT@2isGrnN>&!Zw>zlE~;PD#dS$tT9?)3 z^~?HIT~SxoRrTw-x~{2f>o;{>U0=Vg8|ucoscx>{)h%^v-B!P^+v|?Hv+k<9>z=x| z?yEo4AM5`5Q$0`*)WA%7FQGc!{>#2IWo~dW+xq7|^`gdhm4X+V3vL>lX zYqFZWrl=`vs+zi9Rj;mTYT9~DO;@k2>Fag%`g%jnP;aa^)r|G#nyKDWGuNn^rDmrFN~| z>Jznl?NOhsJ!`MpyY{Jl>r?gV`b>ScK3AWw{c8U@pbo5q>I-#n9a3McFV&apEA`d- zT7A8~QQxd@)uHw6I;_4^hu0DH-8!<4s-x?e`d)p%j;$Zm59_!(zD}qU>!dolPN`Gt zv^u?hR6njW>L>NnIlSJaht zRsFiIu50Sr`b}L|*Vk|BhPttCs+;R~bxYk^x7F|K_PV3)th?&&x~J}~`|1z%$GX4% zR1egH^-w)rkJO{}SUp}()Sv6gda9nTXX@E{uAZ-f5d#CmYIu#Pku^z8T9eh}HAPKX zQ`OY5@}y|0$3 zWox;5e=T1t)CX$CTB%m9Rch5*tyZr!YRy`!)~ofJ)`dod!_N)EtfI6@asxQ>Rbx3`&zEoeXuhduTYxVW|Mt!rs zRfpEM>#+Jx9bQM&ck9SHs*bK>>U;J5I<|gLKdj^G_&T9Ztdr{GI;Bpn)9Up4QT@2i zsGrnN>&!Zw>zlE~;PD#dS$tT9?)3^~?HIT~SxoRrTw-x~{2f z>o;{>U0=Vg8|ucoscx>{)h%^v-B!P^+v|?Hv+k<9>z=x|?yEo4AM5`5Q$0`*)WA%7FQGc!{>#2IWo~dW+xq7|^Mh*-NtKl`GM%E-XX-!s>*Az8nO;uCZtLoJ? zO-);`sp;yqHGRFVUSDsh8S0Jorkb(dTr<^MYUUbMv(&8h)_PmLy=JS~YmRzH%~^BR zJ8N{!UGvnuHDAqN3)H)6!CI)^T?^MDwP-C?i`RSVy|qLwSxeQ@^}brBmaXOL{k43p zP#>rjYo%JbR;g8MwOYN_s5NV?TD#V%b!)v^zc#21Yopq@HYp1M0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U zAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C7 z2oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N z0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ z009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBly zK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF z5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs z0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAVA>1U0}>ExGcLHT@8-P{_y42%~TET2HdSsGq2Gcf#t@{Ajp7}yz# zQ*$!&N-I==f~9%M`FUxXd8sM!rB?Y#S*gh-@kLgp#i>Q{CAkIhC04}-3<3>M-C!F{ zfgCPqz(6Smdl& 0 { - if err := ensureOverlayDir(ovs.WritableLoc, true, 0o755); err != nil { - return err - } - if err := prepareWritableOverlay(ovs.WritableLoc); err != nil { - return err - } - } - - // Perform identity mounts for this OverlaySet - if err := performIdentityMounts(ovs); err != nil { + if err := overlay.DetachMount(olDir); err != nil { return err } - // Perform actual overlay mount - return performOverlayMount(rootFsDir, overlayOptions(rootFsDir, ovs)) -} - -// UnmountOverlay umounts an overlay -func UnmountOverlay(rootFsDir string, ovs OverlaySet) error { - if err := detachMount(rootFsDir); err != nil { - return err - } - - return detachIdentityMounts(ovs) -} - -// prepareWritableOverlay ensures that the upper and work subdirs of a writable -// overlay dir exist, and if not, creates them. -func prepareWritableOverlay(dir string) error { - sylog.Debugf("Ensuring %q exists", upperSubdirOf(dir)) - if err := ensureOverlayDir(upperSubdirOf(dir), true, 0o755); err != nil { - return fmt.Errorf("err encountered while preparing upper subdir of overlay dir %q: %w", upperSubdirOf(dir), err) - } - sylog.Debugf("Ensuring %q exists", workSubdirOf(dir)) - if err := ensureOverlayDir(workSubdirOf(dir), true, 0o700); err != nil { - return fmt.Errorf("err encountered while preparing work subdir of overlay dir %q: %w", workSubdirOf(dir), err) - } - - return nil -} - -// performIdentityMounts creates the writable OverlaySet directory if it does -// not exist, and performs a bind mount & remount of every OverlaySet dir onto -// itself. The pattern of bind mount followed by remount allows application of -// more restrictive mount flags than are in force on the underlying filesystem. -func performIdentityMounts(ovs OverlaySet) error { - var err error - - locsToBind := ovs.ReadonlyLocs - if len(ovs.WritableLoc) > 0 { - // Check if writable overlay dir already exists; if it doesn't, try to - // create it. - if err = ensureOverlayDir(ovs.WritableLoc, true, 0o755); err != nil { - return err - } - - locsToBind = append(locsToBind, ovs.WritableLoc) - } - - // Try to do initial bind-mounts - for _, d := range locsToBind { - if err = ensureOverlayDir(d, false, 0); err != nil { - return fmt.Errorf("error accessing directory %s: %s", d, err) - } - - sylog.Debugf("Performing identity bind-mount of %q", d) - if err = syscall.Mount(d, d, "", syscall.MS_BIND, ""); err != nil { - return fmt.Errorf("failed to bind %s: %s", d, err) - } - - // best effort to cleanup mount - defer func() { - if err != nil { - sylog.Debugf("Encountered error with current OverlaySet; attempting to unmount %q", d) - syscall.Unmount(d, syscall.MNT_DETACH) - } - }() - - // Try to perform remount - sylog.Debugf("Performing remount of %q", d) - if err = syscall.Mount("", d, "", syscall.MS_REMOUNT|syscall.MS_BIND, ""); err != nil { - return fmt.Errorf("failed to remount %s: %s", d, err) - } - } - - return err -} - -// detachIdentityMounts detaches mounts created by the bind-mount & remount -// pattern (as implemented in performIdentityMounts()) -func detachIdentityMounts(ovs OverlaySet) error { - locsToDetach := ovs.ReadonlyLocs - if len(ovs.WritableLoc) > 0 { - locsToDetach = append(locsToDetach, ovs.WritableLoc) - } - - // Don't stop on the first error; try to clean up as much as possible, and - // then return the first error encountered. - errors := []error{} - for _, d := range locsToDetach { - err := detachMount(d) - if err != nil { - sylog.Errorf("Error encountered trying to detach identity mount %s: %s", d, err) - errors = append(errors, err) - } - } - - if len(errors) > 0 { - return errors[0] - } - - return nil -} - -// overlayOptions creates the options string to be used in an overlay mount -func overlayOptions(rootFsDir string, ovs OverlaySet) string { - // Create lowerdir argument of options string - lowerDirJoined := strings.Join(append(ovs.ReadonlyLocs, rootFsDir), ":") - - if len(ovs.WritableLoc) > 0 { - return fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", lowerDirJoined, upperSubdirOf(ovs.WritableLoc), workSubdirOf(ovs.WritableLoc)) - } - - return fmt.Sprintf("lowerdir=%s", lowerDirJoined) -} - -// performOverlayMount mounts an overlay atop a given rootfs directory -func performOverlayMount(rootFsDir, options string) error { - // Try to perform actual mount - sylog.Debugf("Mounting overlay with rootFsDir %q, options: %q", rootFsDir, options) - if err := syscall.Mount("overlay", rootFsDir, "overlay", 0, options); err != nil { - return fmt.Errorf("failed to mount %s: %s", rootFsDir, err) - } - - return nil -} - -// ensureOverlayDir checks if a directory already exists; if it doesn't, and -// createIfMissing is true, it attempts to create it with the specified -// permissions. -func ensureOverlayDir(dir string, createIfMissing bool, createPerm os.FileMode) error { - if len(dir) == 0 { - return fmt.Errorf("internal error: ensureOverlayDir() called with empty dir name") - } - - _, err := os.Stat(dir) - if err == nil { - return nil - } - - if !os.IsNotExist(err) { - return err - } - - if !createIfMissing { - return fmt.Errorf("missing overlay dir %q", dir) - } - - // Create the requested dir - if err := os.Mkdir(dir, createPerm); err != nil { - return fmt.Errorf("failed to create %q: %s", dir, err) - } - - return nil -} - -func upperSubdirOf(overlayDir string) string { - return filepath.Join(overlayDir, "upper") -} - -func workSubdirOf(overlayDir string) string { - return filepath.Join(overlayDir, "work") -} - -func detachAndDelete(overlayDir string) error { - sylog.Debugf("Detaching overlayDir %q", overlayDir) - if err := syscall.Unmount(overlayDir, syscall.MNT_DETACH); err != nil { - return fmt.Errorf("failed to unmount %s: %s", overlayDir, err) - } - - sylog.Debugf("Removing overlayDir %q", overlayDir) - if err := os.RemoveAll(overlayDir); err != nil { - return fmt.Errorf("failed to remove %s: %s", overlayDir, err) - } - return nil -} - -func detachMount(dir string) error { - sylog.Debugf("Calling syscall.Unmount() to detach %q", dir) - if err := syscall.Unmount(dir, syscall.MNT_DETACH); err != nil { - return fmt.Errorf("failed to detach %s: %s", dir, err) - } - - return nil + return overlay.DetachAndDelete(olDir) } From 16972cb600baede594b5ede271237d4f93469ec3 Mon Sep 17 00:00:00 2001 From: Omer Preminger Date: Tue, 30 May 2023 10:20:34 -0400 Subject: [PATCH 107/114] move overlay test images to test/ subtree Signed-off-by: Edita Kizinevic --- e2e/actions/oci.go | 6 +++--- .../pkg/util/fs/overlay/overlay_item_linux_test.go | 4 ++-- .../pkg/util/fs/overlay/overlay_set_linux_test.go | 4 ++-- .../extfs.img => test/images/extfs-for-overlay.img | Bin .../images/squashfs-for-overlay.img | Bin 5 files changed, 7 insertions(+), 7 deletions(-) rename internal/pkg/util/fs/overlay/testdata/extfs.img => test/images/extfs-for-overlay.img (100%) rename internal/pkg/util/fs/overlay/testdata/squashfs.img => test/images/squashfs-for-overlay.img (100%) diff --git a/e2e/actions/oci.go b/e2e/actions/oci.go index 745862d42a..497ab73b2d 100644 --- a/e2e/actions/oci.go +++ b/e2e/actions/oci.go @@ -35,9 +35,9 @@ const ( ) var ( - imgsPath = filepath.Join("..", "internal", "pkg", "util", "fs", "overlay", "testdata") - squashfsImgPath = filepath.Join(imgsPath, "squashfs.img") - extfsImgPath = filepath.Join(imgsPath, "extfs.img") + imgsPath = filepath.Join("..", "test", "images") + squashfsImgPath = filepath.Join(imgsPath, "squashfs-for-overlay.img") + extfsImgPath = filepath.Join(imgsPath, "extfs-for-overlay.img") ) func (c actionTests) actionOciRun(t *testing.T) { diff --git a/internal/pkg/util/fs/overlay/overlay_item_linux_test.go b/internal/pkg/util/fs/overlay/overlay_item_linux_test.go index d11721a3a1..bd1138875c 100644 --- a/internal/pkg/util/fs/overlay/overlay_item_linux_test.go +++ b/internal/pkg/util/fs/overlay/overlay_item_linux_test.go @@ -223,10 +223,10 @@ func tryImageRO(t *testing.T, olStr string, typeCode int, typeStr, expectStr str func TestSquashfsRO(t *testing.T) { require.Command(t, "squashfuse") - tryImageRO(t, filepath.Join(".", "testdata", "squashfs.img"), image.SQUASHFS, "squashfs", squashfsTestString) + tryImageRO(t, filepath.Join("..", "..", "..", "..", "..", "test", "images", "squashfs-for-overlay.img"), image.SQUASHFS, "squashfs", squashfsTestString) } func TestExtfsRO(t *testing.T) { require.Command(t, "fuse2fs") - tryImageRO(t, filepath.Join(".", "testdata", "extfs.img")+":ro", image.EXT3, "extfs", extfsTestString) + tryImageRO(t, filepath.Join("..", "..", "..", "..", "..", "test", "images", "extfs-for-overlay.img")+":ro", image.EXT3, "extfs", extfsTestString) } diff --git a/internal/pkg/util/fs/overlay/overlay_set_linux_test.go b/internal/pkg/util/fs/overlay/overlay_set_linux_test.go index 0a9e6f940f..10a648da0a 100644 --- a/internal/pkg/util/fs/overlay/overlay_set_linux_test.go +++ b/internal/pkg/util/fs/overlay/overlay_set_linux_test.go @@ -35,13 +35,13 @@ func TestAllTypesAtOnce(t *testing.T) { squashfsSupported := false if _, err := exec.LookPath("squashfs"); err == nil { squashfsSupported = true - addROItemOrFatal(t, &s, filepath.Join(".", "testdata", "squashfs.img")) + addROItemOrFatal(t, &s, filepath.Join("..", "..", "..", "..", "..", "test", "images", "squashfs-for-overlay.img")) } extfsSupported := false if _, err := exec.LookPath("fuse2fs"); err == nil { extfsSupported = true - addROItemOrFatal(t, &s, filepath.Join(".", "testdata", "extfs.img")+":ro") + addROItemOrFatal(t, &s, filepath.Join("..", "..", "..", "..", "..", "test", "images", "extfs-for-overlay.img")+":ro") } tmpRWDir := mkTempDirOrFatal(t) diff --git a/internal/pkg/util/fs/overlay/testdata/extfs.img b/test/images/extfs-for-overlay.img similarity index 100% rename from internal/pkg/util/fs/overlay/testdata/extfs.img rename to test/images/extfs-for-overlay.img diff --git a/internal/pkg/util/fs/overlay/testdata/squashfs.img b/test/images/squashfs-for-overlay.img similarity index 100% rename from internal/pkg/util/fs/overlay/testdata/squashfs.img rename to test/images/squashfs-for-overlay.img From ccdb4655beada3bf283155ebdefe4eff6ca1bf93 Mon Sep 17 00:00:00 2001 From: Edita Kizinevic Date: Mon, 3 Jul 2023 11:20:13 +0200 Subject: [PATCH 108/114] Disable check_pkg_no_buildcfg temporarily Signed-off-by: Edita Kizinevic --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 932a0a6fea..c5b8b543fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -377,6 +377,7 @@ jobs: retention-days: 7 check_pkg_no_buildcfg: + if: ${{ github.base_ref != 'oci-action' }} name: check_pkg_no_buildcfg runs-on: ubuntu-22.04 steps: From 765cc92115b3b09fa7bd70bfc2675b78c0df7b55 Mon Sep 17 00:00:00 2001 From: Omer Preminger Date: Thu, 18 May 2023 11:54:59 -0400 Subject: [PATCH 109/114] fix rel. path --workdir with --scratch, add native and oci e2e tests Signed-off-by: Edita Kizinevic --- e2e/actions/actions.go | 79 +++++++++++++++++++++--------------------- e2e/actions/oci.go | 43 +++++++++++++++++++++++ 2 files changed, 83 insertions(+), 39 deletions(-) diff --git a/e2e/actions/actions.go b/e2e/actions/actions.go index 632184d6d3..edd4d04419 100644 --- a/e2e/actions/actions.go +++ b/e2e/actions/actions.go @@ -2893,45 +2893,46 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { np := testhelper.NoParallel return testhelper.Tests{ - "action URI": c.RunFromURI, // action_URI - "singularity link": c.singularityLink, // singularity symlink - "exec": c.actionExec, // apptainer exec - "persistent overlay": c.PersistentOverlay, // Persistent Overlay - "persistent overlay unpriv": c.PersistentOverlayUnpriv, // Persistent Overlay Unprivileged - "run": c.actionRun, // apptainer run - "shell": c.actionShell, // shell interaction - "STDPIPE": c.STDPipe, // stdin/stdout pipe - "action basic profiles": c.actionBasicProfiles, // run basic action under different profiles - "issue 4488": c.issue4488, // https://github.com/apptainer/singularity/issues/4488 - "issue 4587": c.issue4587, // https://github.com/apptainer/singularity/issues/4587 - "issue 4755": c.issue4755, // https://github.com/apptainer/singularity/issues/4755 - "issue 4768": c.issue4768, // https://github.com/apptainer/singularity/issues/4768 - "issue 4797": c.issue4797, // https://github.com/apptainer/singularity/issues/4797 - "issue 4823": c.issue4823, // https://github.com/apptainer/singularity/issues/4823 - "issue 4836": c.issue4836, // https://github.com/apptainer/singularity/issues/4836 - "issue 5211": c.issue5211, // https://github.com/apptainer/singularity/issues/5211 - "issue 5228": c.issue5228, // https://github.com/apptainer/singularity/issues/5228 - "issue 5271": c.issue5271, // https://github.com/apptainer/singularity/issues/5271 - "issue 5399": c.issue5399, // https://github.com/apptainer/singularity/issues/5399 - "issue 5455": c.issue5455, // https://github.com/apptainer/singularity/issues/5455 - "issue 5465": c.issue5465, // https://github.com/apptainer/singularity/issues/5465 - "issue 5599": c.issue5599, // https://github.com/apptainer/singularity/issues/5599 - "issue 5631": c.issue5631, // https://github.com/apptainer/singularity/issues/5631 - "issue 5690": c.issue5690, // https://github.com/apptainer/singularity/issues/5690 - "issue 6165": c.issue6165, // https://github.com/apptainer/singularity/issues/6165 - "issue 619": c.issue619, // https://github.com/apptainer/apptainer/issues/619 - "network": c.actionNetwork, // test basic networking - "binds": c.actionBinds, // test various binds with --bind and --mount - "exit and signals": c.exitSignals, // test exit and signals propagation - "fuse mount": c.fuseMount, // test fusemount option - "bind image": c.bindImage, // test bind image with --bind and --mount - "unsquash": c.actionUnsquash, // test --unsquash - "no-mount": c.actionNoMount, // test --no-mount - "compat": np(c.actionCompat), // test --compat - "umask": np(c.actionUmask), // test umask propagation - "invalidRemote": np(c.invalidRemote), // GHSA-5mv9-q7fq-9394 - "fakeroot home": c.actionFakerootHome, // test home dir in fakeroot - "relWorkdirScratch": np(c.relWorkdirScratch), // test relative --workdir with --scratch + "action URI": c.RunFromURI, // action_URI + "singularity link": c.singularityLink, // singularity symlink + "exec": c.actionExec, // apptainer exec + "persistent overlay": c.PersistentOverlay, // Persistent Overlay + "persistent overlay unpriv": c.PersistentOverlayUnpriv, // Persistent Overlay Unprivileged + "run": c.actionRun, // apptainer run + "shell": c.actionShell, // shell interaction + "STDPIPE": c.STDPipe, // stdin/stdout pipe + "action basic profiles": c.actionBasicProfiles, // run basic action under different profiles + "issue 4488": c.issue4488, // https://github.com/apptainer/singularity/issues/4488 + "issue 4587": c.issue4587, // https://github.com/apptainer/singularity/issues/4587 + "issue 4755": c.issue4755, // https://github.com/apptainer/singularity/issues/4755 + "issue 4768": c.issue4768, // https://github.com/apptainer/singularity/issues/4768 + "issue 4797": c.issue4797, // https://github.com/apptainer/singularity/issues/4797 + "issue 4823": c.issue4823, // https://github.com/apptainer/singularity/issues/4823 + "issue 4836": c.issue4836, // https://github.com/apptainer/singularity/issues/4836 + "issue 5211": c.issue5211, // https://github.com/apptainer/singularity/issues/5211 + "issue 5228": c.issue5228, // https://github.com/apptainer/singularity/issues/5228 + "issue 5271": c.issue5271, // https://github.com/apptainer/singularity/issues/5271 + "issue 5399": c.issue5399, // https://github.com/apptainer/singularity/issues/5399 + "issue 5455": c.issue5455, // https://github.com/apptainer/singularity/issues/5455 + "issue 5465": c.issue5465, // https://github.com/apptainer/singularity/issues/5465 + "issue 5599": c.issue5599, // https://github.com/apptainer/singularity/issues/5599 + "issue 5631": c.issue5631, // https://github.com/apptainer/singularity/issues/5631 + "issue 5690": c.issue5690, // https://github.com/apptainer/singularity/issues/5690 + "issue 6165": c.issue6165, // https://github.com/apptainer/singularity/issues/6165 + "issue 619": c.issue619, // https://github.com/apptainer/apptainer/issues/619 + "network": c.actionNetwork, // test basic networking + "binds": c.actionBinds, // test various binds with --bind and --mount + "exit and signals": c.exitSignals, // test exit and signals propagation + "fuse mount": c.fuseMount, // test fusemount option + "bind image": c.bindImage, // test bind image with --bind and --mount + "unsquash": c.actionUnsquash, // test --unsquash + "no-mount": c.actionNoMount, // test --no-mount + "compat": np(c.actionCompat), // test --compat + "umask": np(c.actionUmask), // test umask propagation + "invalidRemote": np(c.invalidRemote), // GHSA-5mv9-q7fq-9394 + "fakeroot home": c.actionFakerootHome, // test home dir in fakeroot + "relWorkdirScratch": np(c.relWorkdirScratch), // test relative --workdir with --scratch + "ociRelWorkdirScratch": np(c.ociRelWorkdirScratch), // test relative --workdir with --scratch in OCI mode // // OCI Runtime Mode // diff --git a/e2e/actions/oci.go b/e2e/actions/oci.go index 497ab73b2d..515f1f2504 100644 --- a/e2e/actions/oci.go +++ b/e2e/actions/oci.go @@ -1324,6 +1324,49 @@ func haveAllCommands(t *testing.T, cmds []string) bool { return true } +// Make sure --workdir and --scratch work together nicely even when workdir is a +// relative path. Test needs to be run in non-parallel mode, because it changes +// the current working directory of the host. +func (c actionTests) ociRelWorkdirScratch(t *testing.T) { + e2e.EnsureOCIArchive(t, c.env) + imageRef := "oci-archive:" + c.env.OCIArchivePath + + testdir, cleanup := e2e.MakeTempDir(t, c.env.TestDir, "persistent-overlay-", "") + t.Cleanup(func() { + if !t.Failed() { + cleanup(t) + } + }) + + const subdirName string = "mysubdir" + if err := os.Mkdir(filepath.Join(testdir, subdirName), 0o777); err != nil { + t.Fatalf("could not create subdirectory %q in %q: %s", subdirName, testdir, err) + } + + // Change current working directory, with deferred undoing of change. + prevCwd, err := os.Getwd() + if err != nil { + t.Fatalf("could not get current working directory: %s", err) + } + defer os.Chdir(prevCwd) + if err = os.Chdir(testdir); err != nil { + t.Fatalf("could not change cwd to %q: %s", testdir, err) + } + + profiles := e2e.OCIProfiles + + for _, p := range profiles { + c.env.RunApptainer( + t, + e2e.AsSubtest(p.String()), + e2e.WithProfile(p), + e2e.WithCommand("exec"), + e2e.WithArgs("--workdir", "./"+subdirName, "--scratch", "/myscratch", imageRef, "true"), + e2e.ExpectExit(0), + ) + } +} + // ociSTDPipe tests pipe stdin/stdout to apptainer actions cmd func (c actionTests) ociSTDPipe(t *testing.T) { e2e.EnsureOCIArchive(t, c.env) From 37e24e580339c3ea18326017bcd1c5540af4dec3 Mon Sep 17 00:00:00 2001 From: Omer Preminger Date: Fri, 19 May 2023 10:14:26 -0400 Subject: [PATCH 110/114] error out when workdir can't be converted to abs path Signed-off-by: Edita Kizinevic --- internal/pkg/runtime/launcher/oci/mounts_linux.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/pkg/runtime/launcher/oci/mounts_linux.go b/internal/pkg/runtime/launcher/oci/mounts_linux.go index edcb8da550..3ecb969cc0 100644 --- a/internal/pkg/runtime/launcher/oci/mounts_linux.go +++ b/internal/pkg/runtime/launcher/oci/mounts_linux.go @@ -84,7 +84,7 @@ func (l *Launcher) addTmpMounts(mounts *[]specs.Mount) error { workdir, err := filepath.Abs(filepath.Clean(l.cfg.WorkDir)) if err != nil { - sylog.Warningf("Can't determine absolute path of workdir %s", l.cfg.WorkDir) + return fmt.Errorf("can't determine absolute path of workdir %s: %s", workdir, err) } tmpSrc := filepath.Join(workdir, tmpSrcSubdir) @@ -336,7 +336,7 @@ func (l *Launcher) addScratchMounts(mounts *[]specs.Mount) error { if len(l.cfg.WorkDir) > 0 { workdir, err := filepath.Abs(filepath.Clean(l.cfg.WorkDir)) if err != nil { - sylog.Warningf("Can't determine absolute path of workdir %s", l.cfg.WorkDir) + return fmt.Errorf("can't determine absolute path of workdir %s: %s", workdir, err) } scratchContainerDirPath := filepath.Join(workdir, scratchContainerDirName) if err := fs.Mkdir(scratchContainerDirPath, os.ModeSticky|0o777); err != nil && !os.IsExist(err) { From 1516258b027e5eddf488341398ce88317a5e64f1 Mon Sep 17 00:00:00 2001 From: Omer Preminger Date: Fri, 19 May 2023 11:08:45 -0400 Subject: [PATCH 111/114] privileged cleanup in (oci)RelWorkdirScratch tests Signed-off-by: Edita Kizinevic --- e2e/actions/oci.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/actions/oci.go b/e2e/actions/oci.go index 515f1f2504..e1a8b44da5 100644 --- a/e2e/actions/oci.go +++ b/e2e/actions/oci.go @@ -1334,7 +1334,7 @@ func (c actionTests) ociRelWorkdirScratch(t *testing.T) { testdir, cleanup := e2e.MakeTempDir(t, c.env.TestDir, "persistent-overlay-", "") t.Cleanup(func() { if !t.Failed() { - cleanup(t) + e2e.Privileged(cleanup) } }) From 12fb5998f7c489c1edf45cc02e906fe60c0bba06 Mon Sep 17 00:00:00 2001 From: preminger Date: Tue, 6 Jun 2023 10:15:50 -0400 Subject: [PATCH 112/114] oci: fallback to fuse-overlayfs if kernel doesn't support unprivileged overlays (sylabs/singularity#1730) * fallback to fuse-overlayfs if kernel doesn't support unpriv overlays * addressed first round of review comments * fix code-flow in UnprivOverlaysSupported() for unsupported case Signed-off-by: Edita Kizinevic --- CHANGELOG.md | 2 + internal/pkg/util/bin/bin.go | 2 +- .../pkg/util/fs/overlay/overlay_item_linux.go | 40 ++- .../fs/overlay/overlay_item_linux_test.go | 6 +- internal/pkg/util/fs/overlay/overlay_linux.go | 62 +++- .../pkg/util/fs/overlay/overlay_set_linux.go | 58 +++- .../util/fs/overlay/overlay_set_linux_test.go | 290 ++++++++++-------- 7 files changed, 299 insertions(+), 161 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5035c9cdb3..6211392577 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -81,6 +81,8 @@ For older changes see the [archived Singularity change log](https://github.com/a storage. If `--scratch ` is used in conjunction with `--workdir`, scratch directories will be mapped to subdirectories nested under `/scratch` on the host, rather than to tmpfs storage. +- If kernel does not support unprivileged overlays, OCI-mode will attempt to use + `fuse-overlayfs` and `fusermount` for overlay mounting and unmounting. ### New Features & Functionality diff --git a/internal/pkg/util/bin/bin.go b/internal/pkg/util/bin/bin.go index 8268826534..16f80d5cfd 100644 --- a/internal/pkg/util/bin/bin.go +++ b/internal/pkg/util/bin/bin.go @@ -76,7 +76,7 @@ func FindBin(name string) (path string, err error) { "gocryptfs": return findOnPath(name, false) default: - return "", fmt.Errorf("unknown executable name %q", name) + return "", fmt.Errorf("executable name %q is not known to FindBin", name) } } diff --git a/internal/pkg/util/fs/overlay/overlay_item_linux.go b/internal/pkg/util/fs/overlay/overlay_item_linux.go index 766e49004e..b3097ddaa7 100644 --- a/internal/pkg/util/fs/overlay/overlay_item_linux.go +++ b/internal/pkg/util/fs/overlay/overlay_item_linux.go @@ -51,8 +51,13 @@ type Item struct { func NewItemFromString(overlayString string) (*Item, error) { item := Item{Writable: true} + var err error splitted := strings.SplitN(overlayString, ":", 2) - item.SourcePath = splitted[0] + item.SourcePath, err = filepath.Abs(splitted[0]) + if err != nil { + return nil, fmt.Errorf("error while trying to convert overlay path %q to absolute path: %w", splitted[0], err) + } + if len(splitted) > 1 { if splitted[1] == "ro" { item.Writable = false @@ -70,7 +75,7 @@ func NewItemFromString(overlayString string) (*Item, error) { if s.IsDir() { item.Type = image.SANDBOX } else if err := item.analyzeImageFile(); err != nil { - return nil, fmt.Errorf("error encountered while examining image file %s: %s", item.SourcePath, err) + return nil, fmt.Errorf("while examining image file %s: %w", item.SourcePath, err) } return &item, nil @@ -159,12 +164,12 @@ func (i *Item) mountDir() error { } if err = EnsureOverlayDir(i.StagingDir, false, 0); err != nil { - return fmt.Errorf("error accessing directory %s: %s", i.StagingDir, err) + return fmt.Errorf("error accessing directory %s: %w", i.StagingDir, err) } sylog.Debugf("Performing identity bind-mount of %q", i.StagingDir) if err = syscall.Mount(i.StagingDir, i.StagingDir, "", syscall.MS_BIND, ""); err != nil { - return fmt.Errorf("failed to bind %s: %s", i.StagingDir, err) + return fmt.Errorf("failed to bind %s: %w", i.StagingDir, err) } // Best effort to cleanup mount @@ -178,7 +183,7 @@ func (i *Item) mountDir() error { // Try to perform remount sylog.Debugf("Performing remount of %q", i.StagingDir) if err = syscall.Mount("", i.StagingDir, "", syscall.MS_REMOUNT|syscall.MS_BIND, ""); err != nil { - return fmt.Errorf("failed to remount %s: %s", i.StagingDir, err) + return fmt.Errorf("failed to remount %s: %w", i.StagingDir, err) } return nil @@ -191,14 +196,14 @@ func (i *Item) mountWithFuse(fuseMountTool string, additionalArgs ...string) err var err error fuseMountCmd, err := bin.FindBin(fuseMountTool) if err != nil { - return fmt.Errorf("use of image %q as overlay requires %s to be installed: %s", i.SourcePath, fuseMountTool, err) + return fmt.Errorf("use of image %q as overlay requires %s to be installed: %w", i.SourcePath, fuseMountTool, err) } // Even though fusermount is not needed for this step, we shouldn't perform // the mount unless we have the necessary tools to eventually unmount it _, err = bin.FindBin("fusermount") if err != nil { - return fmt.Errorf("use of image %q as overlay requires fusermount to be installed: %s", i.SourcePath, err) + return fmt.Errorf("use of image %q as overlay requires fusermount to be installed: %w", i.SourcePath, err) } // Obtain parent directory in which to create overlay-related mount @@ -206,11 +211,11 @@ func (i *Item) mountWithFuse(fuseMountTool string, additionalArgs ...string) err // related discussion. parentDir, err := i.GetParentDir() if err != nil { - return fmt.Errorf("error while trying to create parent dir for overlay %q: %s", i.SourcePath, err) + return fmt.Errorf("error while trying to create parent dir for overlay %q: %w", i.SourcePath, err) } fuseMountDir, err := os.MkdirTemp(parentDir, "overlay-mountpoint-") if err != nil { - return fmt.Errorf("failed to create temporary dir %q for overlay %q: %s", fuseMountDir, i.SourcePath, err) + return fmt.Errorf("failed to create temporary dir %q for overlay %q: %w", fuseMountDir, i.SourcePath, err) } // Best effort to cleanup temporary dir @@ -230,7 +235,7 @@ func (i *Item) mountWithFuse(fuseMountTool string, additionalArgs ...string) err execCmd.Stderr = os.Stderr _, err = execCmd.Output() if err != nil { - return fmt.Errorf("encountered error while trying to mount image %q as overlay at %s: %s", i.SourcePath, fuseMountDir, err) + return fmt.Errorf("encountered error while trying to mount image %q as overlay at %s: %w", i.SourcePath, fuseMountDir, err) } i.StagingDir = fuseMountDir @@ -263,18 +268,9 @@ func (i Item) unmountDir() error { // unmountFuse unmounts FUSE-based Items. func (i Item) unmountFuse() error { defer os.Remove(i.StagingDir) - fusermountCmd, innerErr := bin.FindBin("fusermount") - if innerErr != nil { - // The code in performIndividualMounts() should not have created - // a FUSE-based overlay without fusermount in place - return fmt.Errorf("internal error: FUSE-based mount created without fusermount installed: %s", innerErr) - } - sylog.Debugf("Executing FUSE unmount command: %s -u %s", fusermountCmd, i.StagingDir) - execCmd := exec.Command(fusermountCmd, "-u", i.StagingDir) - execCmd.Stderr = os.Stderr - _, innerErr = execCmd.Output() - if innerErr != nil { - return fmt.Errorf("error while trying to unmount image %q from %s: %s", i.SourcePath, i.StagingDir, innerErr) + err := UnmountWithFuse(i.StagingDir) + if err != nil { + return fmt.Errorf("error while trying to unmount image %q from %s: %w", i.SourcePath, i.StagingDir, err) } return nil } diff --git a/internal/pkg/util/fs/overlay/overlay_item_linux_test.go b/internal/pkg/util/fs/overlay/overlay_item_linux_test.go index bd1138875c..bd49f1c7d4 100644 --- a/internal/pkg/util/fs/overlay/overlay_item_linux_test.go +++ b/internal/pkg/util/fs/overlay/overlay_item_linux_test.go @@ -181,10 +181,10 @@ func dirMountUnmount(t *testing.T, olStr string) { } if err := item.Mount(); err != nil { - t.Fatalf("error encountered while trying to mount dir %q: %s", olStr, err) + t.Fatalf("while trying to mount dir %q: %s", olStr, err) } if err := item.Unmount(); err != nil { - t.Errorf("error encountered while trying to unmount dir %q: %s", olStr, err) + t.Errorf("while trying to unmount dir %q: %s", olStr, err) } } @@ -223,10 +223,12 @@ func tryImageRO(t *testing.T, olStr string, typeCode int, typeStr, expectStr str func TestSquashfsRO(t *testing.T) { require.Command(t, "squashfuse") + require.Command(t, "fusermount") tryImageRO(t, filepath.Join("..", "..", "..", "..", "..", "test", "images", "squashfs-for-overlay.img"), image.SQUASHFS, "squashfs", squashfsTestString) } func TestExtfsRO(t *testing.T) { require.Command(t, "fuse2fs") + require.Command(t, "fusermount") tryImageRO(t, filepath.Join("..", "..", "..", "..", "..", "test", "images", "extfs-for-overlay.img")+":ro", image.EXT3, "extfs", extfsTestString) } diff --git a/internal/pkg/util/fs/overlay/overlay_linux.go b/internal/pkg/util/fs/overlay/overlay_linux.go index 221ad8c87c..cbe6336a65 100644 --- a/internal/pkg/util/fs/overlay/overlay_linux.go +++ b/internal/pkg/util/fs/overlay/overlay_linux.go @@ -14,6 +14,7 @@ import ( "fmt" "os" "os/exec" + "sync" "syscall" "github.com/apptainer/apptainer/internal/pkg/util/bin" @@ -86,7 +87,7 @@ func check(path string, d dir) error { stfs := &unix.Statfs_t{} if err := statfs(path, stfs); err != nil { - return fmt.Errorf("could not retrieve underlying filesystem information for %s: %s", path, err) + return fmt.Errorf("could not retrieve underlying filesystem information for %s: %w", path, err) } fs, ok := incompatibleFs[int64(stfs.Type)] @@ -152,7 +153,7 @@ var ErrNoRootlessOverlay = errors.New("rootless overlay not supported by kernel" func CheckRootless() error { mountBin, err := bin.FindBin("mount") if err != nil { - return fmt.Errorf("while looking for mount command: %s", err) + return fmt.Errorf("while looking for mount command: %w", err) } args := []string{ @@ -191,6 +192,37 @@ func CheckRootless() error { return nil } +// Info about kernel support for unprivileged overlays +var unprivOverlays struct { + kernelSupport bool + initOnce sync.Once + err error +} + +// UnprivOverlaysSupported checks whether there is kernel support for unprivileged overlays. The actual check is performed only once and cached in the unprivOverlays variable, above. +func UnprivOverlaysSupported() (bool, error) { + unprivOverlays.initOnce.Do(func() { + err := CheckRootless() + if err == nil { + unprivOverlays.kernelSupport = true + return + } + + if err == ErrNoRootlessOverlay { + unprivOverlays.kernelSupport = false + return + } + + unprivOverlays.err = err + }) + + if unprivOverlays.err != nil { + return false, unprivOverlays.err + } + + return unprivOverlays.kernelSupport, nil +} + // ensureOverlayDir checks if a directory already exists; if it doesn't, and // createIfMissing is true, it attempts to create it with the specified // permissions. @@ -214,7 +246,7 @@ func EnsureOverlayDir(dir string, createIfMissing bool, createPerm os.FileMode) // Create the requested dir if err := os.Mkdir(dir, createPerm); err != nil { - return fmt.Errorf("failed to create %q: %s", dir, err) + return fmt.Errorf("failed to create %q: %w", dir, err) } return nil @@ -225,22 +257,38 @@ func EnsureOverlayDir(dir string, createIfMissing bool, createPerm os.FileMode) func DetachAndDelete(overlayDir string) error { sylog.Debugf("Detaching overlayDir %q", overlayDir) if err := syscall.Unmount(overlayDir, syscall.MNT_DETACH); err != nil { - return fmt.Errorf("failed to unmount %s: %s", overlayDir, err) + return fmt.Errorf("failed to unmount %s: %w", overlayDir, err) } sylog.Debugf("Removing overlayDir %q", overlayDir) if err := os.RemoveAll(overlayDir); err != nil { - return fmt.Errorf("failed to remove %s: %s", overlayDir, err) + return fmt.Errorf("failed to remove %s: %w", overlayDir, err) } return nil } -// detachMount performs an unmount system call on the specified directory. +// DetachMount performs an unmount system call on the specified directory. func DetachMount(dir string) error { sylog.Debugf("Calling syscall.Unmount() to detach %q", dir) if err := syscall.Unmount(dir, syscall.MNT_DETACH); err != nil { - return fmt.Errorf("failed to detach %s: %s", dir, err) + return fmt.Errorf("failed to detach %s: %w", dir, err) } return nil } + +// UnmountWithFuse performs an unmount on the specified directory using +// fusermount -u. +func UnmountWithFuse(dir string) error { + fusermountCmd, err := bin.FindBin("fusermount") + if err != nil { + // We should not be creating FUSE-based mounts in the first place + // without checking that fusermount is available. + return fmt.Errorf("fusermount not available while trying to perform unmount: %w", err) + } + sylog.Debugf("Executing FUSE unmount command: %s -u %s", fusermountCmd, dir) + execCmd := exec.Command(fusermountCmd, "-u", dir) + execCmd.Stderr = os.Stderr + _, err = execCmd.Output() + return err +} diff --git a/internal/pkg/util/fs/overlay/overlay_set_linux.go b/internal/pkg/util/fs/overlay/overlay_set_linux.go index 160c17d688..dc50ea389a 100644 --- a/internal/pkg/util/fs/overlay/overlay_set_linux.go +++ b/internal/pkg/util/fs/overlay/overlay_set_linux.go @@ -11,9 +11,12 @@ package overlay import ( "fmt" + "os" + "os/exec" "strings" "syscall" + "github.com/apptainer/apptainer/internal/pkg/util/bin" "github.com/apptainer/apptainer/pkg/sylog" "github.com/samber/lo" ) @@ -43,6 +46,15 @@ type Set struct { // directory. func (s Set) Mount(rootFsDir string) error { // Perform identity mounts for this Set + dups := lo.FindDuplicatesBy(s.ReadonlyOverlays, func(item *Item) string { + return item.SourcePath + }) + if len(dups) > 0 { + return fmt.Errorf("duplicate overlays detected: %v", lo.Map(dups, func(item *Item, _ int) string { + return item.SourcePath + })) + } + if err := s.performIndividualMounts(); err != nil { return err } @@ -53,7 +65,18 @@ func (s Set) Mount(rootFsDir string) error { // UnmountOverlay ummounts a Set from a specified rootfs directory. func (s Set) Unmount(rootFsDir string) error { - if err := DetachMount(rootFsDir); err != nil { + unprivOls, err := UnprivOverlaysSupported() + if err != nil { + return fmt.Errorf("while checking for unprivileged overlay support in kernel: %w", err) + } + + if unprivOls { + err = DetachMount(rootFsDir) + } else { + err = UnmountWithFuse(rootFsDir) + } + + if err != nil { return err } @@ -84,9 +107,36 @@ func (s Set) performIndividualMounts() error { func (s Set) performFinalMount(rootFsDir string) error { // Try to perform actual mount options := s.options(rootFsDir) - sylog.Debugf("Mounting overlay with rootFsDir %q, options: %q", rootFsDir, options) - if err := syscall.Mount("overlay", rootFsDir, "overlay", syscall.MS_NODEV, options); err != nil { - return fmt.Errorf("failed to mount %s: %s", rootFsDir, err) + unprivOls, err := UnprivOverlaysSupported() + if err != nil { + return fmt.Errorf("while checking for unprivileged overlay support in kernel: %w", err) + } + + if unprivOls { + sylog.Debugf("Mounting overlay (via syscall) with rootFsDir %q, options: %q", rootFsDir, options) + if err := syscall.Mount("overlay", rootFsDir, "overlay", syscall.MS_NODEV, options); err != nil { + return fmt.Errorf("failed to mount %s: %w", rootFsDir, err) + } + } else { + fuseOlFsCmd, err := bin.FindBin("fuse-overlayfs") + if err != nil { + return fmt.Errorf("kernel does not support unprivileged overlays, and fuse-overlayfs not available: %w", err) + } + + // Even though fusermount is not needed for this step, we shouldn't perform + // the mount unless we have the necessary tools to eventually unmount it + _, err = bin.FindBin("fusermount") + if err != nil { + return fmt.Errorf("kernel does not support unprivileged overlays, and using fuse-overlayfs fallback requires fusermount to be installed: %w", err) + } + + sylog.Debugf("Mounting overlay (via fuse-overlayfs) with rootFsDir %q, options: %q", rootFsDir, options) + execCmd := exec.Command(fuseOlFsCmd, "-o", options, rootFsDir) + execCmd.Stderr = os.Stderr + _, err = execCmd.Output() + if err != nil { + return fmt.Errorf("failed to mount %s: %w", rootFsDir, err) + } } return nil diff --git a/internal/pkg/util/fs/overlay/overlay_set_linux_test.go b/internal/pkg/util/fs/overlay/overlay_set_linux_test.go index 10a648da0a..93cbae1e50 100644 --- a/internal/pkg/util/fs/overlay/overlay_set_linux_test.go +++ b/internal/pkg/util/fs/overlay/overlay_set_linux_test.go @@ -14,6 +14,8 @@ import ( "os/exec" "path/filepath" "testing" + + "github.com/apptainer/apptainer/internal/pkg/test/tool/require" ) func addROItemOrFatal(t *testing.T, s *Set, olStr string) *Item { @@ -26,151 +28,189 @@ func addROItemOrFatal(t *testing.T, s *Set, olStr string) *Item { return i } -func TestAllTypesAtOnce(t *testing.T) { - s := Set{} +// wrapOverlayTest takes a testing function and wraps it in code that checks if +// the kernel has support for unprivileged overlays. If it does, the underlying +// function will be run twice, once with using kernel overlays and once using +// fuse-overlayfs (if present). Otherwise, only the latter option will be +// attempted. +func wrapOverlayTest(f func(t *testing.T)) func(t *testing.T) { + unprivOls, unprivOlsErr := UnprivOverlaysSupported() + return func(t *testing.T) { + if unprivOlsErr != nil { + t.Fatalf("while checking for unprivileged overlay support in kernel: %s", unprivOlsErr) + } - tmpRODir := mkTempDirOrFatal(t) - addROItemOrFatal(t, &s, tmpRODir+":ro") + fuseOverlayFunc := func(t *testing.T) { + require.Command(t, "fuse-overlayfs") + require.Command(t, "fusermount") + f(t) + } - squashfsSupported := false - if _, err := exec.LookPath("squashfs"); err == nil { - squashfsSupported = true - addROItemOrFatal(t, &s, filepath.Join("..", "..", "..", "..", "..", "test", "images", "squashfs-for-overlay.img")) - } + if unprivOls { + t.Run("kerneloverlay", f) + unprivOverlays.kernelSupport = false + } - extfsSupported := false - if _, err := exec.LookPath("fuse2fs"); err == nil { - extfsSupported = true - addROItemOrFatal(t, &s, filepath.Join("..", "..", "..", "..", "..", "test", "images", "extfs-for-overlay.img")+":ro") + t.Run("fuseoverlayfs", fuseOverlayFunc) + unprivOverlays.kernelSupport = unprivOls } +} - tmpRWDir := mkTempDirOrFatal(t) - i, err := NewItemFromString(tmpRWDir) - if err != nil { - t.Fatalf("failed to create writable-dir overlay item (%q): %s", tmpRWDir, err) - } - s.WritableOverlay = i +func TestAllTypesAtOnce(t *testing.T) { + wrapOverlayTest(func(t *testing.T) { + s := Set{} - rootfsDir := mkTempDirOrFatal(t) - if err := s.Mount(rootfsDir); err != nil { - t.Fatalf("failed to mount overlay set: %s", err) - } - t.Cleanup(func() { - s.Unmount(rootfsDir) - }) - - var expectStr string - if extfsSupported { - expectStr = extfsTestString - } else if squashfsSupported { - expectStr = squashfsTestString - } + tmpRODir := mkTempDirOrFatal(t) + addROItemOrFatal(t, &s, tmpRODir+":ro") - if squashfsSupported || extfsSupported { - testFileMountedPath := filepath.Join(rootfsDir, testFilePath) - data, err := os.ReadFile(testFileMountedPath) - if err != nil { - t.Fatalf("error while trying to read from file %q: %s", testFileMountedPath, err) + squashfsSupported := false + if _, err := exec.LookPath("squashfs"); err == nil { + squashfsSupported = true + addROItemOrFatal(t, &s, filepath.Join("..", "..", "..", "..", "..", "test", "images", "squashfs-for-overlay.img")) } - foundStr := string(data) - if foundStr != expectStr { - t.Errorf("incorrect file contents in mounted overlay set: expected %q, but found: %q", expectStr, foundStr) + + extfsSupported := false + if _, err := exec.LookPath("fuse2fs"); err == nil { + extfsSupported = true + addROItemOrFatal(t, &s, filepath.Join("..", "..", "..", "..", "..", "test", "images", "extfs-for-overlay.img")+":ro") } - } - if err := s.Unmount(rootfsDir); err != nil { - t.Errorf("error encountered while trying to unmount overlay set: %s", err) - } -} + tmpRWDir := mkTempDirOrFatal(t) + i, err := NewItemFromString(tmpRWDir) + if err != nil { + t.Fatalf("failed to create writable-dir overlay item (%q): %s", tmpRWDir, err) + } + s.WritableOverlay = i -func TestPersistentWriteToDir(t *testing.T) { - tmpRWDir := mkTempDirOrFatal(t) - i, err := NewItemFromString(tmpRWDir) - if err != nil { - t.Fatalf("failed to create writable-dir overlay item (%q): %s", tmpRWDir, err) - } - s := Set{WritableOverlay: i} + rootfsDir := mkTempDirOrFatal(t) + if err := s.Mount(rootfsDir); err != nil { + t.Fatalf("failed to mount overlay set: %s", err) + } + t.Cleanup(func() { + if t.Failed() { + s.Unmount(rootfsDir) + } + }) + + var expectStr string + if extfsSupported { + expectStr = extfsTestString + } else if squashfsSupported { + expectStr = squashfsTestString + } - rootfsDir := mkTempDirOrFatal(t) + if squashfsSupported || extfsSupported { + testFileMountedPath := filepath.Join(rootfsDir, testFilePath) + data, err := os.ReadFile(testFileMountedPath) + if err != nil { + t.Fatalf("error while trying to read from file %q: %s", testFileMountedPath, err) + } + foundStr := string(data) + if foundStr != expectStr { + t.Errorf("incorrect file contents in mounted overlay set: expected %q, but found: %q", expectStr, foundStr) + } + } - // This cleanup will serve adequately for both iterations of the overlay-set - // mounting, below. If it happens to get called while the set is not - // mounted, it should fail silently. - t.Cleanup(func() { - s.Unmount(rootfsDir) - }) + if err := s.Unmount(rootfsDir); err != nil { + t.Errorf("while trying to unmount overlay set: %s", err) + } + })(t) +} - // Mount the overlay set, write a string to a file, and unmount. - if err := s.Mount(rootfsDir); err != nil { - t.Fatalf("failed to mount overlay set: %s", err) - } - expectStr := "my_test_string" - bytes := []byte(expectStr) - testFilePath := "my_test_file" - testFileMountedPath := filepath.Join(rootfsDir, testFilePath) - if err := os.WriteFile(testFileMountedPath, bytes, 0o644); err != nil { - t.Fatalf("error encountered while trying to write file inside mounted overlay-set: %s", err) - } +func TestPersistentWriteToDir(t *testing.T) { + wrapOverlayTest(func(t *testing.T) { + tmpRWDir := mkTempDirOrFatal(t) + i, err := NewItemFromString(tmpRWDir) + if err != nil { + t.Fatalf("failed to create writable-dir overlay item (%q): %s", tmpRWDir, err) + } + s := Set{WritableOverlay: i} + + rootfsDir := mkTempDirOrFatal(t) + + // This cleanup will serve adequately for both iterations of the overlay-set + // mounting, below. If it happens to get called while the set is not + // mounted, it should fail silently. + t.Cleanup(func() { + if t.Failed() { + s.Unmount(rootfsDir) + } + }) + + // Mount the overlay set, write a string to a file, and unmount. + if err := s.Mount(rootfsDir); err != nil { + t.Fatalf("failed to mount overlay set: %s", err) + } + expectStr := "my_test_string" + bytes := []byte(expectStr) + testFilePath := "my_test_file" + testFileMountedPath := filepath.Join(rootfsDir, testFilePath) + if err := os.WriteFile(testFileMountedPath, bytes, 0o644); err != nil { + t.Fatalf("while trying to write file inside mounted overlay-set: %s", err) + } - if err := s.Unmount(rootfsDir); err != nil { - t.Fatalf("error encountered while trying to unmount overlay set: %s", err) - } + if err := s.Unmount(rootfsDir); err != nil { + t.Fatalf("while trying to unmount overlay set: %s", err) + } - // Mount the same set again, and check that we see the file with the - // expected contents. - if err := s.Mount(rootfsDir); err != nil { - t.Fatalf("failed to mount overlay set: %s", err) - } - data, err := os.ReadFile(testFileMountedPath) - if err != nil { - t.Fatalf("error while trying to read from file %q: %s", testFileMountedPath, err) - } - foundStr := string(data) - if foundStr != expectStr { - t.Errorf("incorrect file contents in mounted overlay set: expected %q, but found: %q", expectStr, foundStr) - } - if err := s.Unmount(rootfsDir); err != nil { - t.Errorf("error encountered while trying to unmount overlay set: %s", err) - } + // Mount the same set again, and check that we see the file with the + // expected contents. + if err := s.Mount(rootfsDir); err != nil { + t.Fatalf("failed to mount overlay set: %s", err) + } + data, err := os.ReadFile(testFileMountedPath) + if err != nil { + t.Fatalf("error while trying to read from file %q: %s", testFileMountedPath, err) + } + foundStr := string(data) + if foundStr != expectStr { + t.Errorf("incorrect file contents in mounted overlay set: expected %q, but found: %q", expectStr, foundStr) + } + if err := s.Unmount(rootfsDir); err != nil { + t.Errorf("while trying to unmount overlay set: %s", err) + } + })(t) } func TestDuplicateItemsInSet(t *testing.T) { - var rootfsDir string - var rwI *Item - var err error - - s := Set{} - - // First, test mounting of an overlay set with only readonly items, one of - // which is a duplicate of another. - addROItemOrFatal(t, &s, mkTempDirOrFatal(t)+":ro") - roI2 := addROItemOrFatal(t, &s, mkTempDirOrFatal(t)+":ro") - addROItemOrFatal(t, &s, mkTempDirOrFatal(t)+":ro") - addROItemOrFatal(t, &s, roI2.SourcePath+":ro") - addROItemOrFatal(t, &s, mkTempDirOrFatal(t)+":ro") - - rootfsDir = mkTempDirOrFatal(t) - if err := s.Mount(rootfsDir); err == nil { - t.Errorf("unexpected success: Mounting overlay.Set with duplicate (%q) should have failed", roI2.SourcePath) - if err := s.Unmount(rootfsDir); err != nil { - t.Fatalf("could not unmount erroneous successful mount of overlay set: %s", err) + wrapOverlayTest(func(t *testing.T) { + var rootfsDir string + var rwI *Item + var err error + + s := Set{} + + // First, test mounting of an overlay set with only readonly items, one of + // which is a duplicate of another. + addROItemOrFatal(t, &s, mkTempDirOrFatal(t)+":ro") + roI2 := addROItemOrFatal(t, &s, mkTempDirOrFatal(t)+":ro") + addROItemOrFatal(t, &s, mkTempDirOrFatal(t)+":ro") + addROItemOrFatal(t, &s, roI2.SourcePath+":ro") + addROItemOrFatal(t, &s, mkTempDirOrFatal(t)+":ro") + + rootfsDir = mkTempDirOrFatal(t) + if err := s.Mount(rootfsDir); err == nil { + t.Errorf("unexpected success: Mounting overlay.Set with duplicate (%q) should have failed", roI2.SourcePath) + if err := s.Unmount(rootfsDir); err != nil { + t.Fatalf("could not unmount erroneous successful mount of overlay set: %s", err) + } } - } - - // Next, test mounting of an overlay set with a writable item as well as - // several readonly items, one of which is a duplicate of another. - tmpRWDir := mkTempDirOrFatal(t) - rwI, err = NewItemFromString(tmpRWDir) - if err != nil { - t.Fatalf("failed to create writable-dir overlay item (%q): %s", tmpRWDir, err) - } - s.WritableOverlay = rwI - rootfsDir = mkTempDirOrFatal(t) - if err := s.Mount(rootfsDir); err == nil { - t.Errorf("unexpected success: Mounting overlay.Set with duplicate file/dir (%q) should have failed", roI2.SourcePath) - if err := s.Unmount(rootfsDir); err != nil { - t.Fatalf("could not unmount erroneous successful mount of overlay set: %s", err) + // Next, test mounting of an overlay set with a writable item as well as + // several readonly items, one of which is a duplicate of another. + tmpRWDir := mkTempDirOrFatal(t) + rwI, err = NewItemFromString(tmpRWDir) + if err != nil { + t.Fatalf("failed to create writable-dir overlay item (%q): %s", tmpRWDir, err) } - } + s.WritableOverlay = rwI + + rootfsDir = mkTempDirOrFatal(t) + if err := s.Mount(rootfsDir); err == nil { + t.Errorf("unexpected success: Mounting overlay.Set with duplicate (%q) should have failed", roI2.SourcePath) + if err := s.Unmount(rootfsDir); err != nil { + t.Fatalf("could not unmount erroneous successful mount of overlay set: %s", err) + } + } + })(t) } From c3fa965fedd842dac5ec8906da25fea16648ea9a Mon Sep 17 00:00:00 2001 From: preminger Date: Thu, 8 Jun 2023 10:42:06 -0400 Subject: [PATCH 113/114] oci: support for writable extfs img overlay via fuse-overlayfs (sylabs/singularity#1740) * oci: support for writable extfs img overlay via fuse-overlayfs * added unit- and e2e-tests * fixed exposure of "upper" & "work" subdir in readonly overlays * added fix for file-ownership in FUSE-mounted images + e2e test of fix * fix handling of "upper" in :ro overlay dirs, adjust e2e tests Signed-off-by: Edita Kizinevic --- CHANGELOG.md | 11 +- e2e/actions/actions.go | 23 +- e2e/actions/oci.go | 218 +++++++++++++++--- internal/pkg/test/tool/dirs/mkdir.go | 15 ++ .../pkg/util/fs/overlay/overlay_item_linux.go | 60 ++++- .../fs/overlay/overlay_item_linux_test.go | 112 +++++++-- internal/pkg/util/fs/overlay/overlay_linux.go | 6 +- .../pkg/util/fs/overlay/overlay_set_linux.go | 27 ++- .../util/fs/overlay/overlay_set_linux_test.go | 145 +++++++----- test/images/extfs-for-overlay.img | Bin 2097152 -> 2097152 bytes 10 files changed, 474 insertions(+), 143 deletions(-) create mode 100644 internal/pkg/test/tool/dirs/mkdir.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 6211392577..7d118b3e6f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -70,11 +70,12 @@ For older changes see the [archived Singularity change log](https://github.com/a `--dns` flag can be used to pass a comma-separated list of DNS servers that will be used in the container; if this flag is not used, the container will use the same `resolv.conf` settings as the host. -- OCI-mode now supports an `--overlay ` flag. `` can be a writable - directory, in which case changes to the filesystem will persist across runs of - the OCI container. Alternatively, `` can be `:ro` or the path of a - squashfs or extfs image, to be mounted as a read-only overlay. Multiple - overlays can be specified, but all but one must be read-only. +- OCI-mode now supports the `--overlay ` flag. `` can be the path to a + writable directory or writable extfs image, in which case changes to the + filesystem will persist across runs of the OCI container. Alternatively, + `--overlay :ro` can be used, where `` is the path to a directory, to + a squashfs image, or to an extfs image, to be mounted as a read-only overlay. + Multiple overlays can be specified, but all but one must be read-only. - OCI-mode now supports the `--workdir ` option. If this option is specified, `/tmp` and `/var/tmp` will be mapped, respectively, to `/tmp` and `/var_tmp` on the host, rather than to tmpfs diff --git a/e2e/actions/actions.go b/e2e/actions/actions.go index edd4d04419..30bf7f7c45 100644 --- a/e2e/actions/actions.go +++ b/e2e/actions/actions.go @@ -2936,16 +2936,17 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { // // OCI Runtime Mode // - "ociRun": c.actionOciRun, // apptainer run --oci - "ociExec": c.actionOciExec, // apptainer exec --oci - "ociShell": c.actionOciShell, // apptainer shell --oci - "ociSTDPIPE": c.ociSTDPipe, // stdin/stdout pipe --oci - "ociNetwork": c.actionOciNetwork, // apptainer exec --oci --net - "ociBinds": c.actionOciBinds, // apptainer exec --oci --bind / --mount - "ociCdi": c.actionOciCdi, // apptainer exec --oci --cdi - "ociIDMaps": c.actionOciIDMaps, // check uid/gid mapping on host for --oci as user / --fakeroot - "ociCompat": np(c.actionOciCompat), // --oci equivalence to native mode --compat - "ociOverlay": (c.actionOciOverlay), // --overlay in OCI mode - "ociOverlayTeardown": np(c.actionOciOverlayTeardown), // proper overlay unmounting in OCI mode + "ociRun": c.actionOciRun, // apptainer run --oci + "ociExec": c.actionOciExec, // apptainer exec --oci + "ociShell": c.actionOciShell, // apptainer shell --oci + "ociSTDPIPE": c.ociSTDPipe, // stdin/stdout pipe --oci + "ociNetwork": c.actionOciNetwork, // apptainer exec --oci --net + "ociBinds": c.actionOciBinds, // apptainer exec --oci --bind / --mount + "ociCdi": c.actionOciCdi, // apptainer exec --oci --cdi + "ociIDMaps": c.actionOciIDMaps, // check uid/gid mapping on host for --oci as user / --fakeroot + "ociCompat": np(c.actionOciCompat), // --oci equivalence to native mode --compat + "ociOverlay": (c.actionOciOverlay), // --overlay in OCI mode + "ociOverlayExtfsPerms": (c.actionOciOverlayExtfsPerms), // permissions in writable extfs overlays mounted with FUSE in OCI mode + "ociOverlayTeardown": np(c.actionOciOverlayTeardown), // proper overlay unmounting in OCI mode } } diff --git a/e2e/actions/oci.go b/e2e/actions/oci.go index e1a8b44da5..f638883f4e 100644 --- a/e2e/actions/oci.go +++ b/e2e/actions/oci.go @@ -23,6 +23,8 @@ import ( "text/template" "github.com/apptainer/apptainer/e2e/internal/e2e" + "github.com/apptainer/apptainer/internal/pkg/test/tool/dirs" + "github.com/apptainer/apptainer/internal/pkg/test/tool/require" "github.com/apptainer/apptainer/internal/pkg/util/fs" cdispecs "github.com/container-orchestrated-devices/container-device-interface/specs-go" "gotest.tools/v3/assert" @@ -1046,13 +1048,14 @@ func (c actionTests) actionOciOverlay(t *testing.T) { } }) - // Create a few read-only overlay subdirs under testDir + // Create a few writable overlay subdirs under testDir for i := 0; i < 3; i++ { dirName := fmt.Sprintf("my_rw_ol_dir%d", i) fullPath := filepath.Join(testDir, dirName) - if err = os.Mkdir(fullPath, 0o755); err != nil { - t.Fatal(err) - } + dirs.MkdirOrFatal(t, fullPath, 0o755) + upperPath := filepath.Join(fullPath, "upper") + dirs.MkdirOrFatal(t, upperPath, 0o777) + dirs.MkdirOrFatal(t, filepath.Join(fullPath, "work"), 0o777) t.Cleanup(func() { if !t.Failed() { os.RemoveAll(fullPath) @@ -1064,28 +1067,45 @@ func (c actionTests) actionOciOverlay(t *testing.T) { for i := 0; i < 3; i++ { dirName := fmt.Sprintf("my_ro_ol_dir%d", i) fullPath := filepath.Join(testDir, dirName) - if err = os.Mkdir(fullPath, 0o755); err != nil { - t.Fatal(err) - } + dirs.MkdirOrFatal(t, fullPath, 0o755) + upperPath := filepath.Join(fullPath, "upper") + dirs.MkdirOrFatal(t, upperPath, 0o777) + dirs.MkdirOrFatal(t, filepath.Join(fullPath, "work"), 0o777) t.Cleanup(func() { if !t.Failed() { os.RemoveAll(fullPath) } }) if err = os.WriteFile( - filepath.Join(fullPath, fmt.Sprintf("testfile.%d", i)), + filepath.Join(upperPath, fmt.Sprintf("testfile.%d", i)), []byte(fmt.Sprintf("test_string_%d\n", i)), 0o644); err != nil { t.Fatal(err) } if err = os.WriteFile( - filepath.Join(fullPath, "maskable_testfile"), + filepath.Join(upperPath, "maskable_testfile"), []byte(fmt.Sprintf("maskable_string_%d\n", i)), 0o644); err != nil { t.Fatal(err) } } + // Create a copy of the extfs test image to be used for testing readonly + // extfs image overlays + readonlyExtfsImgPath := filepath.Join(testDir, "readonly-extfs.img") + err = fs.CopyFile(extfsImgPath, readonlyExtfsImgPath, 0o444) + if err != nil { + t.Fatalf("could not copy %q to %q: %s", extfsImgPath, readonlyExtfsImgPath, err) + } + + // Create a copy of the extfs test image to be used for testing writable + // extfs image overlays + writableExtfsImgPath := filepath.Join(testDir, "writable-extfs.img") + err = fs.CopyFile(extfsImgPath, writableExtfsImgPath, 0o755) + if err != nil { + t.Fatalf("could not copy %q to %q: %s", extfsImgPath, writableExtfsImgPath, err) + } + tests := []struct { name string args []string @@ -1106,6 +1126,14 @@ func (c actionTests) actionOciOverlay(t *testing.T) { e2e.ExpectOutput(e2e.ExactMatch, "my_test_string"), }, }, + { + name: "ExistRWDirRevisitAsRO", + args: []string{"--overlay", filepath.Join(testDir, "my_rw_ol_dir0:ro"), imageRef, "cat", "/my_test_file"}, + exitCode: 0, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ExactMatch, "my_test_string"), + }, + }, { name: "RWOverlayMissing", args: []string{"--overlay", filepath.Join(testDir, "something_nonexistent"), imageRef, "echo", "hi"}, @@ -1139,13 +1167,13 @@ func (c actionTests) actionOciOverlay(t *testing.T) { name: "AllTypesAtOnce", args: []string{ "--overlay", filepath.Join(testDir, "my_ro_ol_dir2:ro"), - "--overlay", extfsImgPath + ":ro", + "--overlay", readonlyExtfsImgPath + ":ro", "--overlay", squashfsImgPath, "--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"), "--overlay", filepath.Join(testDir, "my_rw_ol_dir0"), imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath), }, - requiredCmds: []string{"squashfuse", "fuse2fs"}, + requiredCmds: []string{"squashfuse", "fuse2fs", "fusermount"}, exitCode: 0, wantOutputs: []e2e.ApptainerCmdResultOp{ e2e.ExpectOutput(e2e.ContainMatch, "test_string_1"), @@ -1162,7 +1190,7 @@ func (c actionTests) actionOciOverlay(t *testing.T) { "--overlay", filepath.Join(testDir, "my_rw_ol_dir0"), imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath), }, - requiredCmds: []string{"squashfuse"}, + requiredCmds: []string{"squashfuse", "fusermount"}, exitCode: 0, wantOutputs: []e2e.ApptainerCmdResultOp{ e2e.ExpectOutput(e2e.ContainMatch, "test_string_1"), @@ -1174,12 +1202,12 @@ func (c actionTests) actionOciOverlay(t *testing.T) { name: "ExtfsAndDirs", args: []string{ "--overlay", filepath.Join(testDir, "my_ro_ol_dir2:ro"), - "--overlay", extfsImgPath + ":ro", + "--overlay", readonlyExtfsImgPath + ":ro", "--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"), "--overlay", filepath.Join(testDir, "my_rw_ol_dir0"), imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath), }, - requiredCmds: []string{"fuse2fs"}, + requiredCmds: []string{"fuse2fs", "fusermount"}, exitCode: 0, wantOutputs: []e2e.ApptainerCmdResultOp{ e2e.ExpectOutput(e2e.ContainMatch, "test_string_1"), @@ -1197,7 +1225,7 @@ func (c actionTests) actionOciOverlay(t *testing.T) { "--overlay", filepath.Join(testDir, "my_rw_ol_dir0"), imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath), }, - requiredCmds: []string{"squashfuse"}, + requiredCmds: []string{"squashfuse", "fusermount"}, exitCode: 255, }, { @@ -1209,7 +1237,7 @@ func (c actionTests) actionOciOverlay(t *testing.T) { "--overlay", filepath.Join(testDir, "something_nonexistent"), imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath), }, - requiredCmds: []string{"squashfuse"}, + requiredCmds: []string{"squashfuse", "fusermount"}, exitCode: 255, }, { @@ -1221,8 +1249,7 @@ func (c actionTests) actionOciOverlay(t *testing.T) { "--overlay", filepath.Join(testDir, "my_rw_ol_dir0"), imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath), }, - requiredCmds: []string{"squashfuse"}, - exitCode: 255, + exitCode: 255, }, { name: "ThreeWritables", @@ -1234,8 +1261,75 @@ func (c actionTests) actionOciOverlay(t *testing.T) { "--overlay", filepath.Join(testDir, "my_rw_ol_dir2"), imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath), }, - requiredCmds: []string{"squashfuse"}, - exitCode: 255, + exitCode: 255, + }, + { + name: "WritableExtfs", + args: []string{"--overlay", writableExtfsImgPath, imageRef, "sh", "-c", "echo my_test_string > /my_test_file"}, + requiredCmds: []string{"fuse2fs", "fuse-overlayfs", "fusermount"}, + exitCode: 0, + }, + { + name: "WritableExtfsRevisit", + args: []string{"--overlay", writableExtfsImgPath, imageRef, "cat", "/my_test_file"}, + requiredCmds: []string{"fuse2fs", "fuse-overlayfs", "fusermount"}, + exitCode: 0, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ExactMatch, "my_test_string"), + }, + }, + { + name: "WritableExtfsRevisitAsRO", + args: []string{"--overlay", writableExtfsImgPath + ":ro", imageRef, "cat", "/my_test_file"}, + requiredCmds: []string{"fuse2fs", "fuse-overlayfs", "fusermount"}, + exitCode: 0, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ExactMatch, "my_test_string"), + }, + }, + { + name: "WritableExtfsWithDirs", + args: []string{ + "--overlay", filepath.Join(testDir, "my_ro_ol_dir0:ro"), + "--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"), + "--overlay", writableExtfsImgPath, + imageRef, "cat", "/my_test_file", + }, + requiredCmds: []string{"fuse2fs", "fuse-overlayfs", "fusermount"}, + exitCode: 0, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ExactMatch, "my_test_string"), + }, + }, + { + name: "WritableExtfsWithMix", + args: []string{ + "--overlay", filepath.Join(testDir, "my_ro_ol_dir0:ro"), + "--overlay", readonlyExtfsImgPath + ":ro", + "--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"), + "--overlay", writableExtfsImgPath, + imageRef, "cat", "/my_test_file", + }, + exitCode: 0, + requiredCmds: []string{"fuse2fs", "fuse-overlayfs", "fusermount"}, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ExactMatch, "my_test_string"), + }, + }, + { + name: "WritableExtfsWithAll", + args: []string{ + "--overlay", filepath.Join(testDir, "my_ro_ol_dir0:ro"), + "--overlay", readonlyExtfsImgPath + ":ro", + "--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"), + "--overlay", writableExtfsImgPath, + imageRef, "cat", "/my_test_file", + }, + exitCode: 0, + requiredCmds: []string{"squashfuse", "fuse2fs", "fuse-overlayfs", "fusermount"}, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ExactMatch, "my_test_string"), + }, }, } @@ -1261,6 +1355,16 @@ func (c actionTests) actionOciOverlay(t *testing.T) { } } +func haveAllCommands(t *testing.T, cmds []string) bool { + for _, c := range cmds { + if _, err := exec.LookPath(c); err != nil { + return false + } + } + + return true +} + // actionOciOverlayTeardown checks that OCI-mode overlays are correctly // unmounted even in root mode (i.e., when user namespaces are not involved). func (c actionTests) actionOciOverlayTeardown(t *testing.T) { @@ -1280,6 +1384,9 @@ func (c actionTests) actionOciOverlayTeardown(t *testing.T) { } }) + dirs.MkdirOrFatal(t, filepath.Join(tmpDir, "upper"), 0o777) + dirs.MkdirOrFatal(t, filepath.Join(tmpDir, "work"), 0o777) + c.env.RunApptainer( t, e2e.WithProfile(e2e.OCIRootProfile), @@ -1314,14 +1421,73 @@ func countLines(path string) (int, error) { return lines, nil } -func haveAllCommands(t *testing.T, cmds []string) bool { - for _, c := range cmds { - if _, err := exec.LookPath(c); err != nil { - return false +// Check that write permissions are indeed available for writable FUSE-mounted +// extfs image overlays. +func (c actionTests) actionOciOverlayExtfsPerms(t *testing.T) { + require.Command(t, "fuse2fs") + require.Command(t, "fuse-overlayfs") + require.Command(t, "fusermount") + + for _, profile := range e2e.OCIProfiles { + // First, create a writable extfs overlay with `apptainer overlay create`. + tmpDir, cleanup := e2e.MakeTempDir(t, c.env.TestDir, "oci_overlay_extfs_perms-", "") + t.Cleanup(func() { + if !t.Failed() { + cleanup(t) + } + }) + + imgPath := filepath.Join(tmpDir, "extfs-perms-test.img") + + c.env.RunApptainer( + t, + e2e.WithProfile(e2e.UserProfile), + e2e.WithCommand("overlay"), + e2e.WithArgs("create", "--size", "64", imgPath), + e2e.ExpectExit(0), + ) + + // Now test whether we can write to, and subsequently read from, the image + // we created. + e2e.EnsureOCIArchive(t, c.env) + imageRef := "oci-archive:" + c.env.OCIArchivePath + + tests := []struct { + name string + args []string + exitCode int + wantOutputs []e2e.ApptainerCmdResultOp + }{ + { + name: "FirstWrite", + args: []string{"--overlay", imgPath, imageRef, "sh", "-c", "echo my_test_string > /my_test_file"}, + exitCode: 0, + }, + { + name: "ThenRead", + args: []string{"--overlay", imgPath, imageRef, "cat", "/my_test_file"}, + exitCode: 0, + wantOutputs: []e2e.ApptainerCmdResultOp{ + e2e.ExpectOutput(e2e.ExactMatch, "my_test_string"), + }, + }, } + t.Run(profile.String(), func(t *testing.T) { + for _, tt := range tests { + c.env.RunApptainer( + t, + e2e.AsSubtest(tt.name), + e2e.WithProfile(profile), + e2e.WithCommand("exec"), + e2e.WithArgs(tt.args...), + e2e.ExpectExit( + tt.exitCode, + tt.wantOutputs..., + ), + ) + } + }) } - - return true } // Make sure --workdir and --scratch work together nicely even when workdir is a diff --git a/internal/pkg/test/tool/dirs/mkdir.go b/internal/pkg/test/tool/dirs/mkdir.go new file mode 100644 index 0000000000..08f1d25059 --- /dev/null +++ b/internal/pkg/test/tool/dirs/mkdir.go @@ -0,0 +1,15 @@ +package dirs + +import ( + "os" + "testing" +) + +func MkdirOrFatal(t *testing.T, dir string, perm os.FileMode) { + if err := os.Mkdir(dir, perm); err != nil { + t.Fatalf("could not create %q: %s", dir, err) + } + if err := os.Chmod(dir, perm); err != nil { + t.Fatalf("could not chmod %q to %o: %s", dir, perm, err) + } +} diff --git a/internal/pkg/util/fs/overlay/overlay_item_linux.go b/internal/pkg/util/fs/overlay/overlay_item_linux.go index b3097ddaa7..4eabc08eb4 100644 --- a/internal/pkg/util/fs/overlay/overlay_item_linux.go +++ b/internal/pkg/util/fs/overlay/overlay_item_linux.go @@ -18,6 +18,7 @@ import ( "syscall" "github.com/apptainer/apptainer/internal/pkg/util/bin" + "github.com/apptainer/apptainer/internal/pkg/util/fs" "github.com/apptainer/apptainer/pkg/image" "github.com/apptainer/apptainer/pkg/sylog" ) @@ -132,25 +133,51 @@ func (i *Item) GetParentDir() (string, error) { // this method does not mount the assembled overlay itself. That happens in // Set.Mount(). func (i *Item) Mount() error { - if i.Writable { - if err := i.prepareWritableOverlay(); err != nil { - return err - } - } - + var err error switch i.Type { case image.SANDBOX: - return i.mountDir() + err = i.mountDir() case image.SQUASHFS: - return i.mountWithFuse("squashfuse") + err = i.mountWithFuse("squashfuse") case image.EXT3: if i.Writable { - return fmt.Errorf("mounting writable extfs images is not currently supported, please use :ro suffix on image specification for read-only mode") + err = i.mountWithFuse("fuse2fs", "-o", "rw") + } else { + err = i.mountWithFuse("fuse2fs", "-o", "ro") } - return i.mountWithFuse("fuse2fs", "-o", "ro") default: return fmt.Errorf("internal error: unrecognized image type in overlay.Item.Mount() (type: %v)", i.Type) } + + if err != nil { + return err + } + + if i.Writable { + return i.prepareWritableOverlay() + } + + return nil +} + +// GetMountDir returns the path to the directory that will actually be mounted +// for this overlay. For squashfs overlays, this is equivalent to the +// Item.StagingDir field. But for all other overlays, it is the "upper" +// subdirectory of Item.StagingDir. +func (i Item) GetMountDir() string { + switch i.Type { + case image.SQUASHFS: + return i.StagingDir + + case image.SANDBOX: + if i.Writable || fs.IsDir(i.Upper()) { + return i.Upper() + } + return i.StagingDir + + default: + return i.Upper() + } } // mountDir mounts directory-based Items. This involves bind-mounting followed @@ -226,10 +253,15 @@ func (i *Item) mountWithFuse(fuseMountTool string, additionalArgs ...string) err } }() - args := make([]string, 0, len(additionalArgs)+2) + args := make([]string, 0, len(additionalArgs)+4) + + // TODO: Think through what makes sense for file ownership in FUSE-mounted + // images, vis a vis id-mappings and user-namespaces. + args = append(args, "-o") + args = append(args, "uid=0,gid=0") + args = append(args, i.SourcePath) args = append(args, fuseMountDir) - args = append(args, additionalArgs...) sylog.Debugf("Executing FUSE mount command: %s %s", fuseMountCmd, strings.Join(args, " ")) execCmd := exec.Command(fuseMountCmd, args...) execCmd.Stderr = os.Stderr @@ -281,15 +313,19 @@ func (i *Item) prepareWritableOverlay() error { switch i.Type { case image.SANDBOX: i.StagingDir = i.SourcePath + fallthrough + case image.EXT3: if err := EnsureOverlayDir(i.StagingDir, true, 0o755); err != nil { return err } sylog.Debugf("Ensuring %q exists", i.Upper()) if err := EnsureOverlayDir(i.Upper(), true, 0o755); err != nil { + sylog.Errorf("Could not create overlay upper dir. If using an overlay image ensure it contains 'upper' and 'work' directories") return fmt.Errorf("err encountered while preparing upper subdir of overlay dir %q: %w", i.Upper(), err) } sylog.Debugf("Ensuring %q exists", i.Work()) if err := EnsureOverlayDir(i.Work(), true, 0o700); err != nil { + sylog.Errorf("Could not create overlay work dir. If using an overlay image ensure it contains 'upper' and 'work' directories") return fmt.Errorf("err encountered while preparing work subdir of overlay dir %q: %w", i.Work(), err) } default: diff --git a/internal/pkg/util/fs/overlay/overlay_item_linux_test.go b/internal/pkg/util/fs/overlay/overlay_item_linux_test.go index bd49f1c7d4..bd7320bdf4 100644 --- a/internal/pkg/util/fs/overlay/overlay_item_linux_test.go +++ b/internal/pkg/util/fs/overlay/overlay_item_linux_test.go @@ -15,7 +15,9 @@ import ( "strings" "testing" + "github.com/apptainer/apptainer/internal/pkg/test/tool/dirs" "github.com/apptainer/apptainer/internal/pkg/test/tool/require" + "github.com/apptainer/apptainer/internal/pkg/util/fs" "github.com/apptainer/apptainer/pkg/image" ) @@ -25,8 +27,14 @@ const ( extfsTestString string = "extfs-test-string\n" ) +var ( + imgsPath = filepath.Join("..", "..", "..", "..", "..", "test", "images") + squashfsImgPath = filepath.Join(imgsPath, "squashfs-for-overlay.img") + extfsImgPath = filepath.Join(imgsPath, "extfs-for-overlay.img") +) + func mkTempDirOrFatal(t *testing.T) string { - tmpDir, err := os.MkdirTemp("", "testoverlayitem-") + tmpDir, err := os.MkdirTemp(t.TempDir(), "testoverlayitem-") if err != nil { t.Fatalf("failed to create temporary dir: %s", err) } @@ -39,10 +47,18 @@ func mkTempDirOrFatal(t *testing.T) string { return tmpDir } +func mkTempOlDirOrFatal(t *testing.T) string { + tmpOlDir := mkTempDirOrFatal(t) + dirs.MkdirOrFatal(t, filepath.Join(tmpOlDir, "upper"), 0o777) + dirs.MkdirOrFatal(t, filepath.Join(tmpOlDir, "lower"), 0o777) + + return tmpOlDir +} + func TestItemWritableField(t *testing.T) { - tmpDir := mkTempDirOrFatal(t) - rwOverlayStr := tmpDir - roOverlayStr := tmpDir + ":ro" + tmpOlDir := mkTempOlDirOrFatal(t) + rwOverlayStr := tmpOlDir + roOverlayStr := tmpOlDir + ":ro" rwItem, err := NewItemFromString(rwOverlayStr) if err != nil { @@ -87,9 +103,9 @@ func verifyAutoParentDir(t *testing.T, item *Item) { } func TestAutofillParentDir(t *testing.T) { - tmpDir := mkTempDirOrFatal(t) - rwOverlayStr := tmpDir - roOverlayStr := tmpDir + ":ro" + tmpOlDir := mkTempOlDirOrFatal(t) + rwOverlayStr := tmpOlDir + roOverlayStr := tmpOlDir + ":ro" rwItem, err := NewItemFromString(rwOverlayStr) if err != nil { @@ -114,9 +130,9 @@ func verifyExplicitParentDir(t *testing.T, item *Item, dir string) { } func TestExplicitParentDir(t *testing.T) { - tmpDir := mkTempDirOrFatal(t) - rwOverlayStr := tmpDir - roOverlayStr := tmpDir + ":ro" + tmpOlDir := mkTempOlDirOrFatal(t) + rwOverlayStr := tmpOlDir + roOverlayStr := tmpOlDir + ":ro" rwItem, err := NewItemFromString(rwOverlayStr) if err != nil { @@ -189,8 +205,8 @@ func dirMountUnmount(t *testing.T, olStr string) { } func TestDirMounts(t *testing.T) { - dirMountUnmount(t, mkTempDirOrFatal(t)+":ro") - dirMountUnmount(t, mkTempDirOrFatal(t)) + dirMountUnmount(t, mkTempOlDirOrFatal(t)+":ro") + dirMountUnmount(t, mkTempOlDirOrFatal(t)) } func tryImageRO(t *testing.T, olStr string, typeCode int, typeStr, expectStr string) { @@ -210,25 +226,75 @@ func tryImageRO(t *testing.T, olStr string, typeCode int, typeStr, expectStr str item.Unmount() }) - testFileStagedPath := filepath.Join(item.StagingDir, testFilePath) - data, err := os.ReadFile(testFileStagedPath) - if err != nil { - t.Fatalf("error while trying to read from file %q: %s", testFileStagedPath, err) - } - foundStr := string(data) - if foundStr != expectStr { - t.Errorf("incorrect file contents in %s img: expected %q, but found: %q", typeStr, expectStr, foundStr) - } + testFileStagedPath := filepath.Join(item.GetMountDir(), testFilePath) + checkForStringInOverlay(t, typeStr, testFileStagedPath, expectStr) } func TestSquashfsRO(t *testing.T) { require.Command(t, "squashfuse") require.Command(t, "fusermount") - tryImageRO(t, filepath.Join("..", "..", "..", "..", "..", "test", "images", "squashfs-for-overlay.img"), image.SQUASHFS, "squashfs", squashfsTestString) + tryImageRO(t, squashfsImgPath, image.SQUASHFS, "squashfs", squashfsTestString) } func TestExtfsRO(t *testing.T) { require.Command(t, "fuse2fs") require.Command(t, "fusermount") - tryImageRO(t, filepath.Join("..", "..", "..", "..", "..", "test", "images", "extfs-for-overlay.img")+":ro", image.EXT3, "extfs", extfsTestString) + tmpDir := mkTempDirOrFatal(t) + readonlyExtfsImgPath := filepath.Join(tmpDir, "readonly-extfs.img") + if err := fs.CopyFile(extfsImgPath, readonlyExtfsImgPath, 0o444); err != nil { + t.Fatalf("could not copy %q to %q: %s", extfsImgPath, readonlyExtfsImgPath, err) + } + tryImageRO(t, readonlyExtfsImgPath+":ro", image.EXT3, "extfs", extfsTestString) +} + +func TestExtfsRW(t *testing.T) { + require.Command(t, "fuse2fs") + require.Command(t, "fuse-overlayfs") + require.Command(t, "fusermount") + tmpDir := mkTempDirOrFatal(t) + + // Create a copy of the extfs test image to be used for testing writable + // extfs image overlays + writableExtfsImgPath := filepath.Join(tmpDir, "writable-extfs.img") + err := fs.CopyFile(extfsImgPath, writableExtfsImgPath, 0o755) + if err != nil { + t.Fatalf("could not copy %q to %q: %s", extfsImgPath, writableExtfsImgPath, err) + } + + item, err := NewItemFromString(writableExtfsImgPath) + if err != nil { + t.Fatalf("failed to mount extfs image at %q: %s", writableExtfsImgPath, err) + } + + if item.Type != image.EXT3 { + t.Errorf("item.Type is %v (should be %v)", item.Type, image.EXT3) + } + + if err := item.Mount(); err != nil { + t.Fatalf("unable to mount extfs image for reading & writing: %s", err) + } + t.Cleanup(func() { + item.Unmount() + }) + + testFileStagedPath := filepath.Join(item.GetMountDir(), testFilePath) + checkForStringInOverlay(t, "extfs", testFileStagedPath, extfsTestString) + otherTestFileStagedPath := item.GetMountDir() + "_other" + otherExtfsTestString := "another string" + err = os.WriteFile(otherTestFileStagedPath, []byte(otherExtfsTestString), 0o755) + if err != nil { + t.Errorf("could not write to file %q in extfs image %q: %s", otherTestFileStagedPath, writableExtfsImgPath, err) + } + checkForStringInOverlay(t, "extfs", otherTestFileStagedPath, otherExtfsTestString) +} + +func checkForStringInOverlay(t *testing.T, typeStr, stagedPath, expectStr string) { + data, err := os.ReadFile(stagedPath) + if err != nil { + t.Fatalf("error while trying to read from file %q: %s", stagedPath, err) + } + foundStr := string(data) + if foundStr != expectStr { + t.Errorf("incorrect file contents in %s img: expected %q, but found: %q", typeStr, expectStr, foundStr) + } } diff --git a/internal/pkg/util/fs/overlay/overlay_linux.go b/internal/pkg/util/fs/overlay/overlay_linux.go index cbe6336a65..5da3d3c678 100644 --- a/internal/pkg/util/fs/overlay/overlay_linux.go +++ b/internal/pkg/util/fs/overlay/overlay_linux.go @@ -35,7 +35,7 @@ const ( fuseDir ) -type fs struct { +type filesys struct { name string overlayDir dir } @@ -49,7 +49,7 @@ const ( Panfs int64 = 0xAAD7AAEA ) -var incompatibleFs = map[int64]fs{ +var incompatibleFilesys = map[int64]filesys{ // NFS filesystem Nfs: { name: "NFS", @@ -90,7 +90,7 @@ func check(path string, d dir) error { return fmt.Errorf("could not retrieve underlying filesystem information for %s: %w", path, err) } - fs, ok := incompatibleFs[int64(stfs.Type)] + fs, ok := incompatibleFilesys[int64(stfs.Type)] if !ok || (ok && fs.overlayDir&d == 0) { return nil } diff --git a/internal/pkg/util/fs/overlay/overlay_set_linux.go b/internal/pkg/util/fs/overlay/overlay_set_linux.go index dc50ea389a..630e3257f1 100644 --- a/internal/pkg/util/fs/overlay/overlay_set_linux.go +++ b/internal/pkg/util/fs/overlay/overlay_set_linux.go @@ -17,6 +17,7 @@ import ( "syscall" "github.com/apptainer/apptainer/internal/pkg/util/bin" + "github.com/apptainer/apptainer/pkg/image" "github.com/apptainer/apptainer/pkg/sylog" "github.com/samber/lo" ) @@ -70,7 +71,8 @@ func (s Set) Unmount(rootFsDir string) error { return fmt.Errorf("while checking for unprivileged overlay support in kernel: %w", err) } - if unprivOls { + useKernelMount := unprivOls && !s.hasWritableExtfsImg() + if useKernelMount { err = DetachMount(rootFsDir) } else { err = UnmountWithFuse(rootFsDir) @@ -112,22 +114,25 @@ func (s Set) performFinalMount(rootFsDir string) error { return fmt.Errorf("while checking for unprivileged overlay support in kernel: %w", err) } - if unprivOls { - sylog.Debugf("Mounting overlay (via syscall) with rootFsDir %q, options: %q", rootFsDir, options) - if err := syscall.Mount("overlay", rootFsDir, "overlay", syscall.MS_NODEV, options); err != nil { + useKernelMount := unprivOls && !s.hasWritableExtfsImg() + + if useKernelMount { + flags := uintptr(syscall.MS_NODEV) + sylog.Debugf("Mounting overlay (via syscall) with rootFsDir %q, options: %q, mount flags: %#v", rootFsDir, options, flags) + if err := syscall.Mount("overlay", rootFsDir, "overlay", flags, options); err != nil { return fmt.Errorf("failed to mount %s: %w", rootFsDir, err) } } else { fuseOlFsCmd, err := bin.FindBin("fuse-overlayfs") if err != nil { - return fmt.Errorf("kernel does not support unprivileged overlays, and fuse-overlayfs not available: %w", err) + return fmt.Errorf("'fuse-overlayfs' must be used for this overlay specification, but is not available: %w", err) } // Even though fusermount is not needed for this step, we shouldn't perform // the mount unless we have the necessary tools to eventually unmount it _, err = bin.FindBin("fusermount") if err != nil { - return fmt.Errorf("kernel does not support unprivileged overlays, and using fuse-overlayfs fallback requires fusermount to be installed: %w", err) + return fmt.Errorf("'fuse-overlayfs' must be used for this overlay specification, and this also requires 'fusermount' to be installed: %w", err) } sylog.Debugf("Mounting overlay (via fuse-overlayfs) with rootFsDir %q, options: %q", rootFsDir, options) @@ -148,7 +153,7 @@ func (s Set) performFinalMount(rootFsDir string) error { func (s Set) options(rootFsDir string) string { // Create lowerdir argument of options string lowerDirs := lo.Map(s.ReadonlyOverlays, func(o *Item, _ int) string { - return o.StagingDir + return o.GetMountDir() }) lowerDirJoined := strings.Join(append(lowerDirs, rootFsDir), ":") @@ -160,6 +165,14 @@ func (s Set) options(rootFsDir string) string { lowerDirJoined, s.WritableOverlay.Upper(), s.WritableOverlay.Work()) } +func (s Set) hasWritableExtfsImg() bool { + if (s.WritableOverlay != nil) && (s.WritableOverlay.Type == image.EXT3) { + return true + } + + return false +} + // detachIndividualMounts detaches the bind mounts & remounts created by // performIndividualMounts, above. func (s Set) detachIndividualMounts() error { diff --git a/internal/pkg/util/fs/overlay/overlay_set_linux_test.go b/internal/pkg/util/fs/overlay/overlay_set_linux_test.go index 93cbae1e50..6dc114e374 100644 --- a/internal/pkg/util/fs/overlay/overlay_set_linux_test.go +++ b/internal/pkg/util/fs/overlay/overlay_set_linux_test.go @@ -16,6 +16,7 @@ import ( "testing" "github.com/apptainer/apptainer/internal/pkg/test/tool/require" + "github.com/apptainer/apptainer/internal/pkg/util/fs" ) func addROItemOrFatal(t *testing.T, s *Set, olStr string) *Item { @@ -60,25 +61,30 @@ func TestAllTypesAtOnce(t *testing.T) { wrapOverlayTest(func(t *testing.T) { s := Set{} - tmpRODir := mkTempDirOrFatal(t) - addROItemOrFatal(t, &s, tmpRODir+":ro") + tmpRoOlDir := mkTempOlDirOrFatal(t) + addROItemOrFatal(t, &s, tmpRoOlDir+":ro") squashfsSupported := false if _, err := exec.LookPath("squashfs"); err == nil { squashfsSupported = true - addROItemOrFatal(t, &s, filepath.Join("..", "..", "..", "..", "..", "test", "images", "squashfs-for-overlay.img")) + addROItemOrFatal(t, &s, squashfsImgPath) } extfsSupported := false if _, err := exec.LookPath("fuse2fs"); err == nil { extfsSupported = true - addROItemOrFatal(t, &s, filepath.Join("..", "..", "..", "..", "..", "test", "images", "extfs-for-overlay.img")+":ro") + tmpDir := mkTempDirOrFatal(t) + readonlyExtfsImgPath := filepath.Join(tmpDir, "readonly-extfs.img") + if err := fs.CopyFile(extfsImgPath, readonlyExtfsImgPath, 0o444); err != nil { + t.Fatalf("could not copy %q to %q: %s", extfsImgPath, readonlyExtfsImgPath, err) + } + addROItemOrFatal(t, &s, readonlyExtfsImgPath+":ro") } - tmpRWDir := mkTempDirOrFatal(t) - i, err := NewItemFromString(tmpRWDir) + tmpRwOlDir := mkTempOlDirOrFatal(t) + i, err := NewItemFromString(tmpRwOlDir) if err != nil { - t.Fatalf("failed to create writable-dir overlay item (%q): %s", tmpRWDir, err) + t.Fatalf("failed to create writable-dir overlay item (%q): %s", tmpRwOlDir, err) } s.WritableOverlay = i @@ -119,57 +125,84 @@ func TestAllTypesAtOnce(t *testing.T) { func TestPersistentWriteToDir(t *testing.T) { wrapOverlayTest(func(t *testing.T) { - tmpRWDir := mkTempDirOrFatal(t) - i, err := NewItemFromString(tmpRWDir) + tmpRwOlDir := mkTempOlDirOrFatal(t) + i, err := NewItemFromString(tmpRwOlDir) if err != nil { - t.Fatalf("failed to create writable-dir overlay item (%q): %s", tmpRWDir, err) + t.Fatalf("failed to create writable-dir overlay item (%q): %s", tmpRwOlDir, err) } s := Set{WritableOverlay: i} - rootfsDir := mkTempDirOrFatal(t) + performPersistentWriteTest(t, s) + })(t) +} - // This cleanup will serve adequately for both iterations of the overlay-set - // mounting, below. If it happens to get called while the set is not - // mounted, it should fail silently. - t.Cleanup(func() { - if t.Failed() { - s.Unmount(rootfsDir) - } - }) +func TestPersistentWriteToExtfsImg(t *testing.T) { + require.Command(t, "fuse2fs") + require.Command(t, "fuse-overlayfs") + require.Command(t, "fusermount") + tmpDir := mkTempDirOrFatal(t) - // Mount the overlay set, write a string to a file, and unmount. - if err := s.Mount(rootfsDir); err != nil { - t.Fatalf("failed to mount overlay set: %s", err) - } - expectStr := "my_test_string" - bytes := []byte(expectStr) - testFilePath := "my_test_file" - testFileMountedPath := filepath.Join(rootfsDir, testFilePath) - if err := os.WriteFile(testFileMountedPath, bytes, 0o644); err != nil { - t.Fatalf("while trying to write file inside mounted overlay-set: %s", err) - } + // Create a copy of the extfs test image to be used for testing writable + // extfs image overlays + writableExtfsImgPath := filepath.Join(tmpDir, "writable-extfs.img") + err := fs.CopyFile(extfsImgPath, writableExtfsImgPath, 0o755) + if err != nil { + t.Fatalf("could not copy %q to %q: %s", extfsImgPath, writableExtfsImgPath, err) + } - if err := s.Unmount(rootfsDir); err != nil { - t.Fatalf("while trying to unmount overlay set: %s", err) - } + i, err := NewItemFromString(writableExtfsImgPath) + if err != nil { + t.Fatalf("failed to create writable-dir overlay item (%q): %s", writableExtfsImgPath, err) + } + s := Set{WritableOverlay: i} - // Mount the same set again, and check that we see the file with the - // expected contents. - if err := s.Mount(rootfsDir); err != nil { - t.Fatalf("failed to mount overlay set: %s", err) - } - data, err := os.ReadFile(testFileMountedPath) - if err != nil { - t.Fatalf("error while trying to read from file %q: %s", testFileMountedPath, err) - } - foundStr := string(data) - if foundStr != expectStr { - t.Errorf("incorrect file contents in mounted overlay set: expected %q, but found: %q", expectStr, foundStr) - } - if err := s.Unmount(rootfsDir); err != nil { - t.Errorf("while trying to unmount overlay set: %s", err) + performPersistentWriteTest(t, s) +} + +func performPersistentWriteTest(t *testing.T, s Set) { + rootfsDir := mkTempDirOrFatal(t) + + // This cleanup will serve adequately for both iterations of the overlay-set + // mounting, below. If it happens to get called while the set is not + // mounted, it should fail silently. + // Mount the overlay set, write a string to a file, and unmount. + // Mount the same set again, and check that we see the file with the + // expected contents. + t.Cleanup(func() { + if t.Failed() { + s.Unmount(rootfsDir) } - })(t) + }) + + if err := s.Mount(rootfsDir); err != nil { + t.Fatalf("failed to mount overlay set: %s", err) + } + expectStr := "my_test_string" + bytes := []byte(expectStr) + testFilePath := "my_test_file" + testFileMountedPath := filepath.Join(rootfsDir, testFilePath) + if err := os.WriteFile(testFileMountedPath, bytes, 0o644); err != nil { + t.Fatalf("while trying to write file inside mounted overlay-set: %s", err) + } + + if err := s.Unmount(rootfsDir); err != nil { + t.Fatalf("while trying to unmount overlay set: %s", err) + } + + if err := s.Mount(rootfsDir); err != nil { + t.Fatalf("failed to mount overlay set: %s", err) + } + data, err := os.ReadFile(testFileMountedPath) + if err != nil { + t.Fatalf("error while trying to read from file %q: %s", testFileMountedPath, err) + } + foundStr := string(data) + if foundStr != expectStr { + t.Errorf("incorrect file contents in mounted overlay set: expected %q, but found: %q", expectStr, foundStr) + } + if err := s.Unmount(rootfsDir); err != nil { + t.Errorf("while trying to unmount overlay set: %s", err) + } } func TestDuplicateItemsInSet(t *testing.T) { @@ -182,11 +215,11 @@ func TestDuplicateItemsInSet(t *testing.T) { // First, test mounting of an overlay set with only readonly items, one of // which is a duplicate of another. - addROItemOrFatal(t, &s, mkTempDirOrFatal(t)+":ro") - roI2 := addROItemOrFatal(t, &s, mkTempDirOrFatal(t)+":ro") - addROItemOrFatal(t, &s, mkTempDirOrFatal(t)+":ro") + addROItemOrFatal(t, &s, mkTempOlDirOrFatal(t)+":ro") + roI2 := addROItemOrFatal(t, &s, mkTempOlDirOrFatal(t)+":ro") + addROItemOrFatal(t, &s, mkTempOlDirOrFatal(t)+":ro") addROItemOrFatal(t, &s, roI2.SourcePath+":ro") - addROItemOrFatal(t, &s, mkTempDirOrFatal(t)+":ro") + addROItemOrFatal(t, &s, mkTempOlDirOrFatal(t)+":ro") rootfsDir = mkTempDirOrFatal(t) if err := s.Mount(rootfsDir); err == nil { @@ -198,10 +231,10 @@ func TestDuplicateItemsInSet(t *testing.T) { // Next, test mounting of an overlay set with a writable item as well as // several readonly items, one of which is a duplicate of another. - tmpRWDir := mkTempDirOrFatal(t) - rwI, err = NewItemFromString(tmpRWDir) + tmpRwOlDir := mkTempOlDirOrFatal(t) + rwI, err = NewItemFromString(tmpRwOlDir) if err != nil { - t.Fatalf("failed to create writable-dir overlay item (%q): %s", tmpRWDir, err) + t.Fatalf("failed to create writable-dir overlay item (%q): %s", tmpRwOlDir, err) } s.WritableOverlay = rwI diff --git a/test/images/extfs-for-overlay.img b/test/images/extfs-for-overlay.img index 7edacbe7a7faed0f4ac00e70c32427528abe3731..82003e2b72d64e263c6008a9ced05da6f1791dfd 100644 GIT binary patch delta 786 zcmaKqL1@!Z7{}k2_o8u4w{*>HwX^MIpdu`jZqtf%;z24hD8eS_pyHIF1xH~Fl!Vd)i@MM7-%`P*50bm)+v`nq^D{1HZiFz4!lr|L-NXZG(-h z=uCY@3E6gIER~q6+mnk3CBA=s3kVQGX5QQGW!7huU*Gv@UMVgu7Ix^fmHJBu#@r2- z&YbUme?lDi)oLtiVqG{J^!Raq2NH;%bsab z9XC#fIPmL~e+ku$5F5jcA)`p!k*Guo9tfr;*Blsk?Ez+->(|x|wzIHybFfz$VXX?Q z4QV~0s~iI%5W|6jduu4i~|?YL76#_XUzj{=I+Jypb{aW z+w$WU?Z(M4i5%sg7KME`d3LwkZa400SNF1+-=6N4Bwsc*OHw6AJix7tm(_m!_jW%1 z+yX0U5LKC(p1xZ}`zX?+OqqRHeXt5^w4MVi_s`!mKQaY~f>4AZDsl)pjA+Oaqyvc{ zIub=XkuJn&j7-Ho#zB7p79`Y8*=A$Fg3r`3Oo%oh2R;=%CV02taluW&djvl!ctY@` V;Ku|%E_h1tUcpZYo_@nK{5L5M_xk_< delta 582 zcmaLUPbdUY90&0Cn|Zs2UF25Y((Btn0Tbp0_4$WSH(!T35>xv}%NvQ$ z&?#WZi8=XT)^gZD#Vg*3V(Hp^02T$zS-Dn4g-QWFE71O+I`LepNWfFub3cY@KA#%G znv`!yRH76%`PoJUOqxa~bv?+qn5Qs}auQ?ryQu$FpI@_f= zew;I_x)$=KeMl?W_^(>LP{Wvadx;P?HZ4y0Jtdglo+RBJO+N~#lLJ<-#=9fZ-Fq-1 z&&d^seDPZW?dIQCw`VO3VYoVM%IKHsK@`H#KmTH-Zw8QpR78;t*-;77P$?=y4&+4T zr~*|Y*J0m`ep1Dm&Y7Dt+*EVs;jD(UTF&Y?tLLnNGcRY2oHcRg+hP99RzTCf07I6k A+yDRo From d4fdf44220568115e2398c29fe736a10b2a17207 Mon Sep 17 00:00:00 2001 From: Dave Dykstra <2129743+DrDaveD@users.noreply.github.com> Date: Thu, 6 Jul 2023 14:07:33 -0500 Subject: [PATCH 114/114] do not re-use the rootfs name for overlay upper dir Signed-off-by: Dave Dykstra <2129743+DrDaveD@users.noreply.github.com> --- pkg/ocibundle/tools/overlay_linux.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/pkg/ocibundle/tools/overlay_linux.go b/pkg/ocibundle/tools/overlay_linux.go index f27be60ae4..ffaf715d73 100644 --- a/pkg/ocibundle/tools/overlay_linux.go +++ b/pkg/ocibundle/tools/overlay_linux.go @@ -45,16 +45,21 @@ func CreateOverlay(bundlePath string) error { Writable: true, }} - return olSet.Mount(RootFs(bundlePath).Path()) + upDir := filepath.Join(bundlePath, "upper") + if err := os.MkdirAll(upDir, 0o755); err != nil { + return fmt.Errorf("failed to create %s: %s", upDir, err) + } + + return olSet.Mount(filepath.Join(upDir)) } // DeleteOverlay deletes an overlay previously created using a directory inside // the OCI bundle. func DeleteOverlay(bundlePath string) error { olDir := filepath.Join(bundlePath, "overlay") - rootFsDir := RootFs(bundlePath).Path() + upDir := filepath.Join(bundlePath, "upper") - if err := overlay.DetachMount(rootFsDir); err != nil { + if err := overlay.DetachMount(upDir); err != nil { return err } @@ -100,7 +105,7 @@ func CreateOverlayTmpfs(bundlePath string, sizeMiB int) (string, error) { Writable: true, }} - err = olSet.Mount(RootFs(bundlePath).Path()) + err = olSet.Mount(filepath.Join(bundlePath, "upper")) if err != nil { return "", err } @@ -110,9 +115,9 @@ func CreateOverlayTmpfs(bundlePath string, sizeMiB int) (string, error) { // DeleteOverlayTmpfs deletes an overlay previously created using tmpfs. func DeleteOverlayTmpfs(bundlePath, olDir string) error { - rootFsDir := RootFs(bundlePath).Path() + upDir := filepath.Join(bundlePath, "upper") - if err := overlay.DetachMount(rootFsDir); err != nil { + if err := overlay.DetachMount(upDir); err != nil { return err }