Add the CreateRuntime, CreateContainer and StartContainer Hooks

Signed-off-by: Renaud Gaubert <rgaubert@nvidia.com>

libcontainer/configs/config.go

@@ -8,7 +8,7 @@ import (
 	"time"
 
 	"github.com/opencontainers/runtime-spec/specs-go"
-
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 )
 
@@ -176,7 +176,7 @@ type Config struct {
 
 	// Hooks are a collection of actions to perform at various container lifecycle events.
 	// CommandHooks are serialized to JSON, but other hooks are not.
-	Hooks *Hooks
+	Hooks Hooks
 
 	// Version is the version of opencontainer specification that is supported.
 	Version string `json:"version"`
@@ -203,17 +203,54 @@ type Config struct {
 	RootlessCgroups bool `json:"rootless_cgroups,omitempty"`
 }
 
-type Hooks struct {
+type HookName string
+type HookList []Hook
+type Hooks map[HookName]HookList
+
+const (
 	// Prestart commands are executed after the container namespaces are created,
 	// but before the user supplied command is executed from init.
-	Prestart []Hook
+	// Note: This hook is now deprecated
+	// Prestart commands are called in the Runtime namespace.
+	Prestart HookName = "prestart"
+
+	// CreateRuntime commands MUST be called as part of the create operation after
+	// the runtime environment has been created but before the pivot_root has been executed.
+	// CreateRuntime is called immediately after the deprecated Prestart hook.
+	// CreateRuntime commands are called in the Runtime Namespace.
+	CreateRuntime = "createRuntime"
+
+	// CreateContainer commands MUST be called as part of the create operation after
+	// the runtime environment has been created but before the pivot_root has been executed.
+	// CreateContainer commands are called in the Container namespace.
+	CreateContainer = "createContainer"
+
+	// StartContainer commands MUST be called as part of the start operation and before
+	// the container process is started.
+	// StartContainer commands are called in the Container namespace.
+	StartContainer = "startContainer"
 
 	// Poststart commands are executed after the container init process starts.
-	Poststart []Hook
+	// Poststart commands are called in the Runtime Namespace.
+	Poststart = "poststart"
 
 	// Poststop commands are executed after the container init process exits.
-	Poststop []Hook
-}
+	// Poststop commands are called in the Runtime Namespace.
+	Poststop = "poststop"
+)
+
+var (
+	HookNameList = []HookName{Prestart, CreateRuntime, CreateContainer, StartContainer, Poststart, Poststop}
+)
+
+// TODO move this to runtime-spec
+// See: https://github.com/opencontainers/runtime-spec/pull/1046
+const (
+	Creating = "creating"
+	Created  = "created"
+	Running  = "running"
+	Stopped  = "stopped"
+)
 
 type Capabilities struct {
 	// Bounding is the set of capabilities checked by the kernel.
@@ -228,32 +265,39 @@ type Capabilities struct {
 	Ambient []string
 }
 
-func (hooks *Hooks) UnmarshalJSON(b []byte) error {
-	var state struct {
-		Prestart  []CommandHook
-		Poststart []CommandHook
-		Poststop  []CommandHook
+func (hooks HookList) RunHooks(state *specs.State) error {
+	for i, h := range hooks {
+		if err := h.Run(state); err != nil {
+			return errors.Wrapf(err, "Running hook #%d:", i)
+		}
 	}
 
+	return nil
+}
+
+func (hooks *Hooks) UnmarshalJSON(b []byte) error {
+	var state map[HookName][]CommandHook
+
 	if err := json.Unmarshal(b, &state); err != nil {
 		return err
 	}
 
-	deserialize := func(shooks []CommandHook) (hooks []Hook) {
-		for _, shook := range shooks {
-			hooks = append(hooks, shook)
+	*hooks = Hooks{}
+	for n, commandHooks := range state {
+		if len(commandHooks) == 0 {
+			continue
 		}
 
-		return hooks
+		(*hooks)[n] = HookList{}
+		for _, h := range commandHooks {
+			(*hooks)[n] = append((*hooks)[n], h)
+		}
 	}
 
-	hooks.Prestart = deserialize(state.Prestart)
-	hooks.Poststart = deserialize(state.Poststart)
-	hooks.Poststop = deserialize(state.Poststop)
 	return nil
 }
 
-func (hooks Hooks) MarshalJSON() ([]byte, error) {
+func (hooks *Hooks) MarshalJSON() ([]byte, error) {
 	serialize := func(hooks []Hook) (serializableHooks []CommandHook) {
 		for _, hook := range hooks {
 			switch chook := hook.(type) {
@@ -268,9 +312,12 @@ func (hooks Hooks) MarshalJSON() ([]byte, error) {
 	}
 
 	return json.Marshal(map[string]interface{}{
-		"prestart":  serialize(hooks.Prestart),
-		"poststart": serialize(hooks.Poststart),
-		"poststop":  serialize(hooks.Poststop),
+		"prestart":        serialize((*hooks)[Prestart]),
+		"createRuntime":   serialize((*hooks)[CreateRuntime]),
+		"createContainer": serialize((*hooks)[CreateContainer]),
+		"startContainer":  serialize((*hooks)[StartContainer]),
+		"poststart":       serialize((*hooks)[Poststart]),
+		"poststop":        serialize((*hooks)[Poststop]),
 	})
 }
 

libcontainer/container_linux.go

@@ -29,6 +29,7 @@ import (
 	"github.com/checkpoint-restore/go-criu/v4"
 	criurpc "github.com/checkpoint-restore/go-criu/v4/rpc"
 	"github.com/golang/protobuf/proto"
+	errorsf "github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/vishvananda/netlink/nl"
 	"golang.org/x/sys/unix"
@@ -379,13 +380,12 @@ func (c *linuxContainer) start(process *Process) error {
 			if err != nil {
 				return err
 			}
-			for i, hook := range c.config.Hooks.Poststart {
-				if err := hook.Run(s); err != nil {
-					if err := ignoreTerminateErrors(parent.terminate()); err != nil {
-						logrus.Warn(err)
-					}
-					return newSystemErrorWithCausef(err, "running poststart hook %d", i)
+
+			if err := c.config.Hooks[configs.Poststart].RunHooks(s); err != nil {
+				if err := ignoreTerminateErrors(parent.terminate()); err != nil {
+					logrus.Warn(errorsf.Wrapf(err, "Running Poststart hook"))
 				}
+				return err
 			}
 		}
 	}
@@ -1621,10 +1621,12 @@ func (c *linuxContainer) criuNotifications(resp *criurpc.CriuResp, process *Proc
 				return nil
 			}
 			s.Pid = int(notify.GetPid())
-			for i, hook := range c.config.Hooks.Prestart {
-				if err := hook.Run(s); err != nil {
-					return newSystemErrorWithCausef(err, "running prestart hook %d", i)
-				}
+
+			if err := c.config.Hooks[configs.Prestart].RunHooks(s); err != nil {
+				return err
+			}
+			if err := c.config.Hooks[configs.CreateRuntime].RunHooks(s); err != nil {
+				return err
 			}
 		}
 	case "post-restore":

libcontainer/factory_linux_test.go

@@ -157,14 +157,14 @@ func TestFactoryLoadContainer(t *testing.T) {
 	// setup default container config and state for mocking
 	var (
 		id            = "1"
-		expectedHooks = &configs.Hooks{
-			Prestart: []configs.Hook{
+		expectedHooks = configs.Hooks{
+			configs.Prestart: configs.HookList{
 				configs.CommandHook{Command: configs.Command{Path: "prestart-hook"}},
 			},
-			Poststart: []configs.Hook{
+			configs.Poststart: configs.HookList{
 				configs.CommandHook{Command: configs.Command{Path: "poststart-hook"}},
 			},
-			Poststop: []configs.Hook{
+			configs.Poststop: configs.HookList{
 				unserializableHook{},
 				configs.CommandHook{Command: configs.Command{Path: "poststop-hook"}},
 			},
@@ -201,7 +201,7 @@ func TestFactoryLoadContainer(t *testing.T) {
 	if config.Rootfs != expectedConfig.Rootfs {
 		t.Fatalf("expected rootfs %q but received %q", expectedConfig.Rootfs, config.Rootfs)
 	}
-	expectedHooks.Poststop = expectedHooks.Poststop[1:] // expect unserializable hook to be skipped
+	expectedHooks[configs.Poststop] = expectedHooks[configs.Poststop][1:] // expect unserializable hook to be skipped
 	if !reflect.DeepEqual(config.Hooks, expectedHooks) {
 		t.Fatalf("expects hooks %q but received %q", expectedHooks, config.Hooks)
 	}

libcontainer/init_linux.go

@@ -20,6 +20,7 @@ import (
 	"github.com/opencontainers/runc/libcontainer/system"
 	"github.com/opencontainers/runc/libcontainer/user"
 	"github.com/opencontainers/runc/libcontainer/utils"
+	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/vishvananda/netlink"
@@ -67,6 +68,7 @@ type initConfig struct {
 	ConsoleHeight    uint16                `json:"console_height"`
 	RootlessEUID     bool                  `json:"rootless_euid,omitempty"`
 	RootlessCgroups  bool                  `json:"rootless_cgroups,omitempty"`
+	SpecState        *specs.State          `json:"spec_state,omitempty"`
 }
 
 type initer interface {

libcontainer/process_linux.go

@@ -362,6 +362,9 @@ func (p *initProcess) start() (retErr error) {
 	if err := p.createNetworkInterfaces(); err != nil {
 		return newSystemErrorWithCause(err, "creating network interfaces")
 	}
+	if err := p.updateSpecState(); err != nil {
+		return newSystemErrorWithCause(err, "updating the spec state")
+	}
 	if err := p.sendConfig(); err != nil {
 		return newSystemErrorWithCause(err, "sending config to init process")
 	}
@@ -378,9 +381,9 @@ func (p *initProcess) start() (retErr error) {
 			if err := setupRlimits(p.config.Rlimits, p.pid()); err != nil {
 				return newSystemErrorWithCause(err, "setting rlimits for ready process")
 			}
-			// call prestart hooks
+			// call prestart and CreateRuntime hooks
 			if !p.config.Config.Namespaces.Contains(configs.NEWNS) {
-				// Setup cgroup before prestart hook, so that the prestart hook could apply cgroup permissions.
+				// Setup cgroup before the hook, so that the prestart and CreateRuntime hook could apply cgroup permissions.
 				if err := p.manager.Set(p.config.Config); err != nil {
 					return newSystemErrorWithCause(err, "setting cgroup config for ready process")
 				}
@@ -397,11 +400,14 @@ func (p *initProcess) start() (retErr error) {
 					}
 					// initProcessStartTime hasn't been set yet.
 					s.Pid = p.cmd.Process.Pid
-					s.Status = "creating"
-					for i, hook := range p.config.Config.Hooks.Prestart {
-						if err := hook.Run(s); err != nil {
-							return newSystemErrorWithCausef(err, "running prestart hook %d", i)
-						}
+					s.Status = configs.Creating
+					hooks := p.config.Config.Hooks
+
+					if err := hooks[configs.Prestart].RunHooks(s); err != nil {
+						return err
+					}
+					if err := hooks[configs.CreateRuntime].RunHooks(s); err != nil {
+						return err
 					}
 				}
 			}
@@ -427,11 +433,14 @@ func (p *initProcess) start() (retErr error) {
 				}
 				// initProcessStartTime hasn't been set yet.
 				s.Pid = p.cmd.Process.Pid
-				s.Status = "creating"
-				for i, hook := range p.config.Config.Hooks.Prestart {
-					if err := hook.Run(s); err != nil {
-						return newSystemErrorWithCausef(err, "running prestart hook %d", i)
-					}
+				s.Status = configs.Creating
+				hooks := p.config.Config.Hooks
+
+				if err := hooks[configs.Prestart].RunHooks(s); err != nil {
+					return err
+				}
+				if err := hooks[configs.CreateRuntime].RunHooks(s); err != nil {
+					return err
 				}
 			}
 			// Sync with child.
@@ -450,7 +459,7 @@ func (p *initProcess) start() (retErr error) {
 		return newSystemErrorWithCause(ierr, "container init")
 	}
 	if p.config.Config.Namespaces.Contains(configs.NEWNS) && !sentResume {
-		return newSystemError(errors.New("could not synchronise after executing prestart hooks with container process"))
+		return newSystemError(errors.New("could not synchronise after executing prestart and CreateRuntime hooks with container process"))
 	}
 	if err := unix.Shutdown(int(p.messageSockPair.parent.Fd()), unix.SHUT_WR); err != nil {
 		return newSystemErrorWithCause(err, "shutting down init pipe")
@@ -492,6 +501,16 @@ func (p *initProcess) startTime() (uint64, error) {
 	return stat.StartTime, err
 }
 
+func (p *initProcess) updateSpecState() error {
+	s, err := p.container.currentOCIState()
+	if err != nil {
+		return err
+	}
+
+	p.config.SpecState = s
+	return nil
+}
+
 func (p *initProcess) sendConfig() error {
 	// send the config to the container's init process, we don't use JSON Encode
 	// here because there might be a problem in JSON decoder in some cases, see:

libcontainer/rootfs_linux.go

@@ -98,6 +98,13 @@ func prepareRootfs(pipe io.ReadWriter, iConfig *initConfig) (err error) {
 		return newSystemErrorWithCausef(err, "changing dir to %q", config.Rootfs)
 	}
 
+	s := iConfig.SpecState
+	s.Pid = unix.Getpid()
+	s.Status = configs.Creating
+	if err := iConfig.Config.Hooks[configs.CreateContainer].RunHooks(s); err != nil {
+		return err
+	}
+
 	if config.NoPivotRoot {
 		err = msMoveRoot(config.Rootfs)
 	} else if config.Namespaces.Contains(configs.NEWNS) {

libcontainer/specconv/spec_linux.go

@@ -881,20 +881,31 @@ func SetupSeccomp(config *specs.LinuxSeccomp) (*configs.Seccomp, error) {
 }
 
 func createHooks(rspec *specs.Spec, config *configs.Config) {
-	config.Hooks = &configs.Hooks{}
+	config.Hooks = configs.Hooks{}
 	if rspec.Hooks != nil {
-
 		for _, h := range rspec.Hooks.Prestart {
 			cmd := createCommandHook(h)
-			config.Hooks.Prestart = append(config.Hooks.Prestart, configs.NewCommandHook(cmd))
+			config.Hooks[configs.Prestart] = append(config.Hooks[configs.Prestart], configs.NewCommandHook(cmd))
+		}
+		for _, h := range rspec.Hooks.CreateRuntime {
+			cmd := createCommandHook(h)
+			config.Hooks[configs.CreateRuntime] = append(config.Hooks[configs.CreateRuntime], configs.NewCommandHook(cmd))
+		}
+		for _, h := range rspec.Hooks.CreateContainer {
+			cmd := createCommandHook(h)
+			config.Hooks[configs.CreateContainer] = append(config.Hooks[configs.CreateContainer], configs.NewCommandHook(cmd))
+		}
+		for _, h := range rspec.Hooks.StartContainer {
+			cmd := createCommandHook(h)
+			config.Hooks[configs.StartContainer] = append(config.Hooks[configs.StartContainer], configs.NewCommandHook(cmd))
 		}
 		for _, h := range rspec.Hooks.Poststart {
 			cmd := createCommandHook(h)
-			config.Hooks.Poststart = append(config.Hooks.Poststart, configs.NewCommandHook(cmd))
+			config.Hooks[configs.Poststart] = append(config.Hooks[configs.Poststart], configs.NewCommandHook(cmd))
 		}
 		for _, h := range rspec.Hooks.Poststop {
 			cmd := createCommandHook(h)
-			config.Hooks.Poststop = append(config.Hooks.Poststop, configs.NewCommandHook(cmd))
+			config.Hooks[configs.Poststop] = append(config.Hooks[configs.Poststop], configs.NewCommandHook(cmd))
 		}
 	}
 }

libcontainer/standard_init_linux.go

@@ -207,6 +207,14 @@ func (l *linuxStandardInit) Init() error {
 			return newSystemErrorWithCause(err, "init seccomp")
 		}
 	}
+
+	s := l.config.SpecState
+	s.Pid = unix.Getpid()
+	s.Status = configs.Created
+	if err := l.config.Config.Hooks[configs.StartContainer].RunHooks(s); err != nil {
+		return err
+	}
+
 	if err := unix.Exec(name, l.config.Args[0:], os.Environ()); err != nil {
 		return newSystemErrorWithCause(err, "exec user process")
 	}