Add start_containers_with_systemd for running Noble stemcell containers

Currently mixed mode is not supported because the config.ini from garden needs
to be modified to change the entrypoint of the container.

So this property will need to be enabled for Noble, but that will prevent older stemcell
lines from working.

jobs/warden_cpi/spec

@@ -6,6 +6,7 @@ templates:
   cpi_ctl.erb: bin/cpi_ctl
   cpi.json.erb: config/cpi.json
   sudoers: config/sudoers
+  pre-start.erb: bin/pre-start
 
 packages:
 - warden_cpi
@@ -74,3 +75,7 @@ properties:
   warden_cpi.actions.guest_persistent_bind_mounts_dir:
     description: "Directory with sub-directories at which persistent disks are mounted inside VMs"
     default: "/warden-cpi-dev"
+
+  warden_cpi.start_containers_with_systemd:
+    description: "Containers will use /sbin/init as the entry point. Enabling this is required for Noble stemcells, but currently breaks all pre-Noble stemcells"
+    default: false

jobs/warden_cpi/templates/cpi.json.erb

@@ -1,5 +1,6 @@
 <%=
 JSON.dump(
+  "start_containers_with_systemd" => p("warden_cpi.start_containers_with_systemd"),
   "Warden" => {
     "ConnectNetwork" => p("warden_cpi.warden.connect_network"),
     "ConnectAddress" => p("warden_cpi.warden.connect_address"),

jobs/warden_cpi/templates/pre-start.erb

@@ -0,0 +1,9 @@
+#!/bin/bash
+
+set -eu
+
+<% if p("warden_cpi.start_containers_with_systemd") %>
+sed -i 's/\/var\/vcap\/data\/garden\/bin\/init/\/sbin\/init/' /var/vcap/jobs/garden/config/config.ini
+<% else %>
+sed -i 's/\/sbin\/init/\/var\/vcap\/data\/garden\/bin\/init/' /var/vcap/jobs/garden/config/config.ini
+<% end %>

src/bosh-warden-cpi/action/factory.go

@@ -8,6 +8,7 @@ import (
 	boshsys "github.com/cloudfoundry/bosh-utils/system"
 	boshuuid "github.com/cloudfoundry/bosh-utils/uuid"
 
+	. "bosh-warden-cpi/config"
 	bwcdisk "bosh-warden-cpi/disk"
 	bwcstem "bosh-warden-cpi/stemcell"
 	bwcutil "bosh-warden-cpi/util"
@@ -56,6 +57,7 @@ func NewFactory(
 	uuidGen boshuuid.Generator,
 	opts FactoryOpts,
 	logger boshlog.Logger,
+	config Config,
 ) Factory {
 	var decompressor bwcutil.Decompressor
 	if opts.ExpandStemcellTarball {
@@ -91,7 +93,7 @@ func NewFactory(
 
 	vmCreator := bwcvm.NewWardenCreator(
 		uuidGen, wardenClient, metadataService, agentEnvServiceFactory, ports,
-		hostBindMounts, guestBindMounts, systemResolvConfProvider, opts.Agent, logger)
+		hostBindMounts, guestBindMounts, systemResolvConfProvider, opts.Agent, logger, config)
 
 	vmFinder := bwcvm.NewWardenFinder(
 		wardenClient, agentEnvServiceFactory, ports, hostBindMounts, guestBindMounts, logger)

src/bosh-warden-cpi/main/config.go → src/bosh-warden-cpi/config/config.go

@@ -1,18 +1,18 @@
-package main
+package config
 
 import (
 	"encoding/json"
 
 	bosherr "github.com/cloudfoundry/bosh-utils/errors"
 	boshsys "github.com/cloudfoundry/bosh-utils/system"
-
-	bwcaction "bosh-warden-cpi/action"
 )
 
 type Config struct {
 	Warden WardenConfig
 
-	Actions bwcaction.FactoryOpts
+	Actions FactoryOpts
+
+	StartContainersWithSystemD bool `json:"start_containers_with_systemd"`
 }
 
 type WardenConfig struct {

src/bosh-warden-cpi/config/config_suite_test.go

@@ -0,0 +1,13 @@
+package config_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestConfigPackage(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Config Suite")
+}

src/bosh-warden-cpi/main/config_test.go → src/bosh-warden-cpi/config/config_test.go

@@ -1,15 +1,13 @@
-package main_test
+package config_test
 
 import (
+	. "bosh-warden-cpi/config"
 	"errors"
 
 	"github.com/cloudfoundry/bosh-cpi-go/apiv1"
 	fakesys "github.com/cloudfoundry/bosh-utils/system/fakes"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-
-	bwcaction "bosh-warden-cpi/action"
-	. "bosh-warden-cpi/main"
 )
 
 var validConfig = Config{
@@ -22,7 +20,7 @@ var validWardenConfig = WardenConfig{
 	ConnectAddress: "fake-address",
 }
 
-var validActionsOptions = bwcaction.FactoryOpts{
+var validActionsOptions = FactoryOpts{
 	StemcellsDir: "/tmp/stemcells",
 	DisksDir:     "/tmp/disks",
 

src/bosh-warden-cpi/action/factory_opts.go → src/bosh-warden-cpi/config/factory_opts.go

@@ -1,4 +1,4 @@
-package action
+package config
 
 import (
 	bosherr "github.com/cloudfoundry/bosh-utils/errors"

src/bosh-warden-cpi/action/factory_opts_test.go → src/bosh-warden-cpi/config/factory_opts_test.go

@@ -1,11 +1,11 @@
-package action_test
+package config_test
 
 import (
 	"github.com/cloudfoundry/bosh-cpi-go/apiv1"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
-	. "bosh-warden-cpi/action"
+	. "bosh-warden-cpi/config"
 )
 
 var _ = Describe("FactoryOpts", func() {

src/bosh-warden-cpi/main/main.go

@@ -4,6 +4,8 @@ import (
 	"flag"
 	"os"
 
+	. "bosh-warden-cpi/config"
+
 	wrdnclient "code.cloudfoundry.org/garden/client"
 	wrdnconn "code.cloudfoundry.org/garden/client/connection"
 	"github.com/cloudfoundry/bosh-cpi-go/rpc"
@@ -34,7 +36,7 @@ func main() {
 	wardenClient := wrdnclient.New(wardenConn)
 
 	cpiFactory := bwcaction.NewFactory(
-		wardenClient, fs, cmdRunner, uuidGen, config.Actions, logger)
+		wardenClient, fs, cmdRunner, uuidGen, config.Actions, logger, config)
 
 	cli := rpc.NewFactory(logger).NewCLI(cpiFactory)
 

src/bosh-warden-cpi/vm/warden_creator.go

@@ -9,6 +9,7 @@ import (
 	boshlog "github.com/cloudfoundry/bosh-utils/logger"
 	boshuuid "github.com/cloudfoundry/bosh-utils/uuid"
 
+	"bosh-warden-cpi/config"
 	bwcstem "bosh-warden-cpi/stemcell"
 )
 
@@ -27,6 +28,7 @@ type WardenCreator struct {
 
 	agentOptions apiv1.AgentOptions
 	logger       boshlog.Logger
+	Config       config.Config
 }
 
 func NewWardenCreator(
@@ -40,9 +42,11 @@ func NewWardenCreator(
 	systemResolvConfProvider func() (ResolvConf, error),
 	agentOptions apiv1.AgentOptions,
 	logger boshlog.Logger,
+	config config.Config,
 ) WardenCreator {
 	return WardenCreator{
 		uuidGen: uuidGen,
+		Config:  config,
 
 		wardenClient:           wardenClient,
 		metadataService:        metadataService,
@@ -113,6 +117,15 @@ func (c WardenCreator) Create(
 		Privileged: true,
 	}
 
+	if c.Config.StartContainersWithSystemD {
+		containerSpec.BindMounts = append(containerSpec.BindMounts, wrdn.BindMount{
+			SrcPath: "/sys/fs/cgroup",
+			DstPath: "/sys/fs/cgroup",
+			Mode:    wrdn.BindMountModeRW,
+			Origin:  wrdn.BindMountOriginHost,
+		})
+	}
+
 	c.logger.Debug("WardenCreator", "Creating container with spec %#v", containerSpec)
 
 	container, err := c.wardenClient.Create(containerSpec)
@@ -150,10 +163,18 @@ func (c WardenCreator) Create(
 		return WardenVM{}, bosherr.WrapError(err, "Updating container's metadata")
 	}
 
-	err = c.startAgentInContainer(container)
-	if err != nil {
-		c.cleanUpContainer(container)
-		return WardenVM{}, err
+	if c.Config.StartContainersWithSystemD {
+		err = c.prepareContainer(container)
+		if err != nil {
+			c.cleanUpContainer(container)
+			return WardenVM{}, err
+		}
+	} else {
+		err = c.startAgentInContainer(container)
+		if err != nil {
+			c.cleanUpContainer(container)
+			return WardenVM{}, err
+		}
 	}
 
 	vm := NewWardenVM(
@@ -220,6 +241,25 @@ func (c WardenCreator) startAgentInContainer(container wrdn.Container) error {
 	return nil
 }
 
+func (c WardenCreator) prepareContainer(container wrdn.Container) error {
+	processSpec := wrdn.ProcessSpec{
+		Path: "/bin/bash",
+		User: "root",
+		Args: []string{
+			"-c",
+			"umount /etc/hosts",
+		},
+	}
+
+	// Do not Wait() for the process to finish
+	_, err := container.Run(processSpec, wrdn.ProcessIO{})
+	if err != nil {
+		return bosherr.WrapError(err, "Preparing Container")
+	}
+
+	return nil
+}
+
 func (c WardenCreator) cleanUpContainer(container wrdn.Container) {
 	// false is to kill immediately
 	err := container.Stop(false)

src/bosh-warden-cpi/vm/warden_creator_test.go

@@ -13,6 +13,7 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
+	"bosh-warden-cpi/config"
 	fakestem "bosh-warden-cpi/stemcell/fakes"
 	. "bosh-warden-cpi/vm"
 	fakevm "bosh-warden-cpi/vm/fakes"
@@ -36,6 +37,7 @@ var _ = Describe("WardenCreator", func() {
 
 		logger  boshlog.Logger
 		creator WardenCreator
+		config  config.Config
 	)
 
 	BeforeEach(func() {
@@ -63,7 +65,7 @@ var _ = Describe("WardenCreator", func() {
 
 		creator = NewWardenCreator(
 			uuidGen, wardenClient, fakeMetadataService, agentEnvServiceFactory,
-			ports, hostBindMounts, guestBindMounts, resolvProvider, agentOptions, logger)
+			ports, hostBindMounts, guestBindMounts, resolvProvider, agentOptions, logger, config)
 	})
 
 	Describe("Create", func() {
@@ -193,6 +195,22 @@ var _ = Describe("WardenCreator", func() {
 				Expect(hostBindMounts.MakePersistentID).To(Equal(apiv1.NewVMCID("fake-vm-id")))
 			})
 
+			It("bind mounts cgroups into the container when starting container with systemd", func() {
+				creator.Config.StartContainersWithSystemD = true
+				_, err := creator.Create(apiv1.NewAgentID("fake-agent-id"), stemcell, VMProps{}, networks, env)
+				Expect(err).ToNot(HaveOccurred())
+
+				containerSpec := wardenConn.CreateArgsForCall(0)
+				Expect(containerSpec.BindMounts).To(ContainElement(
+					wrdn.BindMount{
+						SrcPath: "/sys/fs/cgroup",
+						DstPath: "/sys/fs/cgroup",
+						Mode:    wrdn.BindMountModeRW,
+						Origin:  wrdn.BindMountOriginHost,
+					},
+				))
+			})
+
 			It("returns error if making host ephemeral bind mount fails", func() {
 				hostBindMounts.MakeEphemeralErr = errors.New("fake-make-ephemeral-err")
 
@@ -335,6 +353,29 @@ var _ = Describe("WardenCreator", func() {
 						Expect(processIO).To(Equal(wrdn.ProcessIO{}))
 					})
 
+					It("does not start the bosh agent when starting container with systemd", func() {
+						creator.Config.StartContainersWithSystemD = true
+						_, err := creator.Create(apiv1.NewAgentID("fake-agent-id"), stemcell, VMProps{}, networks, env)
+						Expect(err).ToNot(HaveOccurred())
+
+						count := wardenConn.RunCallCount()
+						Expect(count).To(Equal(1))
+
+						expectedProcessSpec := wrdn.ProcessSpec{
+							Path: "/bin/bash",
+							User: "root",
+							Args: []string{
+								"-c",
+								"umount /etc/hosts",
+							},
+						}
+
+						handle, processSpec, processIO := wardenConn.RunArgsForCall(0)
+						Expect(handle).To(Equal("fake-vm-id"))
+						Expect(processSpec).To(Equal(expectedProcessSpec))
+						Expect(processIO).To(Equal(wrdn.ProcessIO{}))
+					})
+
 					Context("when BOSH Agent fails to start", func() {
 						BeforeEach(func() {
 							wardenConn.RunReturns(nil, errors.New("fake-run-err"))