fix: create rootfs for system services via /system tmpfs

Container rootfs should be writeable as containerd mounts standard
filesystems `/proc` et al.

When `/opt` was used as a root of container filesystem this results in a
problem: Talos overlay mounts `/opt` on `/var/system` which means that
as long as `apid` running `/var` can't be unmounted which breaks
upgrades.

So instead use `/system/libexec` as rootfs for the containers, `/system`
is `tmpfs`, and bind-mount actually executable (`/sbin/init`, machined)
into rootfs.

This fixes upgrades for 0.10.

See also #3425

Signed-off-by: Andrey Smirnov <smirnov.andrey@gmail.com>

Dockerfile

@@ -267,9 +267,6 @@ RUN ln -s /etc/ssl /rootfs/etc/pki
 RUN ln -s /etc/ssl /rootfs/usr/share/ca-certificates
 RUN ln -s /etc/ssl /rootfs/usr/local/share/ca-certificates
 RUN ln -s /etc/ssl /rootfs/etc/ca-certificates
-RUN mkdir -pv /rootfs/opt/{apid,trustd}
-RUN ln /rootfs/sbin/init /rootfs/opt/apid/apid
-RUN ln /rootfs/sbin/init /rootfs/opt/trustd/trustd
 
 FROM rootfs-base AS rootfs-squashfs
 RUN mksquashfs /rootfs /rootfs.sqsh -all-root -noappend -comp xz -Xdict-size 100% -no-progress

internal/app/machined/pkg/system/services/apid.go

@@ -52,7 +52,7 @@ func (o *APID) PreFunc(ctx context.Context, r runtime.Runtime) error {
 		o.syncKubeletPKI()
 	}
 
-	return nil
+	return prepareRootfs(o.ID(r))
 }
 
 // PostFunc implements the Service interface.
@@ -155,7 +155,7 @@ func (o *APID) Runner(r runtime.Runtime) (runner.Runner, error) {
 		runner.WithOCISpecOpts(
 			oci.WithHostNamespace(specs.NetworkNamespace),
 			oci.WithMounts(mounts),
-			oci.WithRootFSPath("/opt/apid"),
+			oci.WithRootFSPath(filepath.Join(constants.SystemLibexecPath, o.ID(r))),
 			oci.WithRootFSReadonly(),
 		),
 	),

internal/app/machined/pkg/system/services/trustd.go

@@ -10,6 +10,7 @@ import (
 	"context"
 	"fmt"
 	"net"
+	"path/filepath"
 
 	"github.com/containerd/containerd/oci"
 	specs "github.com/opencontainers/runtime-spec/specs-go"
@@ -36,7 +37,7 @@ func (t *Trustd) ID(r runtime.Runtime) string {
 
 // PreFunc implements the Service interface.
 func (t *Trustd) PreFunc(ctx context.Context, r runtime.Runtime) error {
-	return nil
+	return prepareRootfs(t.ID(r))
 }
 
 // PostFunc implements the Service interface.
@@ -89,7 +90,7 @@ func (t *Trustd) Runner(r runtime.Runtime) (runner.Runner, error) {
 			containerd.WithMemoryLimit(int64(1000000*512)),
 			oci.WithHostNamespace(specs.NetworkNamespace),
 			oci.WithMounts(mounts),
-			oci.WithRootFSPath("/opt/trustd"),
+			oci.WithRootFSPath(filepath.Join(constants.SystemLibexecPath, t.ID(r))),
 			oci.WithRootFSReadonly(),
 		),
 	),

internal/app/machined/pkg/system/services/utils.go

@@ -0,0 +1,37 @@
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+package services
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"golang.org/x/sys/unix"
+
+	"github.com/talos-systems/talos/pkg/machinery/constants"
+)
+
+// prepareRootfs creates /system/libexec/<service> rootfs and bind-mounts /sbin/init there.
+func prepareRootfs(id string) error {
+	rootfsPath := filepath.Join(constants.SystemLibexecPath, id)
+
+	if err := os.MkdirAll(rootfsPath, 0o700); err != nil {
+		return fmt.Errorf("failed to create rootfs %q: %w", rootfsPath, err)
+	}
+
+	executablePath := filepath.Join(rootfsPath, id)
+
+	if err := ioutil.WriteFile(executablePath, nil, 0o500); err != nil {
+		return fmt.Errorf("failed to create empty executable %q: %w", executablePath, err)
+	}
+
+	if err := unix.Mount("/sbin/init", executablePath, "", unix.MS_BIND, ""); err != nil {
+		return fmt.Errorf("failed to create bind mount for %q: %w", executablePath, err)
+	}
+
+	return nil
+}

pkg/machinery/constants/constants.go

@@ -306,9 +306,6 @@ const (
 	// MachineSocketPath is the path to file socket of machine API.
 	MachineSocketPath = SystemRunPath + "/machined/machine.sock"
 
-	// TimeSocketPath is the path to file socket of time API.
-	TimeSocketPath = SystemRunPath + "/timed/timed.sock"
-
 	// NetworkSocketPath is the path to file socket of network API.
 	NetworkSocketPath = SystemRunPath + "/networkd/networkd.sock"
 
@@ -355,6 +352,9 @@ const (
 	// SystemEtcPath is the path to the system etc directory.
 	SystemEtcPath = SystemPath + "/etc"
 
+	// SystemLibexecPath is the path to the system libexec directory.
+	SystemLibexecPath = SystemPath + "/libexec"
+
 	// DefaultCNI is the default CNI.
 	DefaultCNI = "flannel"