fix: Remove files with restrictive perms from --oci temp rootfs

When we run a container in `--oci` mode, we have a temporary rootfs
that must be removed when the container exits.

A container can contain files / dirs with restrictive permissions that
prevent direct removal.

Use fs.ForceRemoveAll, which will chmod & remove anything that
os.RemoveAll is unable to remove.

Fixes sylabs/singularity#1586

Signed-off-by: Edita Kizinevic <edita.kizinevic@cern.ch>

e2e/docker/docker.go

@@ -958,6 +958,7 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests {
 			t.Run("issue 4524", c.issue4524)
 			t.Run("issue 1286", c.issue1286)
 			t.Run("issue 1528", c.issue1528)
+			t.Run("issue 1586", c.issue1586)
 		},
 		// Tests that are especially slow, or run against a local docker
 		// registry, can be run in parallel, with `--disable-cache` used within

e2e/docker/regressions.go

@@ -11,6 +11,7 @@ package docker
 
 import (
 	"fmt"
+	"io"
 	"os"
 	"path"
 	"path/filepath"
@@ -286,3 +287,34 @@ func (c ctx) issue1528(t *testing.T) {
 		})
 	}
 }
+
+// https://github.com/sylabs/singularity/issues/1586
+// In OCI mode, ensure that nothing is left in TMPDIR from a docker:// image with restrictive file permissions.
+func (c ctx) issue1586(t *testing.T) {
+	tmpDir, cleanup := e2e.MakeTempDir(t, c.env.TestDir, "issue1586-", "")
+	t.Cleanup(func() {
+		if !t.Failed() {
+			cleanup(t)
+		}
+	})
+
+	c.env.RunApptainer(
+		t,
+		e2e.WithProfile(e2e.OCIUserProfile),
+		e2e.WithCommand("exec"),
+		e2e.WithArgs("docker://almalinux:9.1-minimal-20230407", "/bin/true"),
+		e2e.WithEnv(append(os.Environ(), "TMPDIR="+tmpDir)),
+		e2e.ExpectExit(0,
+			e2e.ExpectError(e2e.UnwantedContainMatch, "permission denied"),
+		),
+	)
+
+	d, err := os.Open(tmpDir)
+	if err != nil {
+		t.Errorf("Couldn't open TMPDIR %s: %v", tmpDir, err)
+	}
+	defer d.Close()
+	if _, err = d.Readdir(1); err != io.EOF {
+		t.Errorf("TMPDIR is not empty after apptainer exited")
+	}
+}

internal/pkg/runtime/launcher/oci/launcher_linux.go

@@ -27,6 +27,7 @@ import (
 	"github.com/apptainer/apptainer/internal/pkg/cache"
 	"github.com/apptainer/apptainer/internal/pkg/cgroups"
 	"github.com/apptainer/apptainer/internal/pkg/runtime/launcher"
+	"github.com/apptainer/apptainer/internal/pkg/util/fs"
 	"github.com/apptainer/apptainer/internal/pkg/util/fs/files"
 	"github.com/apptainer/apptainer/pkg/ocibundle"
 	"github.com/apptainer/apptainer/pkg/ocibundle/native"
@@ -425,7 +426,7 @@ func (l *Launcher) Exec(ctx context.Context, image string, process string, args
 	}
 	defer func() {
 		sylog.Debugf("Removing OCI bundle at: %s", bundleDir)
-		if err := os.RemoveAll(bundleDir); err != nil {
+		if err := fs.ForceRemoveAll(bundleDir); err != nil {
 			sylog.Errorf("Couldn't remove OCI bundle %s: %v", bundleDir, err)
 		}
 	}()