Skip to content

Commit

Permalink
oci: support for writable extfs img overlay via fuse-overlayfs
Browse files Browse the repository at this point in the history 
(sylabs/singularity#1740)

* oci: support for writable extfs img overlay via fuse-overlayfs

* added unit- and e2e-tests

* fixed exposure of "upper" & "work" subdir in readonly overlays

* added fix for file-ownership in FUSE-mounted images + e2e test of fix

* fix handling of "upper" in :ro overlay dirs, adjust e2e tests

Signed-off-by: Edita Kizinevic <edita.kizinevic@cern.ch>
  • Loading branch information
@edytuk
preminger authored and edytuk committed Jul 24, 2023
1 parent ccee18a commit 360ea69
Show file tree
Hide file tree
Showing 10 changed files with 475 additions and 144 deletions.
11 changes: 6 additions & 5 deletions CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -80,11 +80,12 @@ For older changes see the [archived Singularity change log](https://github.com/a
`--dns` flag can be used to pass a comma-separated list of DNS servers that
will be used in the container; if this flag is not used, the container will
use the same `resolv.conf` settings as the host.
- OCI-mode now supports an `--overlay <arg>` flag. `<arg>` can be a writable
directory, in which case changes to the filesystem will persist across runs of
the OCI container. Alternatively, `<arg>` can be `<dir>:ro` or the path of a
squashfs or extfs image, to be mounted as a read-only overlay. Multiple
overlays can be specified, but all but one must be read-only.
- OCI-mode now supports the `--overlay <arg>` flag. `<arg>` can be the path to a
writable directory or writable extfs image, in which case changes to the
filesystem will persist across runs of the OCI container. Alternatively,
`--overlay <arg>:ro` can be used, where `<arg>` is the path to a directory, to
a squashfs image, or to an extfs image, to be mounted as a read-only overlay.
Multiple overlays can be specified, but all but one must be read-only.
- OCI-mode now supports the `--workdir <workdir>` option. If this option is
specified, `/tmp` and `/var/tmp` will be mapped, respectively, to
`<workdir>/tmp` and `<workdir>/var_tmp` on the host, rather than to tmpfs
Expand Down
25 changes: 13 additions & 12 deletions e2e/actions/actions.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2963,17 +2963,18 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests {
//
// OCI Runtime Mode
//
"ociRun": c.actionOciRun, // apptainer run --oci
"ociExec": c.actionOciExec, // apptainer exec --oci
"ociShell": c.actionOciShell, // apptainer shell --oci
"ociSTDPIPE": c.ociSTDPipe, // stdin/stdout pipe --oci
"ociNetwork": c.actionOciNetwork, // apptainer exec --oci --net
"ociBinds": c.actionOciBinds, // apptainer exec --oci --bind / --mount
"ociCdi": c.actionOciCdi, // apptainer exec --oci --cdi
"ociIDMaps": c.actionOciIDMaps, // check uid/gid mapping on host for --oci as user / --fakeroot
"ociCompat": np(c.actionOciCompat), // --oci equivalence to native mode --compat
"ociOverlay": (c.actionOciOverlay), // --overlay in OCI mode
"ociOverlayTeardown": np(c.actionOciOverlayTeardown), // proper overlay unmounting in OCI mode
"ociNo-mount": c.actionOciNoMount, // --no-mount in OCI mode
"ociRun": c.actionOciRun, // apptainer run --oci
"ociExec": c.actionOciExec, // apptainer exec --oci
"ociShell": c.actionOciShell, // apptainer shell --oci
"ociSTDPIPE": c.ociSTDPipe, // stdin/stdout pipe --oci
"ociNetwork": c.actionOciNetwork, // apptainer exec --oci --net
"ociBinds": c.actionOciBinds, // apptainer exec --oci --bind / --mount
"ociCdi": c.actionOciCdi, // apptainer exec --oci --cdi
"ociIDMaps": c.actionOciIDMaps, // check uid/gid mapping on host for --oci as user / --fakeroot
"ociCompat": np(c.actionOciCompat), // --oci equivalence to native mode --compat
"ociOverlay": (c.actionOciOverlay), // --overlay in OCI mode
"ociOverlayExtfsPerms": (c.actionOciOverlayExtfsPerms), // permissions in writable extfs overlays mounted with FUSE in OCI mode
"ociOverlayTeardown": np(c.actionOciOverlayTeardown), // proper overlay unmounting in OCI mode
"ociNo-mount": c.actionOciNoMount, // --no-mount in OCI mode
}
}
218 changes: 192 additions & 26 deletions e2e/actions/oci.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ import (
"text/template"

"github.com/apptainer/apptainer/e2e/internal/e2e"
"github.com/apptainer/apptainer/internal/pkg/test/tool/dirs"
"github.com/apptainer/apptainer/internal/pkg/test/tool/require"
"github.com/apptainer/apptainer/internal/pkg/util/fs"
cdispecs "github.com/container-orchestrated-devices/container-device-interface/specs-go"
"gotest.tools/v3/assert"
Expand Down Expand Up @@ -1051,13 +1053,14 @@ func (c actionTests) actionOciOverlay(t *testing.T) {
}
})

// Create a few read-only overlay subdirs under testDir
// Create a few writable overlay subdirs under testDir
for i := 0; i < 3; i++ {
dirName := fmt.Sprintf("my_rw_ol_dir%d", i)
fullPath := filepath.Join(testDir, dirName)
if err = os.Mkdir(fullPath, 0o755); err != nil {
t.Fatal(err)
}
dirs.MkdirOrFatal(t, fullPath, 0o755)
upperPath := filepath.Join(fullPath, "upper")
dirs.MkdirOrFatal(t, upperPath, 0o777)
dirs.MkdirOrFatal(t, filepath.Join(fullPath, "work"), 0o777)
t.Cleanup(func() {
if !t.Failed() {
os.RemoveAll(fullPath)
Expand All @@ -1069,28 +1072,45 @@ func (c actionTests) actionOciOverlay(t *testing.T) {
for i := 0; i < 3; i++ {
dirName := fmt.Sprintf("my_ro_ol_dir%d", i)
fullPath := filepath.Join(testDir, dirName)
if err = os.Mkdir(fullPath, 0o755); err != nil {
t.Fatal(err)
}
dirs.MkdirOrFatal(t, fullPath, 0o755)
upperPath := filepath.Join(fullPath, "upper")
dirs.MkdirOrFatal(t, upperPath, 0o777)
dirs.MkdirOrFatal(t, filepath.Join(fullPath, "work"), 0o777)
t.Cleanup(func() {
if !t.Failed() {
os.RemoveAll(fullPath)
}
})
if err = os.WriteFile(
filepath.Join(fullPath, fmt.Sprintf("testfile.%d", i)),
filepath.Join(upperPath, fmt.Sprintf("testfile.%d", i)),
[]byte(fmt.Sprintf("test_string_%d\n", i)),
0o644); err != nil {
t.Fatal(err)
}
if err = os.WriteFile(
filepath.Join(fullPath, "maskable_testfile"),
filepath.Join(upperPath, "maskable_testfile"),
[]byte(fmt.Sprintf("maskable_string_%d\n", i)),
0o644); err != nil {
t.Fatal(err)
}
}

// Create a copy of the extfs test image to be used for testing readonly
// extfs image overlays
readonlyExtfsImgPath := filepath.Join(testDir, "readonly-extfs.img")
err = fs.CopyFile(extfsImgPath, readonlyExtfsImgPath, 0o444)
if err != nil {
t.Fatalf("could not copy %q to %q: %s", extfsImgPath, readonlyExtfsImgPath, err)
}

// Create a copy of the extfs test image to be used for testing writable
// extfs image overlays
writableExtfsImgPath := filepath.Join(testDir, "writable-extfs.img")
err = fs.CopyFile(extfsImgPath, writableExtfsImgPath, 0o755)
if err != nil {
t.Fatalf("could not copy %q to %q: %s", extfsImgPath, writableExtfsImgPath, err)
}

tests := []struct {
name string
args []string
Expand All @@ -1111,6 +1131,14 @@ func (c actionTests) actionOciOverlay(t *testing.T) {
e2e.ExpectOutput(e2e.ExactMatch, "my_test_string"),
},
},
{
name: "ExistRWDirRevisitAsRO",
args: []string{"--overlay", filepath.Join(testDir, "my_rw_ol_dir0:ro"), imageRef, "cat", "/my_test_file"},
exitCode: 0,
wantOutputs: []e2e.ApptainerCmdResultOp{
e2e.ExpectOutput(e2e.ExactMatch, "my_test_string"),
},
},
{
name: "RWOverlayMissing",
args: []string{"--overlay", filepath.Join(testDir, "something_nonexistent"), imageRef, "echo", "hi"},
Expand Down Expand Up @@ -1144,13 +1172,13 @@ func (c actionTests) actionOciOverlay(t *testing.T) {
name: "AllTypesAtOnce",
args: []string{
"--overlay", filepath.Join(testDir, "my_ro_ol_dir2:ro"),
"--overlay", extfsImgPath + ":ro",
"--overlay", readonlyExtfsImgPath + ":ro",
"--overlay", squashfsImgPath,
"--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"),
"--overlay", filepath.Join(testDir, "my_rw_ol_dir0"),
imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath),
},
requiredCmds: []string{"squashfuse", "fuse2fs"},
requiredCmds: []string{"squashfuse", "fuse2fs", "fusermount"},
exitCode: 0,
wantOutputs: []e2e.ApptainerCmdResultOp{
e2e.ExpectOutput(e2e.ContainMatch, "test_string_1"),
Expand All @@ -1167,7 +1195,7 @@ func (c actionTests) actionOciOverlay(t *testing.T) {
"--overlay", filepath.Join(testDir, "my_rw_ol_dir0"),
imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath),
},
requiredCmds: []string{"squashfuse"},
requiredCmds: []string{"squashfuse", "fusermount"},
exitCode: 0,
wantOutputs: []e2e.ApptainerCmdResultOp{
e2e.ExpectOutput(e2e.ContainMatch, "test_string_1"),
Expand All @@ -1179,12 +1207,12 @@ func (c actionTests) actionOciOverlay(t *testing.T) {
name: "ExtfsAndDirs",
args: []string{
"--overlay", filepath.Join(testDir, "my_ro_ol_dir2:ro"),
"--overlay", extfsImgPath + ":ro",
"--overlay", readonlyExtfsImgPath + ":ro",
"--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"),
"--overlay", filepath.Join(testDir, "my_rw_ol_dir0"),
imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath),
},
requiredCmds: []string{"fuse2fs"},
requiredCmds: []string{"fuse2fs", "fusermount"},
exitCode: 0,
wantOutputs: []e2e.ApptainerCmdResultOp{
e2e.ExpectOutput(e2e.ContainMatch, "test_string_1"),
Expand All @@ -1202,7 +1230,7 @@ func (c actionTests) actionOciOverlay(t *testing.T) {
"--overlay", filepath.Join(testDir, "my_rw_ol_dir0"),
imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath),
},
requiredCmds: []string{"squashfuse"},
requiredCmds: []string{"squashfuse", "fusermount"},
exitCode: 255,
},
{
Expand All @@ -1214,7 +1242,7 @@ func (c actionTests) actionOciOverlay(t *testing.T) {
"--overlay", filepath.Join(testDir, "something_nonexistent"),
imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath),
},
requiredCmds: []string{"squashfuse"},
requiredCmds: []string{"squashfuse", "fusermount"},
exitCode: 255,
},
{
Expand All @@ -1226,8 +1254,7 @@ func (c actionTests) actionOciOverlay(t *testing.T) {
"--overlay", filepath.Join(testDir, "my_rw_ol_dir0"),
imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath),
},
requiredCmds: []string{"squashfuse"},
exitCode: 255,
exitCode: 255,
},
{
name: "ThreeWritables",
Expand All @@ -1239,8 +1266,75 @@ func (c actionTests) actionOciOverlay(t *testing.T) {
"--overlay", filepath.Join(testDir, "my_rw_ol_dir2"),
imageRef, "cat", "/testfile.1", "/maskable_testfile", filepath.Join("/", imgTestFilePath),
},
requiredCmds: []string{"squashfuse"},
exitCode: 255,
exitCode: 255,
},
{
name: "WritableExtfs",
args: []string{"--overlay", writableExtfsImgPath, imageRef, "sh", "-c", "echo my_test_string > /my_test_file"},
requiredCmds: []string{"fuse2fs", "fuse-overlayfs", "fusermount"},
exitCode: 0,
},
{
name: "WritableExtfsRevisit",
args: []string{"--overlay", writableExtfsImgPath, imageRef, "cat", "/my_test_file"},
requiredCmds: []string{"fuse2fs", "fuse-overlayfs", "fusermount"},
exitCode: 0,
wantOutputs: []e2e.ApptainerCmdResultOp{
e2e.ExpectOutput(e2e.ExactMatch, "my_test_string"),
},
},
{
name: "WritableExtfsRevisitAsRO",
args: []string{"--overlay", writableExtfsImgPath + ":ro", imageRef, "cat", "/my_test_file"},
requiredCmds: []string{"fuse2fs", "fuse-overlayfs", "fusermount"},
exitCode: 0,
wantOutputs: []e2e.ApptainerCmdResultOp{
e2e.ExpectOutput(e2e.ExactMatch, "my_test_string"),
},
},
{
name: "WritableExtfsWithDirs",
args: []string{
"--overlay", filepath.Join(testDir, "my_ro_ol_dir0:ro"),
"--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"),
"--overlay", writableExtfsImgPath,
imageRef, "cat", "/my_test_file",
},
requiredCmds: []string{"fuse2fs", "fuse-overlayfs", "fusermount"},
exitCode: 0,
wantOutputs: []e2e.ApptainerCmdResultOp{
e2e.ExpectOutput(e2e.ExactMatch, "my_test_string"),
},
},
{
name: "WritableExtfsWithMix",
args: []string{
"--overlay", filepath.Join(testDir, "my_ro_ol_dir0:ro"),
"--overlay", readonlyExtfsImgPath + ":ro",
"--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"),
"--overlay", writableExtfsImgPath,
imageRef, "cat", "/my_test_file",
},
exitCode: 0,
requiredCmds: []string{"fuse2fs", "fuse-overlayfs", "fusermount"},
wantOutputs: []e2e.ApptainerCmdResultOp{
e2e.ExpectOutput(e2e.ExactMatch, "my_test_string"),
},
},
{
name: "WritableExtfsWithAll",
args: []string{
"--overlay", filepath.Join(testDir, "my_ro_ol_dir0:ro"),
"--overlay", readonlyExtfsImgPath + ":ro",
"--overlay", filepath.Join(testDir, "my_ro_ol_dir1:ro"),
"--overlay", writableExtfsImgPath,
imageRef, "cat", "/my_test_file",
},
exitCode: 0,
requiredCmds: []string{"squashfuse", "fuse2fs", "fuse-overlayfs", "fusermount"},
wantOutputs: []e2e.ApptainerCmdResultOp{
e2e.ExpectOutput(e2e.ExactMatch, "my_test_string"),
},
},
}

Expand All @@ -1266,6 +1360,16 @@ func (c actionTests) actionOciOverlay(t *testing.T) {
}
}

func haveAllCommands(t *testing.T, cmds []string) bool {
for _, c := range cmds {
if _, err := exec.LookPath(c); err != nil {
return false
}
}

return true
}

// actionOciOverlayTeardown checks that OCI-mode overlays are correctly
// unmounted even in root mode (i.e., when user namespaces are not involved).
func (c actionTests) actionOciOverlayTeardown(t *testing.T) {
Expand All @@ -1285,6 +1389,9 @@ func (c actionTests) actionOciOverlayTeardown(t *testing.T) {
}
})

dirs.MkdirOrFatal(t, filepath.Join(tmpDir, "upper"), 0o777)
dirs.MkdirOrFatal(t, filepath.Join(tmpDir, "work"), 0o777)

c.env.RunApptainer(
t,
e2e.WithProfile(e2e.OCIRootProfile),
Expand Down Expand Up @@ -1319,14 +1426,73 @@ func countLines(path string) (int, error) {
return lines, nil
}

func haveAllCommands(t *testing.T, cmds []string) bool {
for _, c := range cmds {
if _, err := exec.LookPath(c); err != nil {
return false
// Check that write permissions are indeed available for writable FUSE-mounted
// extfs image overlays.
func (c actionTests) actionOciOverlayExtfsPerms(t *testing.T) {
require.Command(t, "fuse2fs")
require.Command(t, "fuse-overlayfs")
require.Command(t, "fusermount")

for _, profile := range e2e.OCIProfiles {
// First, create a writable extfs overlay with `apptainer overlay create`.
tmpDir, cleanup := e2e.MakeTempDir(t, c.env.TestDir, "oci_overlay_extfs_perms-", "")
t.Cleanup(func() {
if !t.Failed() {
cleanup(t)
}
})

imgPath := filepath.Join(tmpDir, "extfs-perms-test.img")

c.env.RunApptainer(
t,
e2e.WithProfile(e2e.UserProfile),
e2e.WithCommand("overlay"),
e2e.WithArgs("create", "--size", "64", imgPath),
e2e.ExpectExit(0),
)

// Now test whether we can write to, and subsequently read from, the image
// we created.
e2e.EnsureOCIArchive(t, c.env)
imageRef := "oci-archive:" + c.env.OCIArchivePath

tests := []struct {
name string
args []string
exitCode int
wantOutputs []e2e.ApptainerCmdResultOp
}{
{
name: "FirstWrite",
args: []string{"--overlay", imgPath, imageRef, "sh", "-c", "echo my_test_string > /my_test_file"},
exitCode: 0,
},
{
name: "ThenRead",
args: []string{"--overlay", imgPath, imageRef, "cat", "/my_test_file"},
exitCode: 0,
wantOutputs: []e2e.ApptainerCmdResultOp{
e2e.ExpectOutput(e2e.ExactMatch, "my_test_string"),
},
},
}
t.Run(profile.String(), func(t *testing.T) {
for _, tt := range tests {
c.env.RunApptainer(
t,
e2e.AsSubtest(tt.name),
e2e.WithProfile(profile),
e2e.WithCommand("exec"),
e2e.WithArgs(tt.args...),
e2e.ExpectExit(
tt.exitCode,
tt.wantOutputs...,
),
)
}
})
}

return true
}

// Make sure --workdir and --scratch work together nicely even when workdir is a
Expand Down
Loading

0 comments on commit 360ea69

Please sign in to comment.