From d139701b60e833321b56d35fe16560edeec695a0 Mon Sep 17 00:00:00 2001 From: David Trudgian Date: Fri, 4 Mar 2022 14:50:43 -0600 Subject: [PATCH] e2e: non-pid namespace e2e-tests for OCI/CGROUPS The Singularity e2e-tests were previously all run in a mount and PID namespace, to avoid polluting the host filesystem (critical), and process tree (less critical). These namespaces are set up in some CGO init code. The use of the PID namespace prevents testing with systemd as the cgroups manager, as systemd is aware of the host process tree, not the one in the new e2e PID namespace. This is a major omission since #540 switched to using systemd for cgroups management by default. This PR: * Modifies the e2e init CGO code, so that an env var `SINGULARITYE_E2E_NO_PID_NS` will cause it *not* to create a new PID namespace. * Modifies the Makefile so that the e2e suite is called twice, once with PID ns, once without. * Moves the INSTANCE cgroups test into a new package/topic e2e/cgroups, run in the e2e call without PID ns. * Moves the OCI tests into the e2e call without PID ns. * Modifies the CGROUPS and OCI tests so that they test with both systemd and cgroupfs management, using a convenience wrapper function. Fixes: 563 Note - this breaks the e2e CLI coverage, as it only supports collecting / analyzing one e2e run... where we now have two. The CLI coverage metrics are unused in practice, and flawed (don't consider combinations of flags required or possible), so I'm going to propose removing them in a separate issue. --- e2e/cgroups/cgroups.go | 89 +++++++++++++++++++++++++++++ e2e/e2e_test.go | 14 ++--- e2e/instance/instance.go | 39 +------------ e2e/internal/e2e/cgroups.go | 43 ++++++++++++++ e2e/internal/e2e/config.go | 10 +--- e2e/internal/e2e/init/init_linux.go | 10 +++- e2e/oci/oci.go | 23 ++++++-- e2e/suite.go | 17 ++++-- mlocal/frags/Makefile.stub | 9 ++- scripts/e2e-test | 2 +- 10 files changed, 189 insertions(+), 67 deletions(-) create mode 100644 e2e/cgroups/cgroups.go create mode 100644 e2e/internal/e2e/cgroups.go diff --git a/e2e/cgroups/cgroups.go b/e2e/cgroups/cgroups.go new file mode 100644 index 0000000000..03b0f4e814 --- /dev/null +++ b/e2e/cgroups/cgroups.go @@ -0,0 +1,89 @@ +// Copyright (c) 2022, Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package cgroups + +import ( + "fmt" + "testing" + + "github.com/google/uuid" + "github.com/sylabs/singularity/e2e/internal/e2e" + "github.com/sylabs/singularity/e2e/internal/testhelper" + "github.com/sylabs/singularity/internal/pkg/test/tool/require" +) + +// NOTE +// ---- +// Tests in this package/topic are run in a a mount namespace only. There is +// no PID namespace, in order that the systemd cgroups manager functionality +// can be exercised. +// +// You must take extra care not to leave detached process etc. that will +// pollute the host PID namespace. +// + +// randomName generates a random name instance or OCI container name based on a UUID. +func randomName(t *testing.T) string { + t.Helper() + + id, err := uuid.NewRandom() + if err != nil { + t.Fatal(err) + } + return id.String() +} + +type ctx struct { + env e2e.TestEnv +} + +// moved from INSTANCE suite, as testing with systemd cgroup manager requires +// e2e to be run without PID namespace +func (c *ctx) instanceApplyCgroups(t *testing.T) { + require.Cgroups(t) + e2e.EnsureImage(t, c.env) + + // pick up a random name + instanceName := randomName(t) + joinName := fmt.Sprintf("instance://%s", instanceName) + + c.env.RunSingularity( + t, + e2e.WithProfile(e2e.RootProfile), + e2e.WithCommand("instance start"), + e2e.WithArgs("--apply-cgroups", "testdata/cgroups/deny_device.toml", c.env.ImagePath, instanceName), + e2e.ExpectExit(0), + ) + + c.env.RunSingularity( + t, + e2e.WithProfile(e2e.RootProfile), + e2e.WithCommand("exec"), + e2e.WithArgs(joinName, "cat", "/dev/null"), + e2e.ExpectExit(1), + ) + + c.env.RunSingularity( + t, + e2e.WithProfile(e2e.RootProfile), + e2e.WithCommand("instance stop"), + e2e.WithArgs(instanceName), + e2e.ExpectExit(0), + ) +} + +// E2ETests is the main func to trigger the test suite +func E2ETests(env e2e.TestEnv) testhelper.Tests { + c := &ctx{ + env: env, + } + + np := testhelper.NoParallel + + return testhelper.Tests{ + "instance apply cgroups": np(env.WithCgroupManagers(c.instanceApplyCgroups)), + } +} diff --git a/e2e/e2e_test.go b/e2e/e2e_test.go index 0572d40f5a..2cdbc7a8cf 100644 --- a/e2e/e2e_test.go +++ b/e2e/e2e_test.go @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2021, Sylabs Inc. All rights reserved. +// Copyright (c) 2019-2022, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -14,12 +14,12 @@ import ( "os" "testing" - // This import will execute a CGO section with the help of a C - // constructor section "init". As we always require to run e2e - // tests as root, the C part is responsible of finding the original - // user who executes tests; it will also create a dedicated pid - // and mount namespace for e2e tests, and will finally restore - // identity to the original user but will retain privileges for + // This import will execute a CGO section with the help of a C constructor + // section "init". As we always require to run e2e tests as root, the C part + // is responsible of finding the original user who executes tests; it will + // also create a dedicated mount namespace for the e2e tests, and a PID + // namespace if "SINGULARITY_E2E_NO_PID_NS" is not set. Finally, it will + // restore identity to the original user but will retain privileges for // Privileged method enabling the execution of a function with root // privileges when required _ "github.com/sylabs/singularity/e2e/internal/e2e/init" diff --git a/e2e/instance/instance.go b/e2e/instance/instance.go index dff31f95ab..3a3554b395 100644 --- a/e2e/instance/instance.go +++ b/e2e/instance/instance.go @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2021, Sylabs Inc. All rights reserved. +// Copyright (c) 2019-2022, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -7,7 +7,6 @@ package instance import ( "bytes" - "fmt" "io/ioutil" "os" "path/filepath" @@ -20,7 +19,6 @@ import ( "github.com/pkg/errors" "github.com/sylabs/singularity/e2e/internal/e2e" "github.com/sylabs/singularity/e2e/internal/testhelper" - "github.com/sylabs/singularity/internal/pkg/test/tool/require" "github.com/sylabs/singularity/pkg/util/fs/proc" ) @@ -310,40 +308,6 @@ func (c *ctx) testGhostInstance(t *testing.T) { ) } -func (c *ctx) applyCgroupsInstance(t *testing.T) { - require.Cgroups(t) - - if !c.profile.In(e2e.RootProfile) { - t.Skipf("%s requires %s profile, current profile: %s", t.Name(), e2e.RootProfile, c.profile) - } - - // pick up a random name - instanceName := randomName(t) - joinName := fmt.Sprintf("instance://%s", instanceName) - - c.env.RunSingularity( - t, - e2e.WithProfile(c.profile), - e2e.WithCommand("instance start"), - e2e.WithArgs("--apply-cgroups", "testdata/cgroups/deny_device.toml", c.env.ImagePath, instanceName), - e2e.ExpectExit(0), - ) - - c.env.RunSingularity( - t, - e2e.WithProfile(c.profile), - e2e.WithCommand("exec"), - e2e.WithArgs(joinName, "cat", "/dev/null"), - e2e.PostRun(func(t *testing.T) { - if t.Failed() { - return - } - c.stopInstance(t, instanceName) - }), - e2e.ExpectExit(1), - ) -} - // E2ETests is the main func to trigger the test suite func E2ETests(env e2e.TestEnv) testhelper.Tests { c := &ctx{ @@ -371,7 +335,6 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { {"CreateManyInstances", c.testCreateManyInstances}, {"StopAll", c.testStopAll}, {"GhostInstance", c.testGhostInstance}, - {"ApplyCgroupsInstance", c.applyCgroupsInstance}, } profiles := []e2e.Profile{ diff --git a/e2e/internal/e2e/cgroups.go b/e2e/internal/e2e/cgroups.go new file mode 100644 index 0000000000..c3db5b017c --- /dev/null +++ b/e2e/internal/e2e/cgroups.go @@ -0,0 +1,43 @@ +// Copyright (c) 2022 Sylabs Inc. All rights reserved. +// This software is licensed under a 3-clause BSD license. Please consult the +// LICENSE.md file distributed with the sources of this project regarding your +// rights to use or distribute this software. + +package e2e + +import "testing" + +// WithCgroupManagers is a wrapper to call test function f in both the systemd and +// cgroupfs cgroup manager configurations. It *must* be run noparallel, as the +// cgroup manager setting is set / read from global configuration. +func (env TestEnv) WithCgroupManagers(f func(t *testing.T)) func(t *testing.T) { + return func(t *testing.T) { + env.RunSingularity( + t, + WithProfile(RootProfile), + WithCommand("config global"), + WithArgs("--set", "systemd cgroups", "yes"), + ExpectExit(0), + ) + + defer env.RunSingularity( + t, + WithProfile(RootProfile), + WithCommand("config global"), + WithArgs("--reset", "systemd cgroups"), + ExpectExit(0), + ) + + t.Run("systemd", f) + + env.RunSingularity( + t, + WithProfile(RootProfile), + WithCommand("config global"), + WithArgs("--set", "systemd cgroups", "no"), + ExpectExit(0), + ) + + t.Run("cgroupfs", f) + } +} diff --git a/e2e/internal/e2e/config.go b/e2e/internal/e2e/config.go index 0e79966d92..cd3a732e16 100644 --- a/e2e/internal/e2e/config.go +++ b/e2e/internal/e2e/config.go @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2021, Sylabs Inc. All rights reserved. +// Copyright (c) 2019-2022, Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -20,6 +20,8 @@ func SetupDefaultConfig(t *testing.T, path string) { t.Fatalf("while generating singularity configuration: %s", err) } + t.Logf("systemd cgroups: %v", c.SystemdCgroups) + // e2e tests should call the specific external binaries found/coonfigured in the build. // Set default external paths from build time values c.CryptsetupPath = buildcfg.CRYPTSETUP_PATH @@ -28,12 +30,6 @@ func SetupDefaultConfig(t *testing.T, path string) { c.MksquashfsPath = buildcfg.MKSQUASHFS_PATH c.NvidiaContainerCliPath = buildcfg.NVIDIA_CONTAINER_CLI_PATH c.UnsquashfsPath = buildcfg.UNSQUASHFS_PATH - // FIXME - // The e2e tests currently run inside a PID namespace. - // (see internal/init/init_linux.go) - // We can't instruct systemd to manage our cgroups as the PIDs in our test namespace - // won't match what systemd sees. - c.SystemdCgroups = false Privileged(func(t *testing.T) { f, err := os.Create(path) diff --git a/e2e/internal/e2e/init/init_linux.go b/e2e/internal/e2e/init/init_linux.go index 0ab738bc54..595b3ac006 100644 --- a/e2e/internal/e2e/init/init_linux.go +++ b/e2e/internal/e2e/init/init_linux.go @@ -1,4 +1,4 @@ -// Copyright (c) 2019, Sylabs Inc. All rights reserved. +// Copyright (c) 2019, 2022 Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -151,8 +151,14 @@ __attribute__((constructor)) static void init(void) { exit(1); } + fprintf(stderr, "Creating E2E mount namespace\n"); create_mount_namespace(); - create_pid_namespace(); + + char *s = getenv("SINGULARITY_E2E_NO_PID_NS"); + if ( s == NULL || s[0] == '\0' ) { + fprintf(stderr, "Creating E2E PID namespace\n"); + create_pid_namespace(); + } // set original user identity and retain privileges for // Privileged method diff --git a/e2e/oci/oci.go b/e2e/oci/oci.go index bf0df46c4f..5ae4b64411 100644 --- a/e2e/oci/oci.go +++ b/e2e/oci/oci.go @@ -1,4 +1,4 @@ -// Copyright (c) 2019-2021, Sylabs Inc. All rights reserved. +// Copyright (c) 2019-2022 Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -19,6 +19,16 @@ import ( "github.com/sylabs/singularity/pkg/ociruntime" ) +// NOTE +// ---- +// Tests in this package/topic are run in a a mount namespace only. There is +// no PID namespace, in order that the systemd cgroups manager functionality +// can be exercised. +// +// You must take extra care not to leave detached process etc. that will +// pollute the host PID namespace. +// + func randomContainerID(t *testing.T) string { t.Helper() @@ -400,9 +410,12 @@ func E2ETests(env e2e.TestEnv) testhelper.Tests { } return testhelper.Tests{ - "basic": c.testOciBasic, - "attach": c.testOciAttach, - "run": c.testOciRun, - "help": c.testOciHelp, + "ordered": testhelper.NoParallel( + env.WithCgroupManagers(func(t *testing.T) { + t.Run("basic", c.testOciBasic) + t.Run("attach", c.testOciAttach) + t.Run("run", c.testOciRun) + t.Run("help", c.testOciHelp) + })), } } diff --git a/e2e/suite.go b/e2e/suite.go index f95ab58f30..5beda57c26 100644 --- a/e2e/suite.go +++ b/e2e/suite.go @@ -1,5 +1,5 @@ // Copyright (c) 2020, Control Command Inc. All rights reserved. -// Copyright (c) 2019,2020 Sylabs Inc. All rights reserved. +// Copyright (c) 2019-2022 Sylabs Inc. All rights reserved. // This software is licensed under a 3-clause BSD license. Please consult the // LICENSE.md file distributed with the sources of this project regarding your // rights to use or distribute this software. @@ -22,6 +22,7 @@ import ( "github.com/sylabs/singularity/e2e/actions" e2ebuildcfg "github.com/sylabs/singularity/e2e/buildcfg" "github.com/sylabs/singularity/e2e/cache" + "github.com/sylabs/singularity/e2e/cgroups" "github.com/sylabs/singularity/e2e/cmdenvvars" "github.com/sylabs/singularity/e2e/config" "github.com/sylabs/singularity/e2e/delete" @@ -163,9 +164,15 @@ func Run(t *testing.T) { suite := testhelper.NewSuite(t, testenv) - // RunE2ETests by functionality. - // - // Please keep this list sorted. + if os.Getenv("SINGULARITY_E2E_NO_PID_NS") != "" { + // e2e tests that will run in a mount namespace only + suite.AddGroup("CGROUPS", cgroups.E2ETests) + suite.AddGroup("OCI", oci.E2ETests) + suite.Run() + return + } + + // e2e tests that will run in a mount and PID namespace suite.AddGroup("ACTIONS", actions.E2ETests) suite.AddGroup("BUILDCFG", e2ebuildcfg.E2ETests) suite.AddGroup("BUILD", imgbuild.E2ETests) @@ -181,7 +188,6 @@ func Run(t *testing.T) { suite.AddGroup("INSPECT", inspect.E2ETests) suite.AddGroup("INSTANCE", instance.E2ETests) suite.AddGroup("KEY", key.E2ETests) - suite.AddGroup("OCI", oci.E2ETests) suite.AddGroup("OVERLAY", overlay.E2ETests) suite.AddGroup("PLUGIN", plugin.E2ETests) suite.AddGroup("PULL", pull.E2ETests) @@ -193,6 +199,5 @@ func Run(t *testing.T) { suite.AddGroup("SIGN", sign.E2ETests) suite.AddGroup("VERIFY", verify.E2ETests) suite.AddGroup("VERSION", version.E2ETests) - suite.Run() } diff --git a/mlocal/frags/Makefile.stub b/mlocal/frags/Makefile.stub index dfceaa7014..805d6918da 100644 --- a/mlocal/frags/Makefile.stub +++ b/mlocal/frags/Makefile.stub @@ -71,12 +71,19 @@ short-integration-test: .PHONY: e2e-test e2e-test: EXTRA_FLAGS := $(if $(filter yes,$(strip $(JUNIT_OUTPUT))),-junit $(BUILDDIR_ABSPATH)/e2e-test.xml) e2e-test: - @echo " TEST sudo go test [e2e]" + # Run the majority of e2e tests, which will use a separate pid and mount namespace + @echo " TEST sudo go test [e2e pid+mount ns]" $(V)rm -f $(BUILDDIR_ABSPATH)/e2e-cmd-coverage $(V)cd $(SOURCEDIR) && \ SINGULARITY_E2E_COVERAGE=$(BUILDDIR_ABSPATH)/e2e-cmd-coverage \ scripts/e2e-test -v $(GO_RACE) $(EXTRA_FLAGS) @echo " PASS" + # Run the remaining e2e tests, which need to be in the host pid namespace + @echo " TEST sudo go test [e2e mount ns only]" + $(V)cd $(SOURCEDIR) && \ + SINGULARITY_E2E_NO_PID_NS=1 \ + scripts/e2e-test -v $(GO_RACE) $(EXTRA_FLAGS) + @echo " PASS" @echo " TEST e2e-coverage" $(V)cd $(SOURCEDIR) && \ $(GO) run $(GO_MODFLAGS) -tags "$(GO_TAGS)" $(GO_GCFLAGS) $(GO_ASMFLAGS) \ diff --git a/scripts/e2e-test b/scripts/e2e-test index 80c30a5b49..5a54a87637 100755 --- a/scripts/e2e-test +++ b/scripts/e2e-test @@ -30,5 +30,5 @@ fi proxy_vars="HTTP_PROXY=$HTTP_PROXY HTTPS_PROXY=$HTTPS_PROXY ALL_PROXY=$ALL_PROXY NO_PROXY=$NO_PROXY" cred_vars="E2E_DOCKER_USERNAME=$E2E_DOCKER_USERNAME E2E_DOCKER_PASSWORD=$E2E_DOCKER_PASSWORD" -export sudo_args="env -i PATH=$PATH HOME=$HOME $proxy_vars $cred_vars SINGULARITY_E2E_COVERAGE=$SINGULARITY_E2E_COVERAGE" +export sudo_args="env -i PATH=$PATH HOME=$HOME $proxy_vars $cred_vars SINGULARITY_E2E_COVERAGE=$SINGULARITY_E2E_COVERAGE SINGULARITY_E2E_NO_PID_NS=$SINGULARITY_E2E_NO_PID_NS" exec scripts/go-test -sudo -parallel $procs -tags "e2e_test" "$@" ./e2e