From b0a82cb8ecbd5ce57423368b66a54674f4fd9063 Mon Sep 17 00:00:00 2001 From: Maksim An Date: Wed, 5 Apr 2023 09:04:23 -0700 Subject: [PATCH] con-con: write policy, reference info and cert to container's rootfs (#1708) Due to `execve` limitation on the size of environment variable, write the base64 encoded security policy, UVM reference info and host AMD certificate to container's rootfs. Update existing test accordingly. Signed-off-by: Maksim An --- internal/guest/runtime/hcsv2/uvm.go | 77 +++++++++++++++++++--------- pkg/annotations/annotations.go | 6 ++- pkg/securitypolicy/securitypolicy.go | 7 +++ test/cri-containerd/policy_test.go | 65 ++++++++++++----------- 4 files changed, 96 insertions(+), 59 deletions(-) diff --git a/internal/guest/runtime/hcsv2/uvm.go b/internal/guest/runtime/hcsv2/uvm.go index 431ef416e3..d987523bcd 100644 --- a/internal/guest/runtime/hcsv2/uvm.go +++ b/internal/guest/runtime/hcsv2/uvm.go @@ -437,32 +437,47 @@ func (h *Host) CreateContainer(ctx context.Context, id string, settings *prot.VM settings.OCISpecification.Process.Capabilities = capsToKeep } - // Export security policy and signed UVM reference info as one of the - // process's environment variables so that application and sidecar - // containers can have access to it. The security policy is required - // by containers which need to extract init-time claims found in the - // security policy. - // - // We append the variable after the security policy enforcing logic - // completes to bypass it; the security policy variable cannot be included - // in the security policy as its value is not available security policy - // construction time. - - // It may be an error to have a security policy but not expose it to the container as - // in that case it can never be checked as correct by a verifier. + // Write security policy, signed UVM reference and host AMD certificate to + // container's rootfs, so that application and sidecar containers can have + // access to it. The security policy is required by containers which need to + // extract init-time claims found in the security policy. The directory path + // containing the files is exposed via UVM_SECURITY_CONTEXT_DIR env var. + // It may be an error to have a security policy but not expose it to the + // container as in that case it can never be checked as correct by a verifier. if oci.ParseAnnotationsBool(ctx, settings.OCISpecification.Annotations, annotations.UVMSecurityPolicyEnv, true) { encodedPolicy := h.securityPolicyEnforcer.EncodedSecurityPolicy() - if len(encodedPolicy) > 0 { - secPolicyEnv := fmt.Sprintf("UVM_SECURITY_POLICY=%s", encodedPolicy) - settings.OCISpecification.Process.Env = append(settings.OCISpecification.Process.Env, secPolicyEnv) - } - if len(h.uvmReferenceInfo) > 0 { - uvmReferenceInfo := fmt.Sprintf("UVM_REFERENCE_INFO=%s", h.uvmReferenceInfo) - settings.OCISpecification.Process.Env = append(settings.OCISpecification.Process.Env, uvmReferenceInfo) - } - if len(settings.OCISpecification.Annotations[annotations.HostAMDCertificate]) > 0 { - amdCertEnv := fmt.Sprintf("UVM_HOST_AMD_CERTIFICATE=%s", settings.OCISpecification.Annotations[annotations.HostAMDCertificate]) - settings.OCISpecification.Process.Env = append(settings.OCISpecification.Process.Env, amdCertEnv) + hostAMDCert := settings.OCISpecification.Annotations[annotations.HostAMDCertificate] + if len(encodedPolicy) > 0 || len(hostAMDCert) > 0 || len(h.uvmReferenceInfo) > 0 { + // Use os.MkdirTemp to make sure that the directory is unique. + securityContextDir, err := os.MkdirTemp(settings.OCISpecification.Root.Path, securitypolicy.SecurityContextDirTemplate) + if err != nil { + return nil, fmt.Errorf("failed to create security context directory: %w", err) + } + // Make sure it's readable + if err := os.Chmod(securityContextDir, 0744); err != nil { + return nil, fmt.Errorf("failed to chmod security context directory: %w", err) + } + + if len(encodedPolicy) > 0 { + if err := writeFileInDir(securityContextDir, securitypolicy.PolicyFilename, []byte(encodedPolicy), 0744); err != nil { + return nil, fmt.Errorf("failed to write security policy: %w", err) + } + } + if len(h.uvmReferenceInfo) > 0 { + if err := writeFileInDir(securityContextDir, securitypolicy.ReferenceInfoFilename, []byte(h.uvmReferenceInfo), 0744); err != nil { + return nil, fmt.Errorf("failed to write UVM reference info: %w", err) + } + } + + if len(hostAMDCert) > 0 { + if err := writeFileInDir(securityContextDir, securitypolicy.HostAMDCertFilename, []byte(hostAMDCert), 0744); err != nil { + return nil, fmt.Errorf("failed to write host AMD certificate: %w", err) + } + } + + containerCtxDir := fmt.Sprintf("/%s", filepath.Base(securityContextDir)) + secCtxEnv := fmt.Sprintf("UVM_SECURITY_CONTEXT_DIR=%s", containerCtxDir) + settings.OCISpecification.Process.Env = append(settings.OCISpecification.Process.Env, secCtxEnv) } } @@ -1131,3 +1146,17 @@ func processOCIEnvToParam(envs []string) map[string]string { func isPrivilegedContainerCreationRequest(ctx context.Context, spec *specs.Spec) bool { return oci.ParseAnnotationsBool(ctx, spec.Annotations, annotations.LCOWPrivileged, false) } + +func writeFileInDir(dir string, filename string, data []byte, perm os.FileMode) error { + st, err := os.Stat(dir) + if err != nil { + return err + } + + if !st.IsDir() { + return fmt.Errorf("not a directory %q", dir) + } + + targetFilename := filepath.Join(dir, filename) + return os.WriteFile(targetFilename, data, perm) +} diff --git a/pkg/annotations/annotations.go b/pkg/annotations/annotations.go index 74ba7cc379..fb3b3d99a0 100644 --- a/pkg/annotations/annotations.go +++ b/pkg/annotations/annotations.go @@ -271,8 +271,10 @@ const ( // GuestStateFile specifies the path of the vmgs file to use if required. Only applies in SNP mode. GuestStateFile = "io.microsoft.virtualmachine.lcow.gueststatefile" - // UVMSecurityPolicyEnv specifies if UVM_SECURITY_POLICY, UVM_REFERENCE_INFO - // and UVM_HOST_AMD_CERTIFICATE variables should be injected for a container. + // UVMSecurityPolicyEnv specifies if confidential containers' related information + // should be written to containers' rootfs. The filenames and location are defined + // by securitypolicy.PolicyFilename, securitypolicy.HostAMDCertFilename and + // securitypolicy.ReferenceInfoFilename. UVMSecurityPolicyEnv = "io.microsoft.virtualmachine.lcow.securitypolicy.env" // UVMReferenceInfoFile specifies the filename of a signed UVM reference file to be passed to UVM. diff --git a/pkg/securitypolicy/securitypolicy.go b/pkg/securitypolicy/securitypolicy.go index 3b7ceb1132..a3295a861c 100644 --- a/pkg/securitypolicy/securitypolicy.go +++ b/pkg/securitypolicy/securitypolicy.go @@ -45,6 +45,13 @@ const ( const plan9Prefix = "plan9://" +const ( + SecurityContextDirTemplate = "security-context-*" + PolicyFilename = "security-policy-base64" + HostAMDCertFilename = "host-amd-cert-base64" + ReferenceInfoFilename = "reference-info-base64" +) + // PolicyConfig contains toml or JSON config for security policy. type PolicyConfig struct { AllowAll bool `json:"allow_all" toml:"allow_all"` diff --git a/test/cri-containerd/policy_test.go b/test/cri-containerd/policy_test.go index 2c084b4728..304ea0242a 100644 --- a/test/cri-containerd/policy_test.go +++ b/test/cri-containerd/policy_test.go @@ -777,13 +777,16 @@ func Test_RunContainer_WithPolicy_And_SecurityPolicyEnv_Annotation(t *testing.T) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - // The command prints environment variables to stdout, which we can capture - // and validate later - alpineCmd := []string{"ash", "-c", "env && sleep 1"} - + alpineCmd := []string{"ash", "-c", "sleep 10"} + listDirCmd := []string{"ash", "-c", fmt.Sprintf("ls -l /%s", securitypolicy.SecurityContextDirTemplate)} opts := []securitypolicy.ContainerConfigOpt{ securitypolicy.WithCommand(alpineCmd), securitypolicy.WithAllowStdioAccess(true), + securitypolicy.WithExecProcesses([]securitypolicy.ExecProcessConfig{ + { + Command: listDirCmd, + }, + }), } for _, config := range []struct { name string @@ -793,10 +796,6 @@ func Test_RunContainer_WithPolicy_And_SecurityPolicyEnv_Annotation(t *testing.T) name: "OpenDoorPolicy", policy: openDoorPolicy, }, - { - name: "StandardPolicy", - policy: alpineSecurityPolicy(t, "json", false, false, opts...), - }, { name: "RegoPolicy", policy: alpineSecurityPolicy(t, "rego", false, false, opts...), @@ -830,42 +829,42 @@ func Test_RunContainer_WithPolicy_And_SecurityPolicyEnv_Annotation(t *testing.T) } } - // setup logfile to capture stdout - logPath := filepath.Join(t.TempDir(), "log.txt") - containerRequest.Config.LogPath = logPath - containerID := createContainer(t, client, ctx, containerRequest) defer removeContainer(t, client, ctx, containerID) startContainer(t, client, ctx, containerID) requireContainerState(ctx, t, client, containerID, runtime.ContainerState_CONTAINER_RUNNING) - // no need to stop the container since it'll exit by itself - requireContainerState(ctx, t, client, containerID, runtime.ContainerState_CONTAINER_EXITED) - - content, err := os.ReadFile(logPath) - if err != nil { - t.Fatalf("error reading log file: %s", err) - } - targetEnvs := []string{ - fmt.Sprintf("UVM_SECURITY_POLICY=%s", config.policy), - "UVM_REFERENCE_INFO=", - fmt.Sprintf("UVM_HOST_AMD_CERTIFICATE=%s", certValue), - } + r := execSync(t, client, ctx, &runtime.ExecSyncRequest{ + ContainerId: containerID, + Cmd: listDirCmd, + }) + stdout := string(r.Stdout) if setPolicyEnv { - // make sure that the expected environment variable was set - for _, env := range targetEnvs { - if !strings.Contains(string(content), env) { - t.Fatalf("missing init process environment variable: %s", env) - } + if r.ExitCode != 0 { + t.Log(string(r.Stderr)) + t.Fatalf("unexpected exec exit code: %d", r.ExitCode) + } + if !strings.Contains(stdout, securitypolicy.PolicyFilename) { + t.Log(stdout) + t.Fatalf("security policy file not found: %s", securitypolicy.PolicyFilename) + } + if !strings.Contains(stdout, securitypolicy.HostAMDCertFilename) { + t.Log(stdout) + t.Fatalf("AMD certificate file not found: %s", securitypolicy.HostAMDCertFilename) } } else { - for _, env := range targetEnvs { - if strings.Contains(string(content), env) { - t.Fatalf("environment variable should not be set for init process: %s", env) - } + if r.ExitCode != 1 { + t.Log(stdout) + t.Fatalf("unexpected exec exit code: %d", r.ExitCode) + } + if !strings.Contains(string(r.Stderr), "No such file") { + t.Fatalf("expected different error output, got:\n %s", string(r.Stderr)) } } + + // no need to stop the container since it'll exit by itself + requireContainerState(ctx, t, client, containerID, runtime.ContainerState_CONTAINER_EXITED) }) } }