con-con: write policy, reference info and cert to container's rootfs (#…

…1708)

Due to `execve` limitation on the size of environment variable, write the
base64 encoded security policy, UVM reference info and host AMD certificate
to container's rootfs.

Update existing test accordingly.

Signed-off-by: Maksim An <maksiman@microsoft.com>

internal/guest/runtime/hcsv2/uvm.go

@@ -437,32 +437,47 @@ func (h *Host) CreateContainer(ctx context.Context, id string, settings *prot.VM
 		settings.OCISpecification.Process.Capabilities = capsToKeep
 	}
 
-	// Export security policy and signed UVM reference info as one of the
-	// process's environment variables so that application and sidecar
-	// containers can have access to it. The security policy is required
-	// by containers which need to extract init-time claims found in the
-	// security policy.
-	//
-	// We append the variable after the security policy enforcing logic
-	// completes to bypass it; the security policy variable cannot be included
-	// in the security policy as its value is not available security policy
-	// construction time.
-
-	// It may be an error to have a security policy but not expose it to the container as
-	// in that case it can never be checked as correct by a verifier.
+	// Write security policy, signed UVM reference and host AMD certificate to
+	// container's rootfs, so that application and sidecar containers can have
+	// access to it. The security policy is required by containers which need to
+	// extract init-time claims found in the security policy. The directory path
+	// containing the files is exposed via UVM_SECURITY_CONTEXT_DIR env var.
+	// It may be an error to have a security policy but not expose it to the
+	// container as in that case it can never be checked as correct by a verifier.
 	if oci.ParseAnnotationsBool(ctx, settings.OCISpecification.Annotations, annotations.UVMSecurityPolicyEnv, true) {
 		encodedPolicy := h.securityPolicyEnforcer.EncodedSecurityPolicy()
-		if len(encodedPolicy) > 0 {
-			secPolicyEnv := fmt.Sprintf("UVM_SECURITY_POLICY=%s", encodedPolicy)
-			settings.OCISpecification.Process.Env = append(settings.OCISpecification.Process.Env, secPolicyEnv)
-		}
-		if len(h.uvmReferenceInfo) > 0 {
-			uvmReferenceInfo := fmt.Sprintf("UVM_REFERENCE_INFO=%s", h.uvmReferenceInfo)
-			settings.OCISpecification.Process.Env = append(settings.OCISpecification.Process.Env, uvmReferenceInfo)
-		}
-		if len(settings.OCISpecification.Annotations[annotations.HostAMDCertificate]) > 0 {
-			amdCertEnv := fmt.Sprintf("UVM_HOST_AMD_CERTIFICATE=%s", settings.OCISpecification.Annotations[annotations.HostAMDCertificate])
-			settings.OCISpecification.Process.Env = append(settings.OCISpecification.Process.Env, amdCertEnv)
+		hostAMDCert := settings.OCISpecification.Annotations[annotations.HostAMDCertificate]
+		if len(encodedPolicy) > 0 || len(hostAMDCert) > 0 || len(h.uvmReferenceInfo) > 0 {
+			// Use os.MkdirTemp to make sure that the directory is unique.
+			securityContextDir, err := os.MkdirTemp(settings.OCISpecification.Root.Path, securitypolicy.SecurityContextDirTemplate)
+			if err != nil {
+				return nil, fmt.Errorf("failed to create security context directory: %w", err)
+			}
+			// Make sure it's readable
+			if err := os.Chmod(securityContextDir, 0744); err != nil {
+				return nil, fmt.Errorf("failed to chmod security context directory: %w", err)
+			}
+
+			if len(encodedPolicy) > 0 {
+				if err := writeFileInDir(securityContextDir, securitypolicy.PolicyFilename, []byte(encodedPolicy), 0744); err != nil {
+					return nil, fmt.Errorf("failed to write security policy: %w", err)
+				}
+			}
+			if len(h.uvmReferenceInfo) > 0 {
+				if err := writeFileInDir(securityContextDir, securitypolicy.ReferenceInfoFilename, []byte(h.uvmReferenceInfo), 0744); err != nil {
+					return nil, fmt.Errorf("failed to write UVM reference info: %w", err)
+				}
+			}
+
+			if len(hostAMDCert) > 0 {
+				if err := writeFileInDir(securityContextDir, securitypolicy.HostAMDCertFilename, []byte(hostAMDCert), 0744); err != nil {
+					return nil, fmt.Errorf("failed to write host AMD certificate: %w", err)
+				}
+			}
+
+			containerCtxDir := fmt.Sprintf("/%s", filepath.Base(securityContextDir))
+			secCtxEnv := fmt.Sprintf("UVM_SECURITY_CONTEXT_DIR=%s", containerCtxDir)
+			settings.OCISpecification.Process.Env = append(settings.OCISpecification.Process.Env, secCtxEnv)
 		}
 	}
 
@@ -1131,3 +1146,17 @@ func processOCIEnvToParam(envs []string) map[string]string {
 func isPrivilegedContainerCreationRequest(ctx context.Context, spec *specs.Spec) bool {
 	return oci.ParseAnnotationsBool(ctx, spec.Annotations, annotations.LCOWPrivileged, false)
 }
+
+func writeFileInDir(dir string, filename string, data []byte, perm os.FileMode) error {
+	st, err := os.Stat(dir)
+	if err != nil {
+		return err
+	}
+
+	if !st.IsDir() {
+		return fmt.Errorf("not a directory %q", dir)
+	}
+
+	targetFilename := filepath.Join(dir, filename)
+	return os.WriteFile(targetFilename, data, perm)
+}

pkg/annotations/annotations.go

@@ -271,8 +271,10 @@ const (
 	// GuestStateFile specifies the path of the vmgs file to use if required. Only applies in SNP mode.
 	GuestStateFile = "io.microsoft.virtualmachine.lcow.gueststatefile"
 
-	// UVMSecurityPolicyEnv specifies if UVM_SECURITY_POLICY, UVM_REFERENCE_INFO
-	// and UVM_HOST_AMD_CERTIFICATE variables should be injected for a container.
+	// UVMSecurityPolicyEnv specifies if confidential containers' related information
+	// should be written to containers' rootfs. The filenames and location are defined
+	// by securitypolicy.PolicyFilename, securitypolicy.HostAMDCertFilename and
+	// securitypolicy.ReferenceInfoFilename.
 	UVMSecurityPolicyEnv = "io.microsoft.virtualmachine.lcow.securitypolicy.env"
 
 	// UVMReferenceInfoFile specifies the filename of a signed UVM reference file to be passed to UVM.

pkg/securitypolicy/securitypolicy.go

@@ -45,6 +45,13 @@ const (
 
 const plan9Prefix = "plan9://"
 
+const (
+	SecurityContextDirTemplate = "security-context-*"
+	PolicyFilename             = "security-policy-base64"
+	HostAMDCertFilename        = "host-amd-cert-base64"
+	ReferenceInfoFilename      = "reference-info-base64"
+)
+
 // PolicyConfig contains toml or JSON config for security policy.
 type PolicyConfig struct {
 	AllowAll                         bool                    `json:"allow_all" toml:"allow_all"`

test/cri-containerd/policy_test.go

@@ -777,13 +777,16 @@ func Test_RunContainer_WithPolicy_And_SecurityPolicyEnv_Annotation(t *testing.T)
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
-	// The command prints environment variables to stdout, which we can capture
-	// and validate later
-	alpineCmd := []string{"ash", "-c", "env && sleep 1"}
-
+	alpineCmd := []string{"ash", "-c", "sleep 10"}
+	listDirCmd := []string{"ash", "-c", fmt.Sprintf("ls -l /%s", securitypolicy.SecurityContextDirTemplate)}
 	opts := []securitypolicy.ContainerConfigOpt{
 		securitypolicy.WithCommand(alpineCmd),
 		securitypolicy.WithAllowStdioAccess(true),
+		securitypolicy.WithExecProcesses([]securitypolicy.ExecProcessConfig{
+			{
+				Command: listDirCmd,
+			},
+		}),
 	}
 	for _, config := range []struct {
 		name   string
@@ -793,10 +796,6 @@ func Test_RunContainer_WithPolicy_And_SecurityPolicyEnv_Annotation(t *testing.T)
 			name:   "OpenDoorPolicy",
 			policy: openDoorPolicy,
 		},
-		{
-			name:   "StandardPolicy",
-			policy: alpineSecurityPolicy(t, "json", false, false, opts...),
-		},
 		{
 			name:   "RegoPolicy",
 			policy: alpineSecurityPolicy(t, "rego", false, false, opts...),
@@ -830,42 +829,42 @@ func Test_RunContainer_WithPolicy_And_SecurityPolicyEnv_Annotation(t *testing.T)
 					}
 				}
 
-				// setup logfile to capture stdout
-				logPath := filepath.Join(t.TempDir(), "log.txt")
-				containerRequest.Config.LogPath = logPath
-
 				containerID := createContainer(t, client, ctx, containerRequest)
 				defer removeContainer(t, client, ctx, containerID)
 
 				startContainer(t, client, ctx, containerID)
 				requireContainerState(ctx, t, client, containerID, runtime.ContainerState_CONTAINER_RUNNING)
 
-				// no need to stop the container since it'll exit by itself
-				requireContainerState(ctx, t, client, containerID, runtime.ContainerState_CONTAINER_EXITED)
-
-				content, err := os.ReadFile(logPath)
-				if err != nil {
-					t.Fatalf("error reading log file: %s", err)
-				}
-				targetEnvs := []string{
-					fmt.Sprintf("UVM_SECURITY_POLICY=%s", config.policy),
-					"UVM_REFERENCE_INFO=",
-					fmt.Sprintf("UVM_HOST_AMD_CERTIFICATE=%s", certValue),
-				}
+				r := execSync(t, client, ctx, &runtime.ExecSyncRequest{
+					ContainerId: containerID,
+					Cmd:         listDirCmd,
+				})
+				stdout := string(r.Stdout)
 				if setPolicyEnv {
-					// make sure that the expected environment variable was set
-					for _, env := range targetEnvs {
-						if !strings.Contains(string(content), env) {
-							t.Fatalf("missing init process environment variable: %s", env)
-						}
+					if r.ExitCode != 0 {
+						t.Log(string(r.Stderr))
+						t.Fatalf("unexpected exec exit code: %d", r.ExitCode)
+					}
+					if !strings.Contains(stdout, securitypolicy.PolicyFilename) {
+						t.Log(stdout)
+						t.Fatalf("security policy file not found: %s", securitypolicy.PolicyFilename)
+					}
+					if !strings.Contains(stdout, securitypolicy.HostAMDCertFilename) {
+						t.Log(stdout)
+						t.Fatalf("AMD certificate file not found: %s", securitypolicy.HostAMDCertFilename)
 					}
 				} else {
-					for _, env := range targetEnvs {
-						if strings.Contains(string(content), env) {
-							t.Fatalf("environment variable should not be set for init process: %s", env)
-						}
+					if r.ExitCode != 1 {
+						t.Log(stdout)
+						t.Fatalf("unexpected exec exit code: %d", r.ExitCode)
+					}
+					if !strings.Contains(string(r.Stderr), "No such file") {
+						t.Fatalf("expected different error output, got:\n %s", string(r.Stderr))
 					}
 				}
+
+				// no need to stop the container since it'll exit by itself
+				requireContainerState(ctx, t, client, containerID, runtime.ContainerState_CONTAINER_EXITED)
 			})
 		}
 	}