add support for verity checking partitioned disks (microsoft#1810)

Add an option to mount partitioned disks with dmverity.

Additionally add support for reading verity information from
within the guest. The expectation is that verity hash device
is appended to the read-only file system. The functionality
can be enabled by passing a container annotation.

Host no longer reads verity superblock and as a result
the `DeviceVerityInfo` protocol message is being
deprecated. The guest will always attempt to read verity
super-block when non-empty security policy is passed.
Security policy is expected to be empty only in regular
LCOW scenarios.

Signed-off-by: Maksim An <maksiman@microsoft.com>

cmd/containerd-shim-runhcs-v1/rootfs.go

@@ -80,7 +80,12 @@ func getLCOWLayers(rootfs []*types.Mount, layerFolders []string) (*layers.LCOWLa
 		// Each read-only layer should have a layer.vhd, and the scratch layer should have a sandbox.vhdx.
 		roLayers := make([]*layers.LCOWLayer, 0, len(parentLayers))
 		for _, parentLayer := range parentLayers {
-			roLayers = append(roLayers, &layers.LCOWLayer{VHDPath: filepath.Join(parentLayer, "layer.vhd")})
+			roLayers = append(
+				roLayers,
+				&layers.LCOWLayer{
+					VHDPath: filepath.Join(parentLayer, "layer.vhd"),
+				},
+			)
 		}
 		return &layers.LCOWLayers{
 			Layers:         roLayers,
@@ -122,7 +127,13 @@ func getLCOWLayers(rootfs []*types.Mount, layerFolders []string) (*layers.LCOWLa
 		}
 		roLayers := make([]*layers.LCOWLayer, 0, len(layerData))
 		for _, layer := range layerData {
-			roLayers = append(roLayers, &layers.LCOWLayer{VHDPath: layer.Path, Partition: layer.Partition})
+			roLayers = append(
+				roLayers,
+				&layers.LCOWLayer{
+					VHDPath:   layer.Path,
+					Partition: layer.Partition,
+				},
+			)
 		}
 		return &layers.LCOWLayers{Layers: roLayers, ScratchVHDPath: scratchPath}, nil
 	default:

cmd/containerd-shim-runhcs-v1/task_hcs.go

@@ -11,11 +11,11 @@ import (
 	"time"
 
 	eventstypes "github.com/containerd/containerd/api/events"
-	task "github.com/containerd/containerd/api/runtime/task/v2"
+	"github.com/containerd/containerd/api/runtime/task/v2"
 	"github.com/containerd/containerd/api/types"
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/runtime"
-	typeurl "github.com/containerd/typeurl/v2"
+	"github.com/containerd/typeurl/v2"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"

internal/guest/runtime/hcsv2/uvm.go

@@ -40,6 +40,7 @@ import (
 	"github.com/Microsoft/hcsshim/internal/oci"
 	"github.com/Microsoft/hcsshim/internal/protocol/guestrequest"
 	"github.com/Microsoft/hcsshim/internal/protocol/guestresource"
+	"github.com/Microsoft/hcsshim/internal/verity"
 	"github.com/Microsoft/hcsshim/pkg/annotations"
 	"github.com/Microsoft/hcsshim/pkg/securitypolicy"
 	"github.com/mattn/go-shellwords"
@@ -967,26 +968,41 @@ func modifyMappedVirtualDisk(
 	mvd *guestresource.LCOWMappedVirtualDisk,
 	securityPolicy securitypolicy.SecurityPolicyEnforcer,
 ) (err error) {
+	var verityInfo *guestresource.DeviceVerityInfo
+	if mvd.ReadOnly {
+		// The only time the policy is empty, and we want it to be empty
+		// is when no policy is provided, and we default to open door
+		// policy. In any other case, e.g. explicit open door or any
+		// other rego policy we would like to mount layers with verity.
+		if len(securityPolicy.EncodedSecurityPolicy()) > 0 {
+			devPath, err := scsi.GetDevicePath(ctx, mvd.Controller, mvd.Lun, mvd.Partition)
+			if err != nil {
+				return err
+			}
+			verityInfo, err = verity.ReadVeritySuperBlock(ctx, devPath)
+			if err != nil {
+				return err
+			}
+		}
+	}
 	switch rt {
 	case guestrequest.RequestTypeAdd:
 		mountCtx, cancel := context.WithTimeout(ctx, time.Second*5)
 		defer cancel()
 		if mvd.MountPath != "" {
 			if mvd.ReadOnly {
-				// containers only have read-only layers so only enforce for them
 				var deviceHash string
-				if mvd.VerityInfo != nil {
-					deviceHash = mvd.VerityInfo.RootDigest
+				if verityInfo != nil {
+					deviceHash = verityInfo.RootDigest
 				}
-
 				err = securityPolicy.EnforceDeviceMountPolicy(ctx, mvd.MountPath, deviceHash)
 				if err != nil {
 					return errors.Wrapf(err, "mounting scsi device controller %d lun %d onto %s denied by policy", mvd.Controller, mvd.Lun, mvd.MountPath)
 				}
 			}
 			config := &scsi.Config{
 				Encrypted:        mvd.Encrypted,
-				VerityInfo:       mvd.VerityInfo,
+				VerityInfo:       verityInfo,
 				EnsureFilesystem: mvd.EnsureFilesystem,
 				Filesystem:       mvd.Filesystem,
 			}
@@ -1003,7 +1019,7 @@ func modifyMappedVirtualDisk(
 			}
 			config := &scsi.Config{
 				Encrypted:        mvd.Encrypted,
-				VerityInfo:       mvd.VerityInfo,
+				VerityInfo:       verityInfo,
 				EnsureFilesystem: mvd.EnsureFilesystem,
 				Filesystem:       mvd.Filesystem,
 			}
@@ -1050,24 +1066,32 @@ func modifyMappedVPMemDevice(ctx context.Context,
 	vpd *guestresource.LCOWMappedVPMemDevice,
 	securityPolicy securitypolicy.SecurityPolicyEnforcer,
 ) (err error) {
+	var verityInfo *guestresource.DeviceVerityInfo
+	var deviceHash string
+	if len(securityPolicy.EncodedSecurityPolicy()) > 0 {
+		if vpd.MappingInfo != nil {
+			return fmt.Errorf("multi mapping is not supported with verity")
+		}
+		verityInfo, err = verity.ReadVeritySuperBlock(ctx, pmem.GetDevicePath(vpd.DeviceNumber))
+		if err != nil {
+			return err
+		}
+		deviceHash = verityInfo.RootDigest
+	}
 	switch rt {
 	case guestrequest.RequestTypeAdd:
-		var deviceHash string
-		if vpd.VerityInfo != nil {
-			deviceHash = vpd.VerityInfo.RootDigest
-		}
 		err = securityPolicy.EnforceDeviceMountPolicy(ctx, vpd.MountPath, deviceHash)
 		if err != nil {
 			return errors.Wrapf(err, "mounting pmem device %d onto %s denied by policy", vpd.DeviceNumber, vpd.MountPath)
 		}
 
-		return pmem.Mount(ctx, vpd.DeviceNumber, vpd.MountPath, vpd.MappingInfo, vpd.VerityInfo)
+		return pmem.Mount(ctx, vpd.DeviceNumber, vpd.MountPath, vpd.MappingInfo, verityInfo)
 	case guestrequest.RequestTypeRemove:
 		if err := securityPolicy.EnforceDeviceUnmountPolicy(ctx, vpd.MountPath); err != nil {
 			return errors.Wrapf(err, "unmounting pmem device from %s denied by policy", vpd.MountPath)
 		}
 
-		return pmem.Unmount(ctx, vpd.DeviceNumber, vpd.MountPath, vpd.MappingInfo, vpd.VerityInfo)
+		return pmem.Unmount(ctx, vpd.DeviceNumber, vpd.MountPath, vpd.MappingInfo, verityInfo)
 	default:
 		return newInvalidRequestTypeError(rt)
 	}

internal/guest/storage/pmem/pmem.go

@@ -56,6 +56,11 @@ func mount(ctx context.Context, source, target string) (err error) {
 	return nil
 }
 
+// GetDevicePath returns VPMem device path
+func GetDevicePath(devNumber uint32) string {
+	return fmt.Sprintf(pMemFmt, devNumber)
+}
+
 // Mount mounts the pmem device at `/dev/pmem<device>` to `target` in a basic scenario.
 // If either mappingInfo or verityInfo are non-nil, the device-mapper framework is used
 // to create linear and verity targets accordingly. If both are non-nil, the linear
@@ -84,7 +89,7 @@ func Mount(
 		trace.Int64Attribute("deviceNumber", int64(device)),
 		trace.StringAttribute("target", target))
 
-	devicePath := fmt.Sprintf(pMemFmt, device)
+	devicePath := GetDevicePath(device)
 
 	// dm-linear target has to be created first. When verity info is also present, the linear target becomes the data
 	// device instead of the original VPMem.

internal/guest/storage/scsi/scsi.go

@@ -146,33 +146,9 @@ func Mount(
 		return err
 	}
 
-	// The `source` found by GetDevicePath can take some time
-	// before its actually available under `/dev/sd*`. Retry while we
-	// wait for `source` to show up.
-	for {
-		if _, err := osStat(source); err != nil {
-			if errors.Is(err, fs.ErrNotExist) || errors.Is(err, unix.ENXIO) {
-				select {
-				case <-ctx.Done():
-					log.G(ctx).Warnf("context timed out while retrying to find device %s: %v", source, err)
-					return err
-				default:
-					time.Sleep(10 * time.Millisecond)
-					continue
-				}
-			}
-			return err
-		}
-		break
-	}
-
 	if readonly {
-		var deviceHash string
-		if config.VerityInfo != nil {
-			deviceHash = config.VerityInfo.RootDigest
-		}
-
 		if config.VerityInfo != nil {
+			deviceHash := config.VerityInfo.RootDigest
 			dmVerityName := fmt.Sprintf(verityDeviceFmt, controller, lun, partition, deviceHash)
 			if source, err = createVerityTarget(spnCtx, source, dmVerityName, config.VerityInfo); err != nil {
 				return err
@@ -328,7 +304,8 @@ func Unmount(
 }
 
 // GetDevicePath finds the `/dev/sd*` path to the SCSI device on `controller`
-// index `lun` with partition index `partition`.
+// index `lun` with partition index `partition` and also ensures that the device
+// is available under that path or context is canceled.
 func GetDevicePath(ctx context.Context, controller, lun uint8, partition uint64) (_ string, err error) {
 	ctx, span := oc.StartSpan(ctx, "scsi::GetDevicePath")
 	defer span.End()
@@ -398,6 +375,26 @@ func GetDevicePath(ctx context.Context, controller, lun uint8, partition uint64)
 
 	devicePath := filepath.Join("/dev", deviceName)
 	log.G(ctx).WithField("devicePath", devicePath).Debug("found device path")
+
+	// devicePath can take some time before its actually available under
+	// `/dev/sd*`. Retry while we wait for it to show up.
+	for {
+		if _, err := osStat(devicePath); err != nil {
+			if errors.Is(err, fs.ErrNotExist) || errors.Is(err, unix.ENXIO) {
+				select {
+				case <-ctx.Done():
+					log.G(ctx).Warnf("context timed out while retrying to find device %s: %v", devicePath, err)
+					return "", err
+				default:
+					time.Sleep(10 * time.Millisecond)
+					continue
+				}
+			}
+			return "", err
+		}
+		break
+	}
+
 	return devicePath, nil
 }
 

internal/guest/storage/scsi/scsi_test.go

@@ -1168,7 +1168,7 @@ func Test_GetDevicePath_Device_With_Partition_Error(t *testing.T) {
 	}
 }
 
-func Test_GetDevicePath_Device_No_Partition(t *testing.T) {
+func Test_GetDevicePath_Device_No_Partition_Retries_Stat(t *testing.T) {
 	clearTestDependencies()
 
 	deviceName := "sdd"
@@ -1179,8 +1179,17 @@ func Test_GetDevicePath_Device_No_Partition(t *testing.T) {
 		return []os.DirEntry{entry}, nil
 	}
 
+	callNum := 0
 	osStat = func(name string) (os.FileInfo, error) {
-		return nil, fmt.Errorf("should not make this call: %v", name)
+		if callNum == 0 {
+			callNum += 1
+			return nil, fs.ErrNotExist
+		}
+		if callNum == 1 {
+			callNum += 1
+			return nil, unix.ENXIO
+		}
+		return nil, nil
 	}
 
 	getDevicePath = GetDevicePath

internal/layers/layers.go

@@ -31,7 +31,7 @@ type LCOWLayer struct {
 	Partition uint64
 }
 
-// Defines a set of LCOW layers.
+// LCOWLayers defines a set of LCOW layers.
 // For future extensibility, the LCOWLayer type could be swapped for an interface,
 // and we could either call some method on the interface to "apply" it directly to the UVM,
 // or type cast it to the various types that we support, and use the one it matches.
@@ -128,8 +128,8 @@ func MountLCOWLayers(ctx context.Context, containerID string, layers *LCOWLayers
 		Encrypted: vm.ScratchEncryptionEnabled(),
 		// For scratch disks, we support formatting the disk if it is not already
 		// formatted.
-		EnsureFileystem: true,
-		Filesystem:      "ext4",
+		EnsureFilesystem: true,
+		Filesystem:       "ext4",
 	}
 	if vm.ScratchEncryptionEnabled() {
 		// Encrypted scratch devices are formatted with xfs
@@ -420,7 +420,16 @@ func addLCOWLayer(ctx context.Context, vm *uvm.UtilityVM, layer *LCOWLayer) (uvm
 		}
 	}
 
-	sm, err := vm.SCSIManager.AddVirtualDisk(ctx, layer.VHDPath, true, "", &scsi.MountConfig{Partition: layer.Partition, Options: []string{"ro"}})
+	sm, err := vm.SCSIManager.AddVirtualDisk(
+		ctx,
+		layer.VHDPath,
+		true,
+		"",
+		&scsi.MountConfig{
+			Partition: layer.Partition,
+			Options:   []string{"ro"},
+		},
+	)
 	if err != nil {
 		return "", nil, fmt.Errorf("failed to add SCSI layer: %s", err)
 	}

internal/protocol/guestresource/resources.go

@@ -78,13 +78,14 @@ type SCSIDevice struct {
 // LCOWMappedVirtualDisk represents a disk on the host which is mapped into a
 // directory in the guest in the V2 schema.
 type LCOWMappedVirtualDisk struct {
-	MountPath        string            `json:"MountPath,omitempty"`
-	Lun              uint8             `json:"Lun,omitempty"`
-	Controller       uint8             `json:"Controller,omitempty"`
-	Partition        uint64            `json:"Partition,omitempty"`
-	ReadOnly         bool              `json:"ReadOnly,omitempty"`
-	Encrypted        bool              `json:"Encrypted,omitempty"`
-	Options          []string          `json:"Options,omitempty"`
+	MountPath  string   `json:"MountPath,omitempty"`
+	Lun        uint8    `json:"Lun,omitempty"`
+	Controller uint8    `json:"Controller,omitempty"`
+	Partition  uint64   `json:"Partition,omitempty"`
+	ReadOnly   bool     `json:"ReadOnly,omitempty"`
+	Encrypted  bool     `json:"Encrypted,omitempty"`
+	Options    []string `json:"Options,omitempty"`
+	// Deprecated: verity info is read by the guest
 	VerityInfo       *DeviceVerityInfo `json:"VerityInfo,omitempty"`
 	EnsureFilesystem bool              `json:"EnsureFilesystem,omitempty"`
 	Filesystem       string            `json:"Filesystem,omitempty"`
@@ -112,6 +113,8 @@ type LCOWVPMemMappingInfo struct {
 
 // DeviceVerityInfo represents dm-verity metadata of a block device.
 // Most of the fields can be directly mapped to table entries https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/verity.html
+// Deprecated: verity info is now read inside the guest and this message will be
+// removed.
 type DeviceVerityInfo struct {
 	// Ext4SizeInBytes is the size of ext4 file system
 	Ext4SizeInBytes int64 `json:",omitempty"`
@@ -136,6 +139,7 @@ type LCOWMappedVPMemDevice struct {
 	// MappingInfo is used when multiple devices are mapped onto a single VPMem device
 	MappingInfo *LCOWVPMemMappingInfo `json:"MappingInfo,omitempty"`
 	// VerityInfo is used when the VPMem has read-only integrity protection enabled
+	// Deprecated: verity info is now read inside the guest.
 	VerityInfo *DeviceVerityInfo `json:"VerityInfo,omitempty"`
 }
 

internal/uvm/scsi/backend.go

@@ -174,8 +174,8 @@ func mountRequest(controller, lun uint, path string, config *mountConfig, osType
 		if controller != 0 {
 			return guestrequest.ModificationRequest{}, errors.New("WCOW only supports SCSI controller 0")
 		}
-		if config.encrypted || config.verity != nil || len(config.options) != 0 ||
-			config.ensureFileystem || config.filesystem != "" || config.partition != 0 {
+		if config.encrypted || len(config.options) != 0 ||
+			config.ensureFilesystem || config.filesystem != "" || config.partition != 0 {
 			return guestrequest.ModificationRequest{},
 				errors.New("WCOW does not support encrypted, verity, guest options, partitions, specifying mount filesystem, or ensuring filesystem on mounts")
 		}
@@ -192,8 +192,7 @@ func mountRequest(controller, lun uint, path string, config *mountConfig, osType
 			ReadOnly:         config.readOnly,
 			Encrypted:        config.encrypted,
 			Options:          config.options,
-			VerityInfo:       config.verity,
-			EnsureFilesystem: config.ensureFileystem,
+			EnsureFilesystem: config.ensureFilesystem,
 			Filesystem:       config.filesystem,
 		}
 	default:
@@ -219,7 +218,6 @@ func unmountRequest(controller, lun uint, path string, config *mountConfig, osTy
 			Lun:        uint8(lun),
 			Partition:  config.partition,
 			Controller: uint8(controller),
-			VerityInfo: config.verity,
 		}
 	default:
 		return guestrequest.ModificationRequest{}, fmt.Errorf("unsupported os type: %s", osType)