Btrfs snapshotter implementation (#1957)

* Implementation of Btrfs snapshotter
* Btrfs based examples
* Refined and adapt features
* Update build-disk to new snapshotter and prevent including State partition on expandable images
* Remove /oem bind mount in initramfs, already mounted by mount command
* Adapt unit tests
* Add mount command unit tests
* Make grubfallback test more generic
* Adding btrfs snapshotter unit tests and fixing default snapshotter config constructor
* Add utils test
* Fix upgrade ENV variables mapping
* Include transactional-update package in example
* Fix persistent bind mounts
* Make sure state is RW mounted upgrading from legacy
* Remove unused passive symlinks for loopdevice
* Fix upgrade from older version

Signed-off-by: David Cassany <dcassany@suse.com>

.obs/specfile/elemental-toolkit.spec

@@ -34,7 +34,9 @@ Requires:       parted
 Requires:       rsync
 Requires:       udev
 Requires:       xfsprogs
-Requires:       xorriso >= 1.5.6
+Requires:       btrfsprogs
+Requires:       snapper
+Requires:       xorriso >= 1.5
 Requires:       mtools
 Requires:       util-linux
 Requires:       gptfdisk

cmd/config/config.go

@@ -301,7 +301,7 @@ func applyKernelCmdline(r *v1.RunConfig, mount *v1.MountSpec) error {
 		}
 
 		switch split[0] {
-		case "elemental.image", "cos-img/filename":
+		case "elemental.mode":
 			switch {
 			case strings.Contains(val, constants.ActiveImgName):
 				mount.Mode = constants.ActiveImgName
@@ -313,14 +313,14 @@ func applyKernelCmdline(r *v1.RunConfig, mount *v1.MountSpec) error {
 				r.Logger.Errorf("Error parsing cmdline %s", cmd)
 				return fmt.Errorf("Unknown image path: %s", val)
 			}
-		case "elemental.disable", "rd.cos.disable":
+		case "elemental.disable":
 			mount.Disable = true
-		case "elemental.overlay", "rd.cos.overlay":
+		case "elemental.overlay":
 			err := applyMountOverlay(mount, val)
 			if err != nil {
 				return err
 			}
-		case "elemental.oemlabel", "rd.cos.oemlabel":
+		case "elemental.oemlabel":
 			oemdev := fmt.Sprintf("LABEL=%s", val)
 			var mnt *v1.VolumeMount
 			for _, mnt = range mount.Volumes {

cmd/config/config_test.go

@@ -288,7 +288,7 @@ var _ = Describe("Config", Label("config"), func() {
 
 			err = fs.Mkdir("/proc", constants.DirPerm)
 			Expect(err).Should(BeNil())
-			err = fs.WriteFile("/proc/cmdline", []byte("root=LABEL=COS_STATE elemental.image=active elemental.overlay=tmpfs:30%"), 0444)
+			err = fs.WriteFile("/proc/cmdline", []byte("root=LABEL=COS_STATE elemental.mode=active elemental.overlay=tmpfs:30%"), 0444)
 			Expect(err).Should(BeNil())
 
 			cfg, err = ReadConfigRun("fixtures/config/", nil, mounter)

examples/green/Dockerfile

@@ -43,6 +43,10 @@ RUN ARCH=$(uname -m); \
       sudo \
       curl \
       iproute2 \
+      btrfsprogs \
+      btrfsmaintenance \
+      snapper \
+      transactional-update \
       podman \
       sed && \
     zypper clean --all
@@ -62,6 +66,9 @@ ADD 05_network.yaml /system/oem/05_network.yaml
 # Generate initrd with required elemental services
 RUN elemental init --debug --force
 
+# Add default snapshotter setup
+ADD snapshotter.yaml /etc/elemental/config.d/snapshotter.yaml
+
 # Update os-release file with some metadata
 RUN echo IMAGE_REPO=\"${REPO}\"         >> /etc/os-release && \
     echo IMAGE_TAG=\"${VERSION}\"           >> /etc/os-release && \

examples/green/snapshotter.yaml

@@ -0,0 +1,3 @@
+snapshotter:
+  type: btrfs
+  max-snaps: 4

pkg/action/build-disk.go

@@ -80,7 +80,18 @@ func NewBuildDiskAction(cfg *v1.BuildConfig, spec *v1.DiskSpec, opts ...BuildDis
 	}
 
 	if b.snapshotter == nil {
-		b.snapshotter, err = snapshotter.NewLoopDeviceSnapshotter(cfg.Config, cfg.Snapshotter, b.bootloader)
+		b.snapshotter, err = snapshotter.NewSnapshotter(cfg.Config, cfg.Snapshotter, b.bootloader)
+	}
+
+	if b.cfg.Snapshotter.Type == constants.BtrfsSnapshotterType {
+		if !b.spec.Expandable {
+			cfg.Logger.Errorf("Non expandable disk images are not supported for btrfs snapshotter")
+			return nil, fmt.Errorf("Not supported")
+		}
+		if spec.Partitions.State.FS != constants.Btrfs {
+			cfg.Logger.Warning("Btrfs snapshotter type, forcing btrfs filesystem on state partition")
+			spec.Partitions.State.FS = constants.Btrfs
+		}
 	}
 
 	return b, err
@@ -104,14 +115,14 @@ func (b *BuildDiskAction) buildDiskChrootHook(hook string, root string) error {
 
 func (b *BuildDiskAction) preparePartitionsRoot() error {
 	var err error
-	var exclude *v1.Partition
+	var excludes []*v1.Partition
 
 	rootMap := map[string]string{}
 
 	if b.spec.Expandable {
-		exclude = b.spec.Partitions.Persistent
+		excludes = append(excludes, b.spec.Partitions.Persistent, b.spec.Partitions.State)
 	}
-	for _, part := range b.spec.Partitions.PartitionsByInstallOrder(v1.PartitionList{}, exclude) {
+	for _, part := range b.spec.Partitions.PartitionsByInstallOrder(v1.PartitionList{}, excludes...) {
 		rootMap[part.Name] = strings.TrimSuffix(part.Path, filepath.Ext(part.Path))
 		err = utils.MkdirAll(b.cfg.Fs, rootMap[part.Name], constants.DirPerm)
 		if err != nil {
@@ -246,8 +257,6 @@ func (b *BuildDiskAction) BuildDiskRun() (err error) { //nolint:gocyclo
 			b.cfg.Logger.Errorf("failed creating expandable cloud-config: %s", err.Error())
 			return err
 		}
-		// Omit persistent partition and minimize state partition size
-		b.spec.Partitions.State.Size = constants.MinPartSize
 	} else {
 		// Run a snapshotter transaction for System source in state partition
 		err = b.snapshotter.InitSnapshotter(b.roots[constants.StatePartName])
@@ -375,7 +384,7 @@ func (b *BuildDiskAction) CreatePartitionImages() ([]*v1.Image, error) {
 
 	excludes = append(excludes, b.spec.Partitions.EFI)
 	if b.spec.Expandable {
-		excludes = append(excludes, b.spec.Partitions.Persistent)
+		excludes = append(excludes, b.spec.Partitions.State, b.spec.Partitions.Persistent)
 	}
 
 	b.cfg.Logger.Infof("Creating EFI partition image")
@@ -611,7 +620,7 @@ func (b *BuildDiskAction) CreateDiskPartitionTable(disk string) error {
 	}
 
 	if b.spec.Expandable {
-		excludes = append(excludes, b.spec.Partitions.Persistent)
+		excludes = append(excludes, b.spec.Partitions.State, b.spec.Partitions.Persistent)
 	}
 	elParts := b.spec.Partitions.PartitionsByInstallOrder(v1.PartitionList{}, excludes...)
 	for i, part := range elParts {
@@ -653,8 +662,11 @@ func (b *BuildDiskAction) applySelinuxLabels(root string, unprivileged bool) err
 }
 
 func (b *BuildDiskAction) createBuildDiskStateYaml(stateRoot, recoveryRoot string) error {
-	if b.spec.Partitions.State == nil || b.spec.Partitions.Recovery == nil {
-		return fmt.Errorf("undefined state or recovery partition")
+	if b.spec.Partitions.Recovery == nil {
+		return fmt.Errorf("undefined recovery partition")
+	}
+	if b.spec.Partitions.State == nil && !b.spec.Expandable {
+		return fmt.Errorf("undefined state partition")
 	}
 
 	snapshots := map[int]*v1.SystemState{}
@@ -702,9 +714,13 @@ func (b *BuildDiskAction) createBuildDiskStateYaml(stateRoot, recoveryRoot strin
 		}
 	}
 
+	statePath := ""
+	if !b.spec.Expandable {
+		statePath = filepath.Join(stateRoot, constants.InstallStateFile)
+	}
+
 	return b.cfg.WriteInstallState(
-		installState,
-		filepath.Join(stateRoot, constants.InstallStateFile),
+		installState, statePath,
 		filepath.Join(recoveryRoot, constants.InstallStateFile),
 	)
 }
@@ -722,20 +738,25 @@ func (b *BuildDiskAction) SetExpandableCloudInitStage() error {
 		Stages: map[string][]schema.Stage{
 			layoutSetStage: {
 				schema.Stage{
-					Name: "Expand state partition",
+					Name: "Add state partition",
 					Layout: schema.Layout{
 						Device: &schema.Device{
-							Label: b.spec.Partitions.State.FilesystemLabel,
+							Label: b.spec.Partitions.Recovery.FilesystemLabel,
 						},
-						Expand: &schema.Expand{
-							Size: b.spec.Partitions.State.Size,
+						Parts: []schema.Partition{
+							{
+								FSLabel:    b.spec.Partitions.State.FilesystemLabel,
+								Size:       b.spec.Partitions.State.Size,
+								PLabel:     b.spec.Partitions.State.Name,
+								FileSystem: b.spec.Partitions.State.FS,
+							},
 						},
 					},
 				}, schema.Stage{
 					Name: "Add persistent partition",
 					Layout: schema.Layout{
 						Device: &schema.Device{
-							Label: b.spec.Partitions.State.FilesystemLabel,
+							Label: b.spec.Partitions.Recovery.FilesystemLabel,
 						},
 						Parts: []schema.Partition{
 							{

pkg/action/build_test.go

@@ -251,7 +251,6 @@ var _ = Describe("Build Actions", func() {
 				{"mkfs.vfat", "-n", "COS_GRUB"},
 				{"mkfs.ext4", "-L", "COS_OEM"},
 				{"mkfs.ext4", "-L", "COS_RECOVERY"},
-				{"mkfs.ext4", "-L", "COS_STATE"},
 				{"sgdisk", "-p", "/tmp/test/elemental.raw"},
 				{"partx", "-u", "/tmp/test/elemental.raw"},
 			})).To(Succeed())

pkg/action/install.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"github.com/rancher/elemental-toolkit/pkg/bootloader"
+	"github.com/rancher/elemental-toolkit/pkg/constants"
 	cnst "github.com/rancher/elemental-toolkit/pkg/constants"
 	"github.com/rancher/elemental-toolkit/pkg/elemental"
 	elementalError "github.com/rancher/elemental-toolkit/pkg/error"
@@ -65,7 +66,14 @@ func NewInstallAction(cfg *v1.RunConfig, spec *v1.InstallSpec, opts ...InstallAc
 	}
 
 	if i.snapshotter == nil {
-		i.snapshotter, err = snapshotter.NewLoopDeviceSnapshotter(cfg.Config, cfg.Snapshotter, i.bootloader)
+		i.snapshotter, err = snapshotter.NewSnapshotter(cfg.Config, cfg.Snapshotter, i.bootloader)
+	}
+
+	if i.cfg.Snapshotter.Type == constants.BtrfsSnapshotterType {
+		if spec.Partitions.State.FS != constants.Btrfs {
+			cfg.Logger.Warning("Btrfs snapshotter type, forcing btrfs filesystem on state partition")
+			spec.Partitions.State.FS = constants.Btrfs
+		}
 	}
 
 	return i, err

pkg/action/mount.go

@@ -70,7 +70,7 @@ func RunMount(cfg *v1.RunConfig, spec *v1.MountSpec) error {
 	}
 
 	cfg.Logger.Debugf("Mounting persistent directories")
-	if err = MountPersistent(cfg, spec.Sysroot, spec.Persistent); err != nil {
+	if err = MountPersistent(cfg, spec); err != nil {
 		cfg.Logger.Errorf("Error mounting persistent overlays: %s", err.Error())
 		return err
 	}
@@ -186,22 +186,22 @@ func MountEphemeral(cfg *v1.RunConfig, sysroot string, overlay v1.EphemeralMount
 	return nil
 }
 
-func MountPersistent(cfg *v1.RunConfig, sysroot string, persistent v1.PersistentMounts) error {
+func MountPersistent(cfg *v1.RunConfig, spec *v1.MountSpec) error {
 	mountFunc := MountOverlayPath
-	if persistent.Mode == "bind" {
+	if spec.Persistent.Mode == "bind" {
 		mountFunc = MountBindPath
 	}
 
-	if persistent.Volume.Device == "" || persistent.Volume.Mountpoint == "" {
+	if !spec.HasPersistent() {
 		cfg.Logger.Debug("No persistent device defined, omitting persistent paths mounts")
 		return nil
 	}
 
-	for _, path := range persistent.Paths {
-		cfg.Logger.Debugf("Mounting path %s into %s", path, sysroot)
+	for _, path := range spec.Persistent.Paths {
+		cfg.Logger.Debugf("Mounting path %s into %s", path, spec.Sysroot)
 
-		target := filepath.Join(persistent.Volume.Mountpoint, constants.PersistentStateDir)
-		if err := mountFunc(cfg, sysroot, target, path); err != nil {
+		target := filepath.Join(spec.Persistent.Volume.Mountpoint, constants.PersistentStateDir)
+		if err := mountFunc(cfg, spec.Sysroot, target, path); err != nil {
 			cfg.Logger.Errorf("Error mounting path %s: %s", path, err.Error())
 			return err
 		}
@@ -224,14 +224,18 @@ func MountBindPath(cfg *v1.RunConfig, sysroot, overlayDir, path string) error {
 	trimmed := strings.TrimPrefix(path, "/")
 	pathName := strings.ReplaceAll(trimmed, "/", "-") + ".bind"
 	stateDir := fmt.Sprintf("%s/%s", overlayDir, pathName)
-	if err := utils.MkdirAll(cfg.Config.Fs, stateDir, constants.DirPerm); err != nil {
-		cfg.Logger.Errorf("Error creating upperdir %s: %s", stateDir, err.Error())
-		return err
-	}
 
-	if err := utils.SyncData(cfg.Logger, cfg.Runner, cfg.Fs, base, stateDir); err != nil {
-		cfg.Logger.Errorf("Error shuffling data: %s", err.Error())
-		return err
+	// Only sync data once, otherwise it could modify persistent data from a previous boot
+	if ok, _ := utils.Exists(cfg.Fs, stateDir); !ok {
+		if err := utils.MkdirAll(cfg.Fs, stateDir, constants.DirPerm); err != nil {
+			cfg.Logger.Errorf("Error creating upperdir %s: %s", stateDir, err.Error())
+			return err
+		}
+
+		if err := utils.SyncData(cfg.Logger, cfg.Runner, cfg.Fs, base, stateDir); err != nil {
+			cfg.Logger.Errorf("Error shuffling data: %s", err.Error())
+			return err
+		}
 	}
 
 	if err := cfg.Mounter.Mount(stateDir, base, "none", []string{"defaults", "bind"}); err != nil {
@@ -246,21 +250,21 @@ func MountOverlayPath(cfg *v1.RunConfig, sysroot, overlayDir, path string) error
 	cfg.Logger.Debugf("Mounting overlay path %s", path)
 
 	lower := filepath.Join(sysroot, path)
-	if err := utils.MkdirAll(cfg.Config.Fs, lower, constants.DirPerm); err != nil {
+	if err := utils.MkdirAll(cfg.Fs, lower, constants.DirPerm); err != nil {
 		cfg.Logger.Errorf("Error creating directory %s: %s", path, err.Error())
 		return err
 	}
 
 	trimmed := strings.TrimPrefix(path, "/")
 	pathName := strings.ReplaceAll(trimmed, "/", "-") + overlaySuffix
 	upper := fmt.Sprintf("%s/%s/upper", overlayDir, pathName)
-	if err := utils.MkdirAll(cfg.Config.Fs, upper, constants.DirPerm); err != nil {
+	if err := utils.MkdirAll(cfg.Fs, upper, constants.DirPerm); err != nil {
 		cfg.Logger.Errorf("Error creating upperdir %s: %s", upper, err.Error())
 		return err
 	}
 
 	work := fmt.Sprintf("%s/%s/work", overlayDir, pathName)
-	if err := utils.MkdirAll(cfg.Config.Fs, work, constants.DirPerm); err != nil {
+	if err := utils.MkdirAll(cfg.Fs, work, constants.DirPerm); err != nil {
 		cfg.Logger.Errorf("Error creating workdir %s: %s", work, err.Error())
 		return err
 	}
@@ -330,7 +334,7 @@ func InitialFstabData(runner v1.Runner, sysroot string) (string, error) {
 	}
 	for _, mnt := range mounts {
 		if mnt.Mountpoint == sysroot {
-			data += fstab(mnt.Device, "/", "auto", mnt.Options)
+			data += fstab(mnt.Device, "/", mnt.FSType, mnt.Options)
 		} else if strings.HasPrefix(mnt.Mountpoint, sysroot) {
 			data += fstab(mnt.Device, strings.TrimPrefix(mnt.Mountpoint, sysroot), mnt.FSType, mnt.Options)
 		} else if strings.HasPrefix(mnt.Mountpoint, constants.RunElementalDir) {
@@ -368,16 +372,16 @@ func findmnt(runner v1.Runner, mountpoint string) ([]*v1.VolumeMount, error) {
 			continue
 		}
 		if lineFields[2] == "btrfs" {
-			r := regexp.MustCompile(`(/.+)\[.*\]`)
-			if r.MatchString(lineFields[0]) {
-				match := r.FindStringSubmatch(lineFields[0])
+			r := regexp.MustCompile(`^(/[^\[\]]+)`)
+			if match := r.FindStringSubmatch(lineFields[0]); match != nil {
 				lineFields[0] = match[1]
 			}
 		}
 		mounts = append(mounts, &v1.VolumeMount{
 			Device:     lineFields[0],
 			Mountpoint: lineFields[1],
 			Options:    strings.Split(lineFields[3], ","),
+			FSType:     lineFields[2],
 		})
 	}
 	return mounts, nil