Skip to content

Commit

Permalink
Snapshottable recovery system
Browse files Browse the repository at this point in the history 
Deploy the entire recovery system to the same folder (kernel, initrd and
rootfs).

During upgrade deploy to a transitional folder and then switch it with
the current recovery system and then delete the old one.

This makes sure we clean up old recovery systems and don't risk mixing
systems during upgrade.

Signed-off-by: Fredrik Lönnegren <fredrik.lonnegren@suse.com>
  • Loading branch information
@frelon
frelon committed Mar 26, 2024
1 parent 5f2cdb0 commit 533f508
Show file tree
Hide file tree
Showing 12 changed files with 103 additions and 83 deletions.
2 changes: 1 addition & 1 deletion cmd/build-iso.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -125,7 +125,7 @@ func NewBuildISO(root *cobra.Command, addCheckRoot bool) *cobra.Command {
}

buildISO := action.NewBuildISOAction(cfg, spec)
return buildISO.ISORun()
return buildISO.Run()
},
}

Expand Down
9 changes: 1 addition & 8 deletions pkg/action/build-disk.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -237,16 +237,9 @@ func (b *BuildDiskAction) BuildDiskRun() (err error) { //nolint:gocyclo
return elementalError.NewFromError(err, elementalError.HookAfterDisk)
}

// Create recovery image
bootDir := filepath.Join(b.roots[constants.RecoveryPartName], "boot")
if err = utils.MkdirAll(b.cfg.Fs, bootDir, constants.DirPerm); err != nil {
b.cfg.Logger.Errorf("failed creating recovery boot dir: %v", err)
return err
}

tmpSrc := b.spec.RecoverySystem.Source
b.spec.RecoverySystem.Source = types.NewDirSrc(recRoot)
err = elemental.DeployRecoverySystem(b.cfg.Config, &b.spec.RecoverySystem, bootDir)
err = elemental.DeployRecoverySystem(b.cfg.Config, &b.spec.RecoverySystem)
if err != nil {
b.cfg.Logger.Errorf("failed deploying recovery system: %v", err)
return err
Expand Down
10 changes: 5 additions & 5 deletions pkg/action/build-iso.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ func grubCfgTemplate(arch string) string {
menuentry "%s" --class os --unrestricted {
echo Loading kernel...
linux ($root)` + constants.ISOKernelPath(arch) + ` cdroot root=live:CDLABEL=%s rd.live.dir=/ rd.live.squashimg=rootfs.squashfs console=tty1 console=ttyS0 elemental.disable elemental.setup=` + constants.ISOCloudInitPath + `
linux ($root)` + constants.ISOKernelPath(arch) + ` cdroot root=live:CDLABEL=%s rd.live.dir=` + constants.ISOLoaderPath(arch) + ` rd.live.squashimg=rootfs.squashfs console=tty1 console=ttyS0 elemental.disable elemental.setup=` + constants.ISOCloudInitPath + `
echo Loading initrd...
initrd ($root)` + constants.ISOInitrdPath(arch) + `
}
Expand Down Expand Up @@ -78,8 +78,8 @@ func NewBuildISOAction(cfg *types.BuildConfig, spec *types.LiveISO, opts ...Buil
return b
}

// BuildISORun will install the system from a given configuration
func (b *BuildISOAction) ISORun() error {
// Run will install the system from a given configuration
func (b *BuildISOAction) Run() error {
cleanup := utils.NewCleanStack()
var err error
defer func() { err = cleanup.Cleanup(err) }()
Expand Down Expand Up @@ -170,11 +170,11 @@ func (b *BuildISOAction) ISORun() error {

image := &types.Image{
Source: types.NewDirSrc(rootDir),
File: filepath.Join(isoDir, constants.ISORootFile),
File: filepath.Join(bootDir, constants.ISORootFile),
FS: constants.SquashFs,
}

err = elemental.DeployRecoverySystem(b.cfg.Config, image, bootDir)
err = elemental.DeployRecoverySystem(b.cfg.Config, image)
if err != nil {
b.cfg.Logger.Errorf("Failed preparing ISO's root tree: %v", err)
return err
Expand Down
20 changes: 10 additions & 10 deletions pkg/action/build_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ var _ = Describe("Build Actions", func() {
}

buildISO := action.NewBuildISOAction(cfg, iso, action.WithLiveBootloader(bootloader))
err := buildISO.ISORun()
err := buildISO.Run()

Expect(err).ShouldNot(HaveOccurred())
})
Expand All @@ -138,7 +138,7 @@ var _ = Describe("Build Actions", func() {
iso.RootFS = append(iso.RootFS, rootSrc)

buildISO := action.NewBuildISOAction(cfg, iso, action.WithLiveBootloader(bootloader))
err := buildISO.ISORun()
err := buildISO.Run()
Expect(err).Should(HaveOccurred())
})
It("Fails on prepare ISO", func() {
Expand All @@ -148,7 +148,7 @@ var _ = Describe("Build Actions", func() {
iso.RootFS = append(iso.RootFS, rootSrc)

buildISO := action.NewBuildISOAction(cfg, iso, action.WithLiveBootloader(bootloader))
err := buildISO.ISORun()
err := buildISO.Run()

Expect(err).Should(HaveOccurred())
})
Expand All @@ -161,14 +161,14 @@ var _ = Describe("Build Actions", func() {

By("fails without kernel")
buildISO := action.NewBuildISOAction(cfg, iso, action.WithLiveBootloader(bootloader))
err = buildISO.ISORun()
err = buildISO.Run()
Expect(err).Should(HaveOccurred())

By("fails without initrd")
_, err = fs.Create("/local/dir/boot/vmlinuz")
Expect(err).ShouldNot(HaveOccurred())
buildISO = action.NewBuildISOAction(cfg, iso, action.WithLiveBootloader(bootloader))
err = buildISO.ISORun()
err = buildISO.Run()
Expect(err).Should(HaveOccurred())
})
It("Fails installing uefi sources", func() {
Expand All @@ -178,7 +178,7 @@ var _ = Describe("Build Actions", func() {
iso.UEFI = []*types.ImageSource{uefiSrc}

buildISO := action.NewBuildISOAction(cfg, iso)
err := buildISO.ISORun()
err := buildISO.Run()
Expect(err).Should(HaveOccurred())
})
It("Fails on ISO filesystem creation", func() {
Expand All @@ -193,7 +193,7 @@ var _ = Describe("Build Actions", func() {
}

buildISO := action.NewBuildISOAction(cfg, iso, action.WithLiveBootloader(bootloader))
err := buildISO.ISORun()
err := buildISO.Run()

Expect(err).Should(HaveOccurred())
})
Expand Down Expand Up @@ -228,7 +228,7 @@ var _ = Describe("Build Actions", func() {
Expect(buildDisk.BuildDiskRun()).To(Succeed())

Expect(runner.MatchMilestones([][]string{
{"mksquashfs", "/tmp/test/build/recovery.img.root", "/tmp/test/build/recovery/recovery.img"},
{"mksquashfs", "/tmp/test/build/recovery.img.root", "/tmp/test/build/recovery/boot/recovery.img"},
{"mkfs.ext4", "-L", "COS_STATE"},
{"losetup", "--show", "-f", "/tmp/test/build/state.part"},
{"mkfs.vfat", "-n", "COS_GRUB"},
Expand All @@ -255,7 +255,7 @@ var _ = Describe("Build Actions", func() {
Expect(buildDisk.BuildDiskRun()).To(Succeed())

Expect(runner.MatchMilestones([][]string{
{"mksquashfs", "/tmp/test/build/recovery.img.root", "/tmp/test/build/recovery/recovery.img"},
{"mksquashfs", "/tmp/test/build/recovery.img.root", "/tmp/test/build/recovery/boot/recovery.img"},
{"mkfs.vfat", "-n", "COS_GRUB"},
{"mkfs.ext4", "-L", "COS_OEM"},
{"mkfs.ext4", "-L", "COS_RECOVERY"},
Expand All @@ -274,7 +274,7 @@ var _ = Describe("Build Actions", func() {
Expect(buildDisk.BuildDiskRun()).NotTo(Succeed())

Expect(runner.MatchMilestones([][]string{
{"mksquashfs", "/tmp/test/build/recovery.img.root", "/tmp/test/build/recovery/recovery.img"},
{"mksquashfs", "/tmp/test/build/recovery.img.root", "/tmp/test/build/recovery/boot/recovery.img"},
})).To(Succeed())

// failed before preparing partitions images
Expand Down
2 changes: 1 addition & 1 deletion pkg/action/install.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -249,7 +249,7 @@ func (i InstallAction) Run() (err error) {
}
recoverySystem.Source.SetDigest(i.spec.System.GetDigest())
}
err = elemental.DeployRecoverySystem(i.cfg.Config, &recoverySystem, recoveryBootDir)
err = elemental.DeployRecoverySystem(i.cfg.Config, &recoverySystem)
if err != nil {
i.cfg.Logger.Errorf("Failed deploying recovery image: %v", err)
return elementalError.NewFromError(err, elementalError.DeployImage)
Expand Down
65 changes: 45 additions & 20 deletions pkg/action/upgrade-recovery.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,15 +84,15 @@ func NewUpgradeRecoveryAction(config *types.RunConfig, spec *types.UpgradeSpec,
return u, nil
}

func (u UpgradeRecoveryAction) Info(s string, args ...interface{}) {
func (u UpgradeRecoveryAction) Infof(s string, args ...interface{}) {
u.cfg.Logger.Infof(s, args...)
}

func (u UpgradeRecoveryAction) Debug(s string, args ...interface{}) {
func (u UpgradeRecoveryAction) Debugf(s string, args ...interface{}) {
u.cfg.Logger.Debugf(s, args...)
}

func (u UpgradeRecoveryAction) Error(s string, args ...interface{}) {
func (u UpgradeRecoveryAction) Errorf(s string, args ...interface{}) {
u.cfg.Logger.Errorf(s, args...)
}

Expand Down Expand Up @@ -146,48 +146,73 @@ func (u *UpgradeRecoveryAction) Run() (err error) {
return err
}

// Create recovery /boot dir if not exists
bootDir := filepath.Join(u.spec.Partitions.Recovery.MountPoint, "boot")
if err := utils.MkdirAll(u.cfg.Fs, bootDir, constants.DirPerm); err != nil {
u.cfg.Logger.Errorf("failed creating recovery boot dir: %v", err)
return elementalError.NewFromError(err, elementalError.CreateDir)
// Remove any traces of previously errored upgrades
transitionDir := filepath.Join(u.spec.Partitions.Recovery.MountPoint, constants.BootTransitionDir)
if ok, _ := utils.Exists(u.cfg.Fs, transitionDir); ok {
u.Debugf("removing orphaned recovery system %s", transitionDir)
err = u.cfg.Fs.RemoveAll(transitionDir)
if err != nil {
u.Errorf("failed removing old recovery image: %s", err.Error())
return err
}
}

// Upgrade recovery
err = elemental.DeployRecoverySystem(u.cfg.Config, &u.spec.RecoverySystem, bootDir)
// Deploy recovery system to transition dir
err = elemental.DeployRecoverySystem(u.cfg.Config, &u.spec.RecoverySystem)
if err != nil {
u.cfg.Logger.Errorf("failed deploying recovery image: %v", err)
u.cfg.Logger.Errorf("failed deploying recovery image: %s", err.Error())
return elementalError.NewFromError(err, elementalError.DeployImage)
}
recoveryFile := filepath.Join(u.spec.Partitions.Recovery.MountPoint, constants.RecoveryImgFile)
transitionFile := filepath.Join(u.spec.Partitions.Recovery.MountPoint, constants.TransitionImgFile)
if ok, _ := utils.Exists(u.cfg.Fs, recoveryFile); ok {
err = u.cfg.Fs.Remove(recoveryFile)

// Switch places on /boot and transition-dir
existingDir := filepath.Join(u.spec.Partitions.Recovery.MountPoint, constants.BootDir)
oldBootDir := filepath.Join(u.spec.Partitions.Recovery.MountPoint, constants.OldBootDir)
if ok, _ := utils.Exists(u.cfg.Fs, existingDir); ok {
err = u.cfg.Fs.Rename(existingDir, oldBootDir)
if err != nil {
u.Error("failed removing old recovery image")
u.Errorf("failed removing old recovery image: %s", err.Error())
return err
}
}
err = u.cfg.Fs.Rename(transitionFile, recoveryFile)
err = u.cfg.Fs.Rename(transitionDir, existingDir)
if err != nil {
u.Error("failed renaming transition recovery image")
u.cfg.Logger.Errorf("failed renaming transition recovery image: %s", err.Error())

// Try to salvage old recovery system
if ok, _ := utils.Exists(u.cfg.Fs, oldBootDir); ok {
err = u.cfg.Fs.Rename(existingDir, oldBootDir)
if err != nil {
u.cfg.Logger.Errorf("failed salvaging old recovery system: %s", err.Error())
}
}

return err
}

// Remove old boot-dir when new recovery system is in place
if ok, _ := utils.Exists(u.cfg.Fs, oldBootDir); ok {
err = u.cfg.Fs.RemoveAll(oldBootDir)
if err != nil {
u.Errorf("failed removing old recovery image: %s", err.Error())
return err
}
}

// Update state.yaml file on recovery and state partitions
if u.updateInstallState {
err = u.upgradeInstallStateYaml()
if err != nil {
u.Error("failed upgrading installation metadata")
u.Errorf("failed upgrading installation metadata: %s", err.Error())
return err
}
}

u.Info("Recovery upgrade completed")
u.Infof("Recovery upgrade completed")

// Do not reboot/poweroff on cleanup errors
err = cleanup.Cleanup(err)
if err != nil {
u.Errorf("failed cleanup: %s", err.Error())
return elementalError.NewFromError(err, elementalError.Cleanup)
}

Expand Down
25 changes: 13 additions & 12 deletions pkg/action/upgrade-recovery_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ var _ = Describe("Upgrade Recovery Actions", func() {
Expect(err).To(HaveOccurred())
})
It("Successfully upgrades recovery from docker image", Label("docker"), func() {
recoveryImgPath := filepath.Join(constants.LiveDir, constants.RecoveryImgFile)
recoveryImgPath := filepath.Join(constants.LiveDir, constants.BootDir, constants.RecoveryImgFile)
spec := PrepareTestRecoveryImage(config, constants.LiveDir, fs, runner)

// This should be the old image
Expand Down Expand Up @@ -212,7 +212,7 @@ var _ = Describe("Upgrade Recovery Actions", func() {
Expect(spec.State.Date).ToNot(BeEmpty(), "post-upgrade state should contain a date")
})
It("Successfully skips updateInstallState", Label("docker"), func() {
recoveryImgPath := filepath.Join(constants.LiveDir, constants.RecoveryImgFile)
recoveryImgPath := filepath.Join(constants.LiveDir, constants.BootDir, constants.RecoveryImgFile)
spec := PrepareTestRecoveryImage(config, constants.LiveDir, fs, runner)

// This should be the old image
Expand Down Expand Up @@ -270,22 +270,23 @@ func PrepareTestRecoveryImage(config *types.RunConfig, recoveryPath string, fs v
}
Expect(config.WriteInstallState(installState, statePath, statePath)).ShouldNot(HaveOccurred())

recoveryImgPath := filepath.Join(recoveryPath, constants.RecoveryImgFile)
Expect(fs.WriteFile(recoveryImgPath, []byte("recovery"), constants.FilePerm)).ShouldNot(HaveOccurred())

transitionDir := filepath.Join(recoveryPath, "transition.imgTree")
Expect(utils.MkdirAll(fs, filepath.Join(transitionDir, "lib/modules/6.6"), constants.DirPerm)).ShouldNot(HaveOccurred())
bootDir := filepath.Join(transitionDir, "boot")
Expect(utils.MkdirAll(fs, bootDir, constants.DirPerm)).ShouldNot(HaveOccurred())
Expect(fs.WriteFile(filepath.Join(bootDir, "vmlinuz-6.6"), []byte("kernel"), constants.FilePerm)).ShouldNot(HaveOccurred())
Expect(fs.WriteFile(filepath.Join(bootDir, "elemental.initrd-6.6"), []byte("initrd"), constants.FilePerm)).ShouldNot(HaveOccurred())
for _, rootDir := range []string{"/some/dir", recoveryPath} {
bootDir := filepath.Join(rootDir, "boot")
Expect(utils.MkdirAll(fs, bootDir, constants.DirPerm)).ShouldNot(HaveOccurred())
recoveryImgPath := filepath.Join(bootDir, constants.RecoveryImgFile)
Expect(fs.WriteFile(recoveryImgPath, []byte("recovery"), constants.FilePerm)).ShouldNot(HaveOccurred())
Expect(utils.MkdirAll(fs, filepath.Join(rootDir, "lib/modules/6.6"), constants.DirPerm)).ShouldNot(HaveOccurred())
Expect(utils.MkdirAll(fs, bootDir, constants.DirPerm)).ShouldNot(HaveOccurred())
Expect(fs.WriteFile(filepath.Join(bootDir, "vmlinuz-6.6"), []byte("kernel"), constants.FilePerm)).ShouldNot(HaveOccurred())
Expect(fs.WriteFile(filepath.Join(bootDir, "elemental.initrd-6.6"), []byte("initrd"), constants.FilePerm)).ShouldNot(HaveOccurred())
}

spec, err := conf.NewUpgradeSpec(config.Config)
Expect(err).ShouldNot(HaveOccurred())

spec.System = types.NewDockerSrc("alpine")
spec.RecoveryUpgrade = true
spec.RecoverySystem.Source = spec.System
spec.RecoverySystem.Source = types.NewDirSrc("/some/dir")
spec.RecoverySystem.Size = 16

runner.SideEffect = func(command string, args ...string) ([]byte, error) {
Expand Down
2 changes: 1 addition & 1 deletion pkg/action/upgrade_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -294,7 +294,7 @@ var _ = Describe("Runtime Actions", func() {
Expect(runner.IncludesCmds([][]string{{"poweroff", "-f"}})).To(BeNil())
})
It("Successfully upgrades recovery from docker image", Label("docker"), func() {
recoveryImgPath := filepath.Join(constants.LiveDir, constants.RecoveryImgFile)
recoveryImgPath := filepath.Join(constants.LiveDir, constants.BootDir, constants.RecoveryImgFile)
spec := PrepareTestRecoveryImage(config, constants.LiveDir, fs, runner)

// This should be the old image
Expand Down
11 changes: 7 additions & 4 deletions pkg/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -194,7 +194,7 @@ func NewInstallSpec(cfg types.Config) *types.InstallSpec {

recoverySystem.Source = system
recoverySystem.FS = constants.SquashFs
recoverySystem.File = filepath.Join(constants.RecoveryDir, constants.RecoveryImgFile)
recoverySystem.File = filepath.Join(constants.RecoveryDir, constants.BootDir, constants.RecoveryImgFile)
recoverySystem.MountPoint = constants.TransitionDir

return &types.InstallSpec{
Expand Down Expand Up @@ -334,7 +334,7 @@ func NewUpgradeSpec(cfg types.Config) (*types.UpgradeSpec, error) {
}

recovery = types.Image{
File: filepath.Join(ep.Recovery.MountPoint, constants.TransitionImgFile),
File: filepath.Join(ep.Recovery.MountPoint, constants.BootDir, constants.TransitionImgFile),
Size: constants.ImgSize,
Label: rState.Label,
FS: rState.FS,
Expand Down Expand Up @@ -440,10 +440,13 @@ func NewResetSpec(cfg types.Config) (*types.ResetSpec, error) {
cfg.Logger.Warnf("no Persistent partition found")
}

recoveryImg := filepath.Join(constants.RunningStateDir, constants.RecoveryImgFile)
recoveryImg := filepath.Join(constants.RunningStateDir, constants.BootDir, constants.RecoveryImgFile)
oldRecoveryImg := filepath.Join(constants.RunningStateDir, constants.RecoveryImgFile)

if exists, _ := utils.Exists(cfg.Fs, recoveryImg); exists {
imgSource = types.NewFileSrc(recoveryImg)
} else if exists, _ := utils.Exists(cfg.Fs, oldRecoveryImg); exists {
imgSource = types.NewFileSrc(oldRecoveryImg)
} else {
imgSource = types.NewEmptySrc()
}
Expand Down Expand Up @@ -514,7 +517,7 @@ func NewDisk(cfg *types.BuildConfig) *types.DiskSpec {
workdir = filepath.Join(cfg.OutDir, constants.DiskWorkDir)

recoveryImg.Size = constants.ImgSize
recoveryImg.File = filepath.Join(workdir, constants.RecoveryPartName, constants.RecoveryImgFile)
recoveryImg.File = filepath.Join(workdir, constants.RecoveryPartName, constants.BootDir, constants.RecoveryImgFile)
recoveryImg.FS = constants.SquashFs
recoveryImg.Source = types.NewEmptySrc()
recoveryImg.MountPoint = filepath.Join(
Expand Down
7 changes: 5 additions & 2 deletions pkg/constants/constants.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,10 @@ const (
PassiveImgName = "passive"
RecoveryImgName = "recovery"
RecoveryImgFile = "recovery.img"
TransitionImgFile = "transition.img"
TransitionImgFile = BootTransitionDir + "/" + RecoveryImgFile
BootTransitionDir = "boot-transition"
BootDir = "boot"
OldBootDir = "boot-old"

// Yip stages evaluated on reset/upgrade/install/build-disk actions
AfterInstallChrootHook = "after-install-chroot"
Expand Down Expand Up @@ -362,7 +365,7 @@ func GetDiskKeyEnvMap() map[string]string {
return map[string]string{}
}

// GetBootPath returns path use to store the boot files
// ISOLoaderPath returns path use to store the boot files
func ISOLoaderPath(arch string) string {
return filepath.Join("/boot", arch, "loader")
}
Expand Down
Loading

0 comments on commit 533f508

Please sign in to comment.