Grub refactor and other distros

.github/workflows/build_and_test.yaml

@@ -24,12 +24,19 @@ jobs:
       tests: ${{ steps.detect.outputs.tests }}
     steps:
       - id: detect
+        env:
+          FLAVOR: ${{ inputs.flavor }}
         run: |
           case "${{inputs.arch}}" in
           x86_64)
             echo "buildon='ubuntu-latest'" >> $GITHUB_OUTPUT
             echo "testson='macos-latest'" >> $GITHUB_OUTPUT
-            echo "tests=['test-upgrade', 'test-recovery', 'test-fallback', 'test-fsck', 'test-grubfallback']" >> $GITHUB_OUTPUT ;;
+            if [ "${FLAVOR}" == green ]; then
+              echo "tests=['test-upgrade', 'test-recovery', 'test-fallback', 'test-fsck', 'test-grubfallback']" >> $GITHUB_OUTPUT
+            else
+              echo "tests=['test-active']" >> $GITHUB_OUTPUT
+            fi
+            ;;
           aarch64)
             echo "buildon=['self-hosted', 'arm64']" >> $GITHUB_OUTPUT
             echo "testson=['self-hosted', 'arm64']" >> $GITHUB_OUTPUT

.github/workflows/pr.yaml

@@ -33,7 +33,7 @@ jobs:
     strategy:
       matrix:
         arch: ${{fromJson(needs.detect.outputs.arch)}}
-        flavor: ['green']
+        flavor: ['green', 'tumbleweed', 'blue', 'orange']
     uses: ./.github/workflows/build_and_test.yaml
     with:
       arch: ${{ matrix.arch }}

Makefile

@@ -87,8 +87,7 @@ build-disk: build-os
 		--entrypoint /usr/bin/elemental \
 		${TOOLKIT_REPO}:${VERSION} --debug build-disk --platform $(PLATFORM) --unprivileged --expandable -n elemental-$(FLAVOR).$(ARCH) --local \
 		--squash-no-compression -o /build ${REPO}:${VERSION}
-	dd if=$(ROOT_DIR)/build/elemental-$(FLAVOR).$(ARCH).raw of=$(ROOT_DIR)/build/elemental-$(FLAVOR).$(ARCH).img conv=notrunc
-	qemu-img convert -O qcow2 $(ROOT_DIR)/build/elemental-$(FLAVOR).$(ARCH).img $(ROOT_DIR)/build/elemental-$(FLAVOR).$(ARCH).qcow2
+	qemu-img convert -O qcow2 $(ROOT_DIR)/build/elemental-$(FLAVOR).$(ARCH).raw $(ROOT_DIR)/build/elemental-$(FLAVOR).$(ARCH).qcow2
 	qemu-img resize $(ROOT_DIR)/build/elemental-$(FLAVOR).$(ARCH).qcow2 $(DISKSIZE) 
 
 .PHONY: build-rpi-disk

cmd/build-iso.go

@@ -65,6 +65,7 @@ func NewBuildISO(root *cobra.Command, addCheckRoot bool) *cobra.Command {
 			flags := cmd.Flags()
 			err = validateCosignFlags(cfg.Logger, flags)
 			if err != nil {
+				cfg.Logger.Errorf("flags validation failed: %v", err)
 				return elementalError.NewFromError(err, elementalError.CosignWrongFlags)
 			}
 
@@ -129,7 +130,7 @@ func NewBuildISO(root *cobra.Command, addCheckRoot bool) *cobra.Command {
 		},
 	}
 
-	firmType := newEnumFlag([]string{v1.EFI, v1.BIOS}, v1.EFI)
+	firmType := newEnumFlag([]string{v1.EFI}, v1.EFI)
 
 	root.AddCommand(c)
 	c.Flags().StringP("name", "n", "", "Basename of the generated ISO file")
@@ -140,8 +141,8 @@ func NewBuildISO(root *cobra.Command, addCheckRoot bool) *cobra.Command {
 	c.Flags().String("overlay-iso", "", "Path of the overlayed iso data")
 	c.Flags().String("label", "", "Label of the ISO volume")
 	c.Flags().Bool("bootloader-in-rootfs", false, "Fetch ISO bootloader binaries from the rootfs")
-	c.Flags().Var(firmType, "firmware", "Firmware to install for: 'efi' or 'bios'. (defaults to 'efi')")
-	_ = c.Flags().MarkDeprecated("firmware", "'firmware' is deprecated. 'bios' firmware support is deprecated.")
+	c.Flags().Var(firmType, "firmware", "Firmware to install, only 'efi' is currently supported")
+	_ = c.Flags().MarkDeprecated("firmware", "'firmware' is deprecated. only efi firmware is supported.")
 	addPlatformFlags(c)
 	addCosignFlags(c)
 	addSquashFsCompressionFlags(c)

cmd/build-iso_test.go

@@ -36,6 +36,12 @@ var _ = Describe("BuidISO", Label("iso", "cmd"), func() {
 	AfterEach(func() {
 		viper.Reset()
 	})
+	It("Errors out setting firmware to anything else than efi", Label("flags"), func() {
+		_, _, err := executeCommandC(rootCmd, "build-iso", "--firmware", "bios")
+		Expect(err).ToNot(BeNil())
+		Expect(err.Error()).To(ContainSubstring("invalid argument"))
+		Expect(err.Error()).To(ContainSubstring("'bios' is not included in: efi"))
+	})
 	It("Errors out setting consign-key without setting cosign", Label("flags"), func() {
 		_, _, err := executeCommandC(rootCmd, "build-iso", "--cosign-key", "pubKey.url")
 		Expect(err).ToNot(BeNil())

cmd/flags.go

@@ -163,7 +163,7 @@ func (a *enum) Set(p string) error {
 		return false
 	}
 	if !isIncluded(a.Allowed, p) {
-		return fmt.Errorf("%s is not included in %s", p, strings.Join(a.Allowed, ","))
+		return fmt.Errorf("'%s' is not included in: %s", p, strings.Join(a.Allowed, ","))
 	}
 	a.Value = p
 	return nil

cmd/install.go

@@ -85,17 +85,19 @@ func NewInstallCmd(root *cobra.Command, addCheckRoot bool) *cobra.Command {
 			return install.Run()
 		},
 	}
-	firmType := newEnumFlag([]string{v1.EFI, v1.BIOS}, v1.EFI)
-	pTableType := newEnumFlag([]string{v1.GPT, v1.MSDOS}, v1.GPT)
+	firmType := newEnumFlag([]string{v1.EFI}, v1.EFI)
+	pTableType := newEnumFlag([]string{v1.GPT}, v1.GPT)
 
 	root.AddCommand(c)
 	c.Flags().StringSliceP("cloud-init", "c", []string{}, "Cloud-init config files")
 	c.Flags().StringP("iso", "i", "", "Performs an installation from the ISO url")
 	c.Flags().Bool("no-format", false, "Don’t format disks. It is implied that COS_STATE, COS_RECOVERY, COS_PERSISTENT, COS_OEM are already existing")
 
-	c.Flags().Var(firmType, "firmware", "Firmware to install for: 'efi' or 'bios'. (defaults to 'efi')")
+	c.Flags().Var(firmType, "firmware", "Firmware to install, only 'efi' is currently supported")
+	_ = c.Flags().MarkDeprecated("firmware", "'firmware' is deprecated. only efi firmware is supported.")
 
-	c.Flags().Var(pTableType, "part-table", "Partition table type to use")
+	c.Flags().Var(pTableType, "part-table", "Partition table type to use, only GPT type is currently supported")
+	_ = c.Flags().MarkDeprecated("part-table", "'part-table' is deprecated. only GPT type is supported.")
 
 	c.Flags().Bool("force", false, "Force install")
 	c.Flags().Bool("eject-cd", false, "Try to eject the cd on reboot, only valid if booting from iso")

cmd/install_test.go

@@ -38,6 +38,18 @@ var _ = Describe("Install", Label("install", "cmd"), func() {
 	AfterEach(func() {
 		viper.Reset()
 	})
+	It("Errors out setting firmware to anything else than efi", Label("flags"), func() {
+		_, _, err := executeCommandC(rootCmd, "install", "--firmware", "bios", "/dev/whatever")
+		Expect(err).ToNot(BeNil())
+		Expect(err.Error()).To(ContainSubstring("invalid argument"))
+		Expect(err.Error()).To(ContainSubstring("'bios' is not included in: efi"))
+	})
+	It("Errors out setting part-table to anything else than GPT", Label("flags"), func() {
+		_, _, err := executeCommandC(rootCmd, "install", "--part-table", "msdos", "/dev/whatever")
+		Expect(err).ToNot(BeNil())
+		Expect(err.Error()).To(ContainSubstring("invalid argument"))
+		Expect(err.Error()).To(ContainSubstring("'msdos' is not included in: gpt"))
+	})
 	It("Errors out setting consign-key without setting cosign", Label("flags"), func() {
 		_, _, err := executeCommandC(rootCmd, "install", "--cosign-key", "pubKey.url", "/dev/whatever")
 		Expect(err).ToNot(BeNil())

config.yaml.example

@@ -8,10 +8,6 @@ install:
   # config, flags or env variables.
   target: /dev/sda
 
-  # basic disk configs for partitioning ('efi|bios' and 'gpt|msdos')
-  firmware: efi
-  part-table: gpt
-
   # partitions setup
   # setting a partition size key to 0 means that the partition will take over the rest of the free space on the disk
   # after creating the rest of the partitions

examples/blue/Dockerfile

@@ -0,0 +1,80 @@
+# run `make build` to build local/elemental-toolkit image
+ARG TOOLKIT_REPO
+ARG VERSION
+FROM ${TOOLKIT_REPO}:${VERSION} as TOOLKIT
+
+# OS base image of our choice
+FROM fedora:39 as OS
+ARG REPO
+ARG VERSION
+ENV VERSION=${VERSION}
+
+# install kernel, systemd, dracut, grub2 and other required tools
+RUN echo "install_weak_deps=False" >> /etc/dnf/dnf.conf
+RUN dnf install -y \
+    kernel \
+    NetworkManager \
+    audit \
+    coreutils \
+    curl \
+    device-mapper \
+    dosfstools \
+    dracut \
+    dracut-live \
+    dracut-network \
+    e2fsprogs \
+    efibootmgr \
+    gawk \
+    grub2 \
+    grub2-efi-x64 \
+    grub2-efi-x64-modules \
+    grub2-pc \
+    haveged \
+    vim \
+    openssh-server \
+    openssh-clients \
+    glibc-langpack-en \
+    parted \
+    gdisk \
+    rsync \
+    shim-x64 \
+    squashfs-tools \
+    systemd \
+    tar \
+    mtools \
+    xorriso \
+    patch \
+    which
+
+# Create non FHS paths
+RUN mkdir -p /oem /system
+
+# Just add the elemental cli
+COPY --from=TOOLKIT /usr/bin/elemental /usr/bin/elemental
+
+# This is patches are fix upstream dracut, see https://github.com/dracutdevs/dracut/pull/2525
+ADD patches /
+
+RUN cd /usr/lib/dracut && \
+    patch -p 1 -f -i /0001-fix-dmsquash-live-restore-compatibility-with-earlier.patch && \
+    patch -p 1 -f -i /0001-fix-overlayfs-split-overlayfs-mount-in-two-steps.patch && \
+    rm /*.patch
+
+# This is for automatic testing purposes, do not do this in production.
+RUN echo "PermitRootLogin yes" > /etc/ssh/sshd_config.d/rootlogin.conf
+
+# Add elemental config dir
+RUN mkdir -p /etc/elemental/config.d
+
+# Generate initrd with required elemental services
+RUN elemental --debug init -f
+
+# Update os-release file with some metadata
+RUN echo IMAGE_REPO=\"${REPO}\"             >> /etc/os-release && \
+    echo IMAGE_TAG=\"${VERSION}\"           >> /etc/os-release && \
+    echo IMAGE=\"${REPO}:${VERSION}\"       >> /etc/os-release && \
+    echo TIMESTAMP="`date +'%Y%m%d%H%M%S'`" >> /etc/os-release && \
+    echo GRUB_ENTRY_NAME=\"Elemental\"      >> /etc/os-release
+
+# Good for validation after the build
+CMD /bin/bash

examples/blue/patches/0001-fix-dmsquash-live-restore-compatibility-with-earlier.patch

@@ -0,0 +1,29 @@
+From 0e780720efe6488c4e07af39926575ee12f40339 Mon Sep 17 00:00:00 2001
+From: Laszlo Gombos <laszlo.gombos@gmail.com>
+Date: Fri, 24 Feb 2023 01:57:19 +0000
+Subject: [PATCH] fix(dmsquash-live): restore compatibility with earlier
+ releases
+
+Follow-up to 40dd5c90e0efcb9ebaa9abb42a38c7316e9706bd .
+---
+ modules.d/90dmsquash-live/dmsquash-live-root.sh | 4 ++++
+ 1 file changed, 4 insertions(+)
+
+diff --git a/modules.d/90dmsquash-live/dmsquash-live-root.sh b/modules.d/90dmsquash-live/dmsquash-live-root.sh
+index 62d1b5e7..a98e258c 100755
+--- a/modules.d/90dmsquash-live/dmsquash-live-root.sh
++++ b/modules.d/90dmsquash-live/dmsquash-live-root.sh
+@@ -403,6 +403,10 @@ fi
+
+ ROOTFLAGS="$(getarg rootflags)"
+
++if [ "$overlayfs" = required ]; then
++    echo "rd.live.overlay.overlayfs=1" > /etc/cmdline.d/dmsquash-need-overlay.conf
++fi
++
+ if [ -n "$overlayfs" ]; then
+     if [ -n "$FSIMG" ]; then
+         mkdir -m 0755 -p /run/rootfsbase
+-- 
+2.35.3
+

examples/blue/patches/0001-fix-overlayfs-split-overlayfs-mount-in-two-steps.patch

@@ -0,0 +1,91 @@
+From bddffedae038ceca263a904e40513a6e92f1b558 Mon Sep 17 00:00:00 2001
+From: David Cassany <dcassany@suse.com>
+Date: Fri, 22 Sep 2023 16:28:48 +0200
+Subject: [PATCH] fix(overlayfs): split overlayfs mount in two steps
+
+This commit splits the creation of required overlayfs underlaying
+directories and the actual overlayfs mount. This way it is still
+possible to mount the overlayfs with the generated sysroot.mount that
+dmsquash-live creates.
+
+The overlayfs tree is created in a pre-mount hook so it is executed
+before sysroot.mount is started. Otherwise sysroot.mount starts and
+fails before mount hooks are executed.
+
+Signed-off-by: David Cassany <dcassany@suse.com>
+---
+ modules.d/90overlayfs/module-setup.sh      |  1 +
+ modules.d/90overlayfs/mount-overlayfs.sh   | 13 -------------
+ modules.d/90overlayfs/prepare-overlayfs.sh | 21 +++++++++++++++++++++
+ 3 files changed, 22 insertions(+), 13 deletions(-)
+ create mode 100755 modules.d/90overlayfs/prepare-overlayfs.sh
+
+diff --git a/modules.d/90overlayfs/module-setup.sh b/modules.d/90overlayfs/module-setup.sh
+index 27aa7cfa..893e2dc3 100755
+--- a/modules.d/90overlayfs/module-setup.sh
++++ b/modules.d/90overlayfs/module-setup.sh
+@@ -15,4 +15,5 @@ installkernel() {
+
+ install() {
+     inst_hook mount 01 "$moddir/mount-overlayfs.sh"
++    inst_hook pre-mount 01 "$moddir/prepare-overlayfs.sh"
+ }
+diff --git a/modules.d/90overlayfs/mount-overlayfs.sh b/modules.d/90overlayfs/mount-overlayfs.sh
+index 7e2da1a8..e1d23fb4 100755
+--- a/modules.d/90overlayfs/mount-overlayfs.sh
++++ b/modules.d/90overlayfs/mount-overlayfs.sh
+@@ -3,24 +3,11 @@
+ type getarg > /dev/null 2>&1 || . /lib/dracut-lib.sh
+
+ getargbool 0 rd.live.overlay.overlayfs && overlayfs="yes"
+-getargbool 0 rd.live.overlay.reset -d -y reset_overlay && reset_overlay="yes"
+ getargbool 0 rd.live.overlay.readonly -d -y readonly_overlay && readonly_overlay="--readonly" || readonly_overlay=""
+
+ ROOTFLAGS="$(getarg rootflags)"
+
+ if [ -n "$overlayfs" ]; then
+-    if ! [ -e /run/rootfsbase ]; then
+-        mkdir -m 0755 -p /run/rootfsbase
+-        mount --bind "$NEWROOT" /run/rootfsbase
+-    fi
+-
+-    mkdir -m 0755 -p /run/overlayfs
+-    mkdir -m 0755 -p /run/ovlwork
+-    if [ -n "$reset_overlay" ] && [ -h /run/overlayfs ]; then
+-        ovlfsdir=$(readlink /run/overlayfs)
+-        info "Resetting the OverlayFS overlay directory."
+-        rm -r -- "${ovlfsdir:?}"/* "${ovlfsdir:?}"/.* > /dev/null 2>&1
+-    fi
+     if [ -n "$readonly_overlay" ] && [ -h /run/overlayfs-r ]; then
+         ovlfs=lowerdir=/run/overlayfs-r:/run/rootfsbase
+     else
+diff --git a/modules.d/90overlayfs/prepare-overlayfs.sh b/modules.d/90overlayfs/prepare-overlayfs.sh
+new file mode 100755
+index 00000000..87bcc196
+--- /dev/null
++++ b/modules.d/90overlayfs/prepare-overlayfs.sh
+@@ -0,0 +1,21 @@
++#!/bin/sh
++
++type getarg > /dev/null 2>&1 || . /lib/dracut-lib.sh
++
++getargbool 0 rd.live.overlay.overlayfs && overlayfs="yes"
++getargbool 0 rd.live.overlay.reset -d -y reset_overlay && reset_overlay="yes"
++
++if [ -n "$overlayfs" ]; then
++    if ! [ -e /run/rootfsbase ]; then
++        mkdir -m 0755 -p /run/rootfsbase
++        mount --bind "$NEWROOT" /run/rootfsbase
++    fi
++
++    mkdir -m 0755 -p /run/overlayfs
++    mkdir -m 0755 -p /run/ovlwork
++    if [ -n "$reset_overlay" ] && [ -h /run/overlayfs ]; then
++        ovlfsdir=$(readlink /run/overlayfs)
++        info "Resetting the OverlayFS overlay directory."
++        rm -r -- "${ovlfsdir:?}"/* "${ovlfsdir:?}"/.* > /dev/null 2>&1
++    fi
++fi
+-- 
+2.35.3
+

examples/green/Dockerfile

@@ -55,12 +55,7 @@ RUN systemctl enable NetworkManager.service
 RUN cp /usr/share/systemd/tmp.mount /etc/systemd/system
 
 # Generate initrd with required elemental services
-RUN elemental init -f && \
-    kernel=$(ls /boot/Image-* 2>/dev/null | head -n1) && \
-    if [ -e "$kernel" ]; then ln -sf "${kernel#/boot/}" /boot/vmlinuz; fi && \
-    rm -rf /var/log/update* && \
-    >/var/log/lastlog && \
-    rm -rf /boot/vmlinux*
+RUN elemental --debug init -f 
 
 # Update os-release file with some metadata
 RUN echo IMAGE_REPO=\"${REPO}\"         >> /etc/os-release && \

examples/orange/05_network.yaml

@@ -0,0 +1,32 @@
+# Example network configuration for Ubuntu based systems
+name: "Default network configuration"
+stages:
+   initramfs:
+     - name: "Setup network"
+       files:
+       - path: /etc/netplan/elemental_setup.yaml
+         content: |
+           network:
+             version: 2
+             renderer: networkd
+             ethernets:
+               lan0:
+                 dhcp4: true
+         permissions: 0600
+         owner: 0
+         group: 0
+       - path: /etc/ssh/sshd_config.d/root_login.conf
+         content: |
+           PermitRootLogin yes
+         permissions: 0600
+         owner: 0
+         group: 0
+       - path: /etc/udev/rules.d/70-persistent-net.rules
+         content: |
+           SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="?*", ATTR{dev_id}=="0x0", ATTR{type}=="1", KERNEL=="?*", NAME="lan0"
+         permissions: 0600
+         owner: 0
+         group: 0
+       commands:
+         - ln -sf /run/systemd/resolve/stub-resolv.conf /etc/resolv.conf
+         - netplan apply