Improve CI stability

Adds several improvements: * Check VM pid on 'EventuallyConnects'. So it does not wait for a command to succeed if the underlaying VM crashed and it is not running. * Does not use '-daemonize' flag of qemu, now it simply runs on the background and the stdout and stderr are redirected to vmstdout file. * Does not install QEMU, the runner already has a recent QEMU version installed. This saves several minutes on each macos job. * Fixes some of the stability issues on macOS by disabling hugepages on the kernel. This is not supported on macOS. Signed-off-by: David Cassany <dcassany@suse.com>
rancher · Oct 31, 2023 · f973c2b · f973c2b
1 parent bf2cdb9
commit f973c2b
Show file tree

Hide file tree

Showing 9 changed files with 100 additions and 42 deletions.
diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml
@@ -157,7 +157,6 @@ jobs:
       - if: ${{ env.ARCH == 'x86_64' }} 
         name: Run VM script dependencies
         run: |
-          brew update; brew upgrade qemu
           brew install bash coreutils
       - if: ${{ env.ARCH == 'x86_64' }} 
         name: Prepare test (x86_64)
@@ -178,6 +177,13 @@ jobs:
           name: serial-${{ env.ARCH }}-${{ env.FLAVOR }}-${{ matrix.test }}.log
           path: tests/serial.log
           if-no-files-found: error
+      - name: Upload qemu stdout for ${{ matrix.test }}
+        uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: vmstdout-${{ env.ARCH }}-${{ env.FLAVOR }}-${{ matrix.test }}.log
+          path: tests/vmstdout
+          if-no-files-found: error
       - name: Stop test VM
         if: always()
         run: |
@@ -216,7 +222,6 @@ jobs:
       - if: ${{ env.ARCH == 'x86_64' }} 
         name: Run VM script dependencies
         run: |
-          brew update; brew upgrade qemu
           brew install bash coreutils
       - if: ${{ env.ARCH == 'x86_64' }} 
         name: Prepare test (x86_64)
@@ -237,6 +242,13 @@ jobs:
           name: serial-${{ env.ARCH }}-${{ env.FLAVOR }}-installer.log
           path: tests/serial.log
           if-no-files-found: error
+      - name: Upload qemu stdout for installer tests
+        uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: vmstdout-${{ env.ARCH }}-${{ env.FLAVOR }}-installer.log
+          path: tests/vmstdout
+          if-no-files-found: error
       - name: Stop test VM
         if: always()
         run: |

diff --git a/go.mod b/go.mod
@@ -16,7 +16,7 @@ require (
 	github.com/mudler/yip v1.4.0
 	github.com/onsi/ginkgo/v2 v2.9.3
 	github.com/onsi/gomega v1.27.6
-	github.com/rancher-sandbox/ele-testhelpers v0.0.0-20230907090440-1cc5d66b1725
+	github.com/rancher-sandbox/ele-testhelpers v0.0.0-20231031082920-8b1cf3f8c16f
 	github.com/sanity-io/litter v1.5.5
 	github.com/sirupsen/logrus v1.9.0
 	github.com/spf13/cobra v1.7.0

diff --git a/go.sum b/go.sum
@@ -410,8 +410,8 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
-github.com/rancher-sandbox/ele-testhelpers v0.0.0-20230907090440-1cc5d66b1725 h1:SW7EoIV5be+tDwwzinXvH9v4qKygHIaQJKt1wwis5ao=
-github.com/rancher-sandbox/ele-testhelpers v0.0.0-20230907090440-1cc5d66b1725/go.mod h1:Ex+a/ng4u2BvcGQdQjTHI48h88bQ6k2a7q8rnvU0XbQ=
+github.com/rancher-sandbox/ele-testhelpers v0.0.0-20231031082920-8b1cf3f8c16f h1:GUKwalxjMDhbxMmhhBFZ91jubBOxwYvu1q8sA+jk2Jg=
+github.com/rancher-sandbox/ele-testhelpers v0.0.0-20231031082920-8b1cf3f8c16f/go.mod h1:Ex+a/ng4u2BvcGQdQjTHI48h88bQ6k2a7q8rnvU0XbQ=
 github.com/rancher-sandbox/linuxkit v1.0.1 h1:LYKmv1XozmQGRV6Ilm88Fx/t54okVa8rx00wLJPZkBw=
 github.com/rancher-sandbox/linuxkit v1.0.1/go.mod h1:n6Fkjc5qoMeWrnLSA5oqUF8ZzFKMrM960CtBwfvH1ZM=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=

diff --git a/make/Makefile.test b/make/Makefile.test
@@ -1,7 +1,4 @@
-GINKGO?=$(shell which ginkgo 2> /dev/null)
-ifeq ("$(GINKGO)","")
-GINKGO="$(HOME)/go/bin/ginkgo"
-endif
+GINKGO?="github.com/onsi/ginkgo/v2/ginkgo"
 
 GINKGO_ARGS?=-v --fail-fast -r --timeout=3h
 
@@ -13,7 +10,10 @@ ifeq ("$(DISK)","")
 endif
 	@scripts/run_vm.sh start $(DISK)
 	@echo "VM started from $(DISK)"
-	$(GINKGO) run $(GINKGO_ARGS) ./tests/wait-active
+
+.PHONY: test-active
+test-active: prepare-test
+	VM_PID=$$(scripts/run_vm.sh vmpid) go run $(GINKGO) $(GINKGO_ARGS) ./tests/wait-active
 
 .PHONY: prepare-installer-test
 prepare-installer-test:
@@ -25,38 +25,41 @@ endif
 	@echo "VM started from $(ISO)"
 
 .PHONY: test-clean
-test-clean:
-	@scripts/run_vm.sh stop
+test-clean: test-stop
 	@scripts/run_vm.sh clean
 
+.PHONY: test-stop
+test-stop:
+	@scripts/run_vm.sh stop
+
 .PHONY: test-installer
 test-installer: prepare-installer-test
-	$(GINKGO) run $(GINKGO_ARGS) ./tests/installer
-	$(GINKGO) run $(GINKGO_ARGS) ./tests/smoke
+	VM_PID=$$(scripts/run_vm.sh vmpid) go run $(GINKGO) $(GINKGO_ARGS) ./tests/installer
+	VM_PID=$$(scripts/run_vm.sh vmpid) go run $(GINKGO) $(GINKGO_ARGS) ./tests/smoke
 
 .PHONY: test-smoke
-test-smoke: prepare-test
-	$(GINKGO) run $(GINKGO_ARGS) ./tests/smoke
+test-smoke: test-active
+	VM_PID=$$(scripts/run_vm.sh vmpid) go run $(GINKGO) $(GINKGO_ARGS) ./tests/smoke
 
 .PHONY: test-grubfallback
-test-grubfallback: prepare-test
-	$(GINKGO) run $(GINKGO_ARGS) ./tests/grubfallback
+test-grubfallback: test-active
+	VM_PID=$$(scripts/run_vm.sh vmpid) go run $(GINKGO) $(GINKGO_ARGS) ./tests/grubfallback
 
 .PHONY: test-recovery
-test-recovery: prepare-test
-	$(GINKGO) run $(GINKGO_ARGS) ./tests/recovery
+test-recovery: test-active
+	VM_PID=$$(scripts/run_vm.sh vmpid) go run $(GINKGO) $(GINKGO_ARGS) ./tests/recovery
 
 .PHONY: test-fallback
-test-fallback: prepare-test
-	$(GINKGO) run $(GINKGO_ARGS) ./tests/fallback
+test-fallback: test-active
+	VM_PID=$$(scripts/run_vm.sh vmpid) go run $(GINKGO) $(GINKGO_ARGS) ./tests/fallback
 
 .PHONY: test-fsck
-test-fsck: prepare-test
-	$(GINKGO) run $(GINKGO_ARGS) ./tests/fsck
+test-fsck: test-active
+	VM_PID=$$(scripts/run_vm.sh vmpid) go run $(GINKGO) $(GINKGO_ARGS) ./tests/fsck
 
 .PHONY: test-upgrade
-test-upgrade: prepare-test
-	$(GINKGO) run $(GINKGO_ARGS) ./tests/upgrade
+test-upgrade: test-active
+	VM_PID=$$(scripts/run_vm.sh vmpid) go run $(GINKGO) $(GINKGO_ARGS) ./tests/upgrade
 
 .PHONY: test-deps
 test-deps:
@@ -65,16 +68,16 @@ test-deps:
 	go install github.com/onsi/ginkgo/v2/ginkgo
 
 .PHONY: test-cli
-test-cli: $(GINKGO)
-	$(GINKGO) run --label-filter '!root' --fail-fast --race --covermode=atomic --coverprofile=coverage.txt --coverpkg=github.com/rancher/elemental-toolkit/... -p -r ${PKG}
+test-cli:
+	go run $(GINKGO) --label-filter '!root' --fail-fast --race --covermode=atomic --coverprofile=coverage.txt --coverpkg=github.com/rancher/elemental-toolkit/... -p -r ${PKG}
 
 .PHONY: test-root
-test-root: $(GINKGO)
+test-root:
 ifneq ($(shell id -u), 0)
 	@echo "This tests require root/sudo to run."
 	@exit 1
 else
-	$(GINKGO) run --label-filter root --fail-fast --race --covermode=atomic --coverprofile=coverage_root.txt --coverpkg=github.com/rancher/elemental-toolkit/... -procs=1 -r ${PKG}
+	go run $(GINKGO) run --label-filter root --fail-fast --race --covermode=atomic --coverprofile=coverage_root.txt --coverpkg=github.com/rancher/elemental-toolkit/... -procs=1 -r ${PKG}
 endif
 
 
diff --git a/scripts/run_vm.sh b/scripts/run_vm.sh
@@ -6,19 +6,20 @@ SCRIPT=$(realpath -s "${0}")
 SCRIPTS_PATH=$(dirname "${SCRIPT}")
 TESTS_PATH=$(realpath -s "${SCRIPTS_PATH}/../tests")
 
+: "${ELMNTL_PREFIX:=}" 
 : "${ELMNTL_FIRMWARE:=/usr/share/qemu/ovmf-x86_64.bin}"
 : "${ELMNTL_FWDIP:=127.0.0.1}"
 : "${ELMNTL_FWDPORT:=2222}"
 : "${ELMNTL_MEMORY:=4096}"
-: "${ELMNTL_LOGFILE:=${TESTS_PATH}/serial.log}"
-: "${ELMNTL_PIDFILE:=${TESTS_PATH}/testvm.pid}"
-: "${ELMNTL_TESTDISK:=${TESTS_PATH}/testdisk.qcow2}"
-: "${ELMNTL_DISKSIZE:=20G}"
+: "${ELMNTL_LOGFILE:=${TESTS_PATH}/${ELMNTL_PREFIX}serial.log}"
+: "${ELMNTL_PIDFILE:=${TESTS_PATH}/${ELMNTL_PREFIX}testvm.pid}"
+: "${ELMNTL_TESTDISK:=${TESTS_PATH}/${ELMNTL_PREFIX}testdisk.qcow2}"
+: "${ELMNTL_VMSTDOUT:=${TESTS_PATH}/${ELMNTL_PREFIX}vmstdout}"
+: "${ELMNTL_DISKSIZE:=16G}"
 : "${ELMNTL_DISPLAY:=none}"
 : "${ELMNTL_ACCEL:=kvm}"
 : "${ELMNTL_TARGETARCH:=$(uname -p)}"
 : "${ELMNTL_MACHINETYPE:=q35}"
-: "${ELMNTL_CPU:=max}"
 
 function _abort {
     echo "$@" && exit 1
@@ -37,7 +38,7 @@ function start {
   local daemon_arg="-daemonize"
   local machine_arg="-machine type=${ELMNTL_MACHINETYPE}"
   local cdrom_arg
-  local cpu_arg="-cpu ${ELMNTL_CPU}"
+  local cpu_arg
   local vmpid
   local kvm_arg
 
@@ -67,12 +68,12 @@ function start {
         ;;
   esac
 
-  [ "hvf" == "${ELMNTL_ACCEL}" ] && accel_arg="-accel ${ELMNTL_ACCEL}" && firmware_arg="-bios ${ELMNTL_FIRMWARE} ${firmware_arg}"
+  [ "hvf" == "${ELMNTL_ACCEL}" ] && accel_arg="-accel ${ELMNTL_ACCEL}" && firmware_arg="-bios ${ELMNTL_FIRMWARE} ${firmware_arg}" && cpu_arg="-cpu max,-pdpe1gb"
   [ "kvm" == "${ELMNTL_ACCEL}" ] && cpu_arg="-cpu host" && kvm_arg="-enable-kvm"
 
   qemu-system-${ELMNTL_TARGETARCH} ${kvm_arg} ${disk_arg} ${cdrom_arg} ${firmware_arg} ${usrnet_arg} \
       ${kvm_arg} ${memory_arg} ${graphics_arg} ${serial_arg} ${pidfile_arg} \
-      ${daemon_arg} ${display_arg} ${machine_arg} ${accel_arg} ${cpu_arg}
+      ${display_arg} ${machine_arg} ${accel_arg} ${cpu_arg} > ${ELMNTL_VMSTDOUT} 2>&1 &
 }
 
 function stop {
@@ -96,6 +97,13 @@ function stop {
 function clean {
   ([ -f "${ELMNTL_LOGFILE}" ] && rm -f "${ELMNTL_LOGFILE}") || true
   ([ -f "${ELMNTL_TESTDISK}" ] && rm -f "${ELMNTL_TESTDISK}") || true
+  ([ -f "${ELMNTL_VMSTDOUT}" ] && rm -f "${ELMNTL_VMSTDOUT}") || true
+}
+
+function vmpid {
+    if [ -f "${ELMNTL_PIDFILE}" ]; then
+        cat "${ELMNTL_PIDFILE}"
+    fi
 }
 
 cmd=$1
@@ -111,6 +119,9 @@ case $cmd in
   clean)
     clean
     ;;
+  vmpid)
+    vmpid
+    ;;
   *)
     _abort "Unknown command: ${cmd}"
     ;;

diff --git a/tests/assets/custom_partitions.yaml b/tests/assets/custom_partitions.yaml
@@ -6,7 +6,7 @@ install:
       size: 4096
       fs: ext2
     persistent:
-      size: 8192
+      size: 1024
       fs: ext2
     oem:
       size: 128
diff --git a/tests/installer/installer_efi_test.go b/tests/installer/installer_efi_test.go
@@ -70,7 +70,7 @@ var _ = Describe("Elemental Installer EFI tests", func() {
 						},
 						{
 							Label:  "COS_PERSISTENT",
-							Size:   8192,
+							Size:   1024,
 							FsType: sut.Ext2,
 						},
 					} {

diff --git a/vendor/github.com/rancher-sandbox/ele-testhelpers/vm/sut.go b/vendor/github.com/rancher-sandbox/ele-testhelpers/vm/sut.go
diff --git a/vendor/modules.txt b/vendor/modules.txt
@@ -476,7 +476,7 @@ github.com/pkg/errors
 # github.com/pkg/xattr v0.4.9
 ## explicit; go 1.14
 github.com/pkg/xattr
-# github.com/rancher-sandbox/ele-testhelpers v0.0.0-20230907090440-1cc5d66b1725
+# github.com/rancher-sandbox/ele-testhelpers v0.0.0-20231031082920-8b1cf3f8c16f
 ## explicit; go 1.20
 github.com/rancher-sandbox/ele-testhelpers/vm
 # github.com/rancher-sandbox/linuxkit v1.0.1