Add ironic and bmo cleanup for upgrade BMO E2E tests

Signed-off-by: Huy Mai <huy.mai@est.tech>

test/e2e/common.go

@@ -1,6 +1,7 @@
 package e2e
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"os"
@@ -21,6 +22,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	metal3api "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1"
+	testexec "sigs.k8s.io/cluster-api/test/framework/exec"
 
 	capm3_e2e "github.com/metal3-io/cluster-api-provider-metal3/test/e2e"
 
@@ -465,6 +467,7 @@ func BuildAndApplyKustomize(ctx context.Context, input *BuildAndApplyKustomizeIn
 	if err != nil {
 		return err
 	}
+
 	err = clusterProxy.Apply(ctx, manifest)
 	if err != nil {
 		return err
@@ -518,3 +521,30 @@ func DeploymentRolledOut(ctx context.Context, clusterProxy framework.ClusterProx
 	}
 	return false
 }
+
+// KubectlDelete shells out to kubectl delete.
+func KubectlDelete(ctx context.Context, kubeconfigPath string, resources []byte, args ...string) error {
+	aargs := append([]string{"delete", "--kubeconfig", kubeconfigPath, "-f", "-"}, args...)
+	rbytes := bytes.NewReader(resources)
+	deleteCmd := testexec.NewCommand(
+		testexec.WithCommand("kubectl"),
+		testexec.WithArgs(aargs...),
+		testexec.WithStdin(rbytes),
+	)
+
+	fmt.Printf("Running kubectl %s\n", strings.Join(aargs, " "))
+	stdout, stderr, err := deleteCmd.Run(ctx)
+	fmt.Printf("stderr:\n%s\n", string(stderr))
+	fmt.Printf("stdout:\n%s\n", string(stdout))
+	return err
+}
+
+// BuildAndRemoveKustomize builds the provided kustomization to resources and removes them from the cluster
+// provided by clusterProxy
+func BuildAndRemoveKustomize(ctx context.Context, kustomization string, clusterProxy framework.ClusterProxy) error {
+	manifest, err := buildKustomizeManifest(kustomization)
+	if err != nil {
+		return err
+	}
+	return KubectlDelete(ctx, clusterProxy.GetKubeconfigPath(), manifest)
+}

test/e2e/upgrade_test.go

@@ -175,13 +175,12 @@ var _ = Describe("BMO Upgrade", func() {
 		specName               = "upgrade"
 		secretName             = "bmc-credentials"
 		namespace              *corev1.Namespace
-		bmoIronicNamespace     string
+		bmoIronicNamespace     = "baremetal-operator-system"
 		upgradeClusterProvider bootstrap.ClusterProvider
 		upgradeClusterProxy    framework.ClusterProxy
 		bmh                    metal3api.BareMetalHost
 	)
 	BeforeEach(func() {
-		bmoIronicNamespace = "baremetal-operator-system"
 		var kubeconfigPath string
 
 		if useExistingCluster {
@@ -203,6 +202,7 @@ var _ = Describe("BMO Upgrade", func() {
 		framework.TryAddDefaultSchemes(scheme)
 		metal3api.AddToScheme(scheme)
 		upgradeClusterProxy = framework.NewClusterProxy("bmo-e2e-upgrade", kubeconfigPath, scheme)
+
 		if e2eConfig.GetVariable("UPGRADE_DEPLOY_CERT_MANAGER") != "false" {
 			By("Installing cert-manager on the upgrade cluster")
 			cmVersion := e2eConfig.GetVariable("CERT_MANAGER_VERSION")
@@ -215,12 +215,15 @@ var _ = Describe("BMO Upgrade", func() {
 			err = checkCertManagerAPI(upgradeClusterProxy)
 			Expect(err).NotTo(HaveOccurred())
 		}
+	})
 
+	It("Should upgrade BMO to latest version", func() {
 		if e2eConfig.GetVariable("UPGRADE_DEPLOY_IRONIC") != "false" {
 			// Install Ironic
+			ironicKustomization := e2eConfig.GetVariable("IRONIC_KUSTOMIZATION")
 			By("Installing Ironic on the upgrade cluster")
 			err := BuildAndApplyKustomize(ctx, &BuildAndApplyKustomizeInput{
-				Kustomization:       e2eConfig.GetVariable("IRONIC_KUSTOMIZATION"),
+				Kustomization:       ironicKustomization,
 				ClusterProxy:        upgradeClusterProxy,
 				WaitForDeployment:   true,
 				WatchDeploymentLogs: true,
@@ -230,6 +233,10 @@ var _ = Describe("BMO Upgrade", func() {
 				WaitIntervals:       e2eConfig.GetIntervals("default", "wait-deployment"),
 			})
 			Expect(err).NotTo(HaveOccurred())
+			DeferCleanup(func() {
+				By("Removing Ironic on the upgrade cluster")
+				BuildAndRemoveKustomize(ctx, ironicKustomization, upgradeClusterProxy)
+			})
 		}
 
 		if e2eConfig.GetVariable("UPGRADE_DEPLOY_BMO") != "false" {
@@ -247,6 +254,10 @@ var _ = Describe("BMO Upgrade", func() {
 				WaitIntervals:       e2eConfig.GetIntervals("default", "wait-deployment"),
 			})
 			Expect(err).NotTo(HaveOccurred())
+			DeferCleanup(func() {
+				By(fmt.Sprintf("Removing BMO from %s on the upgrade cluster", bmoKustomization))
+				BuildAndRemoveKustomize(ctx, bmoKustomization, upgradeClusterProxy)
+			})
 		}
 
 		namespace, cancelWatches = framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{
@@ -255,9 +266,6 @@ var _ = Describe("BMO Upgrade", func() {
 			Name:      fmt.Sprintf("%s-%s", specName, util.RandomString(6)),
 			LogFolder: artifactFolder,
 		})
-	})
-
-	It("Should upgrade BMO to latest version", func() {
 		By("Creating a secret with BMH credentials")
 		bmcCredentialsData := map[string]string{
 			"username": bmc.User,
@@ -314,6 +322,10 @@ var _ = Describe("BMO Upgrade", func() {
 			LogPath:             filepath.Join(artifactFolder, "logs", fmt.Sprintf("%s-%s", bmoIronicNamespace, specName), fmt.Sprintf("bmo-%s", bmoKustomizationName)),
 			WaitIntervals:       e2eConfig.GetIntervals("default", "wait-deployment"),
 		})
+		DeferCleanup(func() {
+			By("Removing BMO main e2e deployment")
+			BuildAndRemoveKustomize(ctx, bmoKustomization, upgradeClusterProxy)
+		})
 		Expect(err).NotTo(HaveOccurred())
 		By("Waiting for BMO update to rollout")
 		Eventually(func() bool {