Tests: apply embedded capi ff after rancher install (#190)

Signed-off-by: Danil Grigorev <danil.grigorev@suse.com>
rancher · Oct 9, 2023 · 8b8ecf3 · 8b8ecf3
1 parent f9cf357
commit 8b8ecf3
Show file tree

Hide file tree

Showing 7 changed files with 130 additions and 81 deletions.
diff --git a/Makefile b/Makefile
@@ -180,7 +180,7 @@ help: ## Display this help.
 ##@ Development
 .PHONY: manifests
 manifests: vendor controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
-	$(CONTROLLER_GEN) rbac:roleName=manager-role crd paths="./..." output:crd:artifacts:config=hack/crd/bases
+	$(CONTROLLER_GEN) rbac:roleName=manager-role crd paths="./internal/rancher/..." output:crd:artifacts:config=hack/crd/bases
 	$(CONTROLLER_GEN) rbac:roleName=manager-role crd paths="./vendor/sigs.k8s.io/cluster-api/..." output:crd:artifacts:config=hack/crd/bases
 	# Vendor is only required for pulling latest CRDs from the dependencies
 	$(MAKE) vendor-clean
@@ -198,7 +198,6 @@ vendor-clean:
 
 .PHONY: generate
 generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
-	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
 
 .PHONY: generate-modules
 generate-modules: ## Run go mod tidy to ensure modules are up to date
@@ -249,11 +248,11 @@ test: $(SETUP_ENVTEST) manifests ## Run tests.
 ##@ Build
 
 .PHONY: build
-build: manifests generate fmt vet ## Build manager binary.
+build: generate fmt vet ## Build manager binary.
 	go build -o bin/manager main.go
 
 .PHONY: run
-run: manifests generate fmt vet ## Run a controller from your host.
+run: generate fmt vet ## Run a controller from your host.
 	go run ./main.go
 
 ## --------------------------------------

diff --git a/charts/rancher-turtles/values.yaml b/charts/rancher-turtles/values.yaml
@@ -3,7 +3,7 @@ rancherTurtles:
   tag: v0.0.0
   imagePullPolicy: Never
   namespace: rancher-turtles-system
-  managerArguments: {}
+  managerArguments: []
   imagePullSecrets: []
   features:
     embedded-capi:
@@ -14,7 +14,7 @@ rancherTurtles:
 cluster-api-operator:
   enabled: true
   cert-manager:
-    enabled: true
+    enabled: false
   cluster-api:
     enabled: true
     version: v1.4.6

diff --git a/test/e2e/config/operator.yaml b/test/e2e/config/operator.yaml
@@ -21,13 +21,16 @@ intervals:
 variables:
   RANCHER_VERSION: "v2.7.6"
   RANCHER_HOSTNAME: "localhost"
-  RANCHER_FEATURES: "embedded-cluster-api=false"
+  RANCHER_FEATURES: ""
   RANCHER_PATH: "rancher-stable/rancher"
   KUBERNETES_VERSION: "v1.26.3"
   RKE2_VERSION: "v1.26.8+rke2r1"
   CAPI_CORE: "cluster-api:v1.4.6"
   RANCHER_REPO_NAME: "rancher-stable"
   RANCHER_URL: "https://releases.rancher.com/server-charts/stable"
+  CERT_MANAGER_URL: "https://charts.jetstack.io"
+  CERT_MANAGER_REPO_NAME: "jetstack"
+  CERT_MANAGER_PATH: "jetstack/cert-manager"
   RANCHER_PASSWORD: "rancheradmin"
   # Additional setup for establishing rancher ingress
   NGROK_REPO_NAME: "ngrok"

diff --git a/test/e2e/const.go b/test/e2e/const.go
@@ -71,16 +71,19 @@ const (
 )
 
 const (
-	KubernetesVersionVar  = "KUBERNETES_VERSION"
-	RancherFeaturesVar    = "RANCHER_FEATURES"
-	RancherHostnameVar    = "RANCHER_HOSTNAME"
-	RancherVersionVar     = "RANCHER_VERSION"
-	RancherPathVar        = "RANCHER_PATH"
-	RancherUrlVar         = "RANCHER_URL"
-	RancherRepoNameVar    = "RANCHER_REPO_NAME"
-	RancherPasswordVar    = "RANCHER_PASSWORD"
-	CapiInfrastructureVar = "CAPI_INFRASTRUCTURE"
-	CapiCoreVar           = "CAPI_CORE"
+	KubernetesVersionVar   = "KUBERNETES_VERSION"
+	RancherFeaturesVar     = "RANCHER_FEATURES"
+	RancherHostnameVar     = "RANCHER_HOSTNAME"
+	RancherVersionVar      = "RANCHER_VERSION"
+	RancherPathVar         = "RANCHER_PATH"
+	RancherUrlVar          = "RANCHER_URL"
+	RancherRepoNameVar     = "RANCHER_REPO_NAME"
+	RancherPasswordVar     = "RANCHER_PASSWORD"
+	CertManagerUrlVar      = "CERT_MANAGER_URL"
+	CertManagerRepoNameVar = "CERT_MANAGER_REPO_NAME"
+	CertManagerPathVar     = "CERT_MANAGER_PATH"
+	CapiInfrastructureVar  = "CAPI_INFRASTRUCTURE"
+	CapiCoreVar            = "CAPI_CORE"
 
 	NgrokRepoNameVar  = "NGROK_REPO_NAME"
 	NgrokUrlVar       = "NGROK_URL"

diff --git a/test/e2e/suites/import-gitops/suite_test.go b/test/e2e/suites/import-gitops/suite_test.go
@@ -104,6 +104,42 @@ var _ = BeforeSuite(func() {
 		hostName = setupClusterResult.IsolatedHostName
 	}
 
+	testenv.RancherDeployIngress(ctx, testenv.RancherDeployIngressInput{
+		BootstrapClusterProxy:    setupClusterResult.BootstrapClusterProxy,
+		HelmBinaryPath:           flagVals.HelmBinaryPath,
+		IsolatedMode:             flagVals.IsolatedMode,
+		NginxIngress:             e2e.NginxIngress,
+		NginxIngressNamespace:    e2e.NginxIngressNamespace,
+		IngressWaitInterval:      e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-rancher"),
+		NgrokApiKey:              e2eConfig.GetVariable(e2e.NgrokApiKeyVar),
+		NgrokAuthToken:           e2eConfig.GetVariable(e2e.NgrokAuthTokenVar),
+		NgrokPath:                e2eConfig.GetVariable(e2e.NgrokPathVar),
+		NgrokRepoName:            e2eConfig.GetVariable(e2e.NgrokRepoNameVar),
+		NgrokRepoURL:             e2eConfig.GetVariable(e2e.NgrokUrlVar),
+		DefaultIngressClassPatch: e2e.IngressClassPatch,
+	})
+
+	testenv.DeployRancher(ctx, testenv.DeployRancherInput{
+		BootstrapClusterProxy:  setupClusterResult.BootstrapClusterProxy,
+		HelmBinaryPath:         flagVals.HelmBinaryPath,
+		CertManagerChartPath:   e2eConfig.GetVariable(e2e.CertManagerPathVar),
+		CertManagerUrl:         e2eConfig.GetVariable(e2e.CertManagerUrlVar),
+		CertManagerRepoName:    e2eConfig.GetVariable(e2e.CertManagerRepoNameVar),
+		RancherChartRepoName:   e2eConfig.GetVariable(e2e.RancherRepoNameVar),
+		RancherChartURL:        e2eConfig.GetVariable(e2e.RancherUrlVar),
+		RancherChartPath:       e2eConfig.GetVariable(e2e.RancherPathVar),
+		RancherVersion:         e2eConfig.GetVariable(e2e.RancherVersionVar),
+		RancherHost:            hostName,
+		RancherNamespace:       e2e.RancherNamespace,
+		RancherPassword:        e2eConfig.GetVariable(e2e.RancherPasswordVar),
+		RancherPatches:         [][]byte{e2e.RancherSettingPatch},
+		RancherWaitInterval:    e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-rancher"),
+		ControllerWaitInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers"),
+		IsolatedMode:           flagVals.IsolatedMode,
+		RancherIngressConfig:   e2e.IngressConfig,
+		RancherServicePatch:    e2e.RancherServicePatch,
+	})
+
 	testenv.DeployRancherTurtles(ctx, testenv.DeployRancherTurtlesInput{
 		BootstrapClusterProxy:        setupClusterResult.BootstrapClusterProxy,
 		HelmBinaryPath:               flagVals.HelmBinaryPath,
@@ -146,40 +182,6 @@ var _ = BeforeSuite(func() {
 		})
 	}
 
-	testenv.RancherDeployIngress(ctx, testenv.RancherDeployIngressInput{
-		BootstrapClusterProxy:    setupClusterResult.BootstrapClusterProxy,
-		HelmBinaryPath:           flagVals.HelmBinaryPath,
-		IsolatedMode:             flagVals.IsolatedMode,
-		NginxIngress:             e2e.NginxIngress,
-		NginxIngressNamespace:    e2e.NginxIngressNamespace,
-		IngressWaitInterval:      e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-rancher"),
-		NgrokApiKey:              e2eConfig.GetVariable(e2e.NgrokApiKeyVar),
-		NgrokAuthToken:           e2eConfig.GetVariable(e2e.NgrokAuthTokenVar),
-		NgrokPath:                e2eConfig.GetVariable(e2e.NgrokPathVar),
-		NgrokRepoName:            e2eConfig.GetVariable(e2e.NgrokRepoNameVar),
-		NgrokRepoURL:             e2eConfig.GetVariable(e2e.NgrokUrlVar),
-		DefaultIngressClassPatch: e2e.IngressClassPatch,
-	})
-
-	testenv.DeployRancher(ctx, testenv.DeployRancherInput{
-		BootstrapClusterProxy:  setupClusterResult.BootstrapClusterProxy,
-		HelmBinaryPath:         flagVals.HelmBinaryPath,
-		RancherChartRepoName:   e2eConfig.GetVariable(e2e.RancherRepoNameVar),
-		RancherChartURL:        e2eConfig.GetVariable(e2e.RancherUrlVar),
-		RancherChartPath:       e2eConfig.GetVariable(e2e.RancherPathVar),
-		RancherVersion:         e2eConfig.GetVariable(e2e.RancherVersionVar),
-		RancherHost:            hostName,
-		RancherNamespace:       e2e.RancherNamespace,
-		RancherPassword:        e2eConfig.GetVariable(e2e.RancherPasswordVar),
-		RancherFeatures:        e2eConfig.GetVariable(e2e.RancherFeaturesVar),
-		RancherSettingsPatch:   e2e.RancherSettingPatch,
-		RancherWaitInterval:    e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-rancher"),
-		ControllerWaitInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers"),
-		IsolatedMode:           flagVals.IsolatedMode,
-		RancherIngressConfig:   e2e.IngressConfig,
-		RancherServicePatch:    e2e.RancherServicePatch,
-	})
-
 	giteaResult = testenv.DeployGitea(ctx, testenv.DeployGiteaInput{
 		BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy,
 		HelmBinaryPath:        flagVals.HelmBinaryPath,

diff --git a/test/e2e/suites/v2prov/suite_test.go b/test/e2e/suites/v2prov/suite_test.go
@@ -103,18 +103,6 @@ var _ = BeforeSuite(func() {
 		hostName = setupClusterResult.IsolatedHostName
 	}
 
-	testenv.DeployRancherTurtles(ctx, testenv.DeployRancherTurtlesInput{
-		BootstrapClusterProxy:        setupClusterResult.BootstrapClusterProxy,
-		HelmBinaryPath:               flagVals.HelmBinaryPath,
-		ChartPath:                    flagVals.ChartPath,
-		CAPIProvidersSecretYAML:      e2e.CapiProvidersSecret,
-		CAPIProvidersYAML:            e2e.CapiProviders,
-		Namespace:                    turtlesframework.DefaultRancherTurtlesNamespace,
-		Image:                        "ghcr.io/rancher-sandbox/rancher-turtles-amd64",
-		Tag:                          "v0.0.1",
-		WaitDeploymentsReadyInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers"),
-	})
-
 	testenv.RancherDeployIngress(ctx, testenv.RancherDeployIngressInput{
 		BootstrapClusterProxy:    setupClusterResult.BootstrapClusterProxy,
 		HelmBinaryPath:           flagVals.HelmBinaryPath,
@@ -133,6 +121,9 @@ var _ = BeforeSuite(func() {
 	testenv.DeployRancher(ctx, testenv.DeployRancherInput{
 		BootstrapClusterProxy: setupClusterResult.BootstrapClusterProxy,
 		HelmBinaryPath:        flagVals.HelmBinaryPath,
+		CertManagerChartPath:  e2eConfig.GetVariable(e2e.CertManagerPathVar),
+		CertManagerUrl:        e2eConfig.GetVariable(e2e.CertManagerUrlVar),
+		CertManagerRepoName:   e2eConfig.GetVariable(e2e.CertManagerRepoNameVar),
 		RancherChartRepoName:  "rancher-latest",
 		RancherChartURL:       "https://releases.rancher.com/server-charts/latest",
 		RancherChartPath:      "rancher-latest/rancher",
@@ -143,13 +134,26 @@ var _ = BeforeSuite(func() {
 		RancherNamespace:       e2e.RancherNamespace,
 		RancherPassword:        e2eConfig.GetVariable(e2e.RancherPasswordVar),
 		RancherFeatures:        e2eConfig.GetVariable(e2e.RancherFeaturesVar),
-		RancherSettingsPatch:   e2e.RancherSettingPatch,
+		RancherPatches:         [][]byte{e2e.RancherSettingPatch},
 		RancherWaitInterval:    e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-rancher"),
 		ControllerWaitInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers"),
 		IsolatedMode:           flagVals.IsolatedMode,
 		RancherIngressConfig:   e2e.IngressConfig,
 		RancherServicePatch:    e2e.RancherServicePatch,
 	})
+
+	testenv.DeployRancherTurtles(ctx, testenv.DeployRancherTurtlesInput{
+		BootstrapClusterProxy:        setupClusterResult.BootstrapClusterProxy,
+		HelmBinaryPath:               flagVals.HelmBinaryPath,
+		ChartPath:                    flagVals.ChartPath,
+		CAPIProvidersSecretYAML:      e2e.CapiProvidersSecret,
+		CAPIProvidersYAML:            e2e.CapiProviders,
+		Namespace:                    turtlesframework.DefaultRancherTurtlesNamespace,
+		Image:                        "ghcr.io/rancher-sandbox/rancher-turtles-amd64",
+		Tag:                          "v0.0.1",
+		WaitDeploymentsReadyInterval: e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers"),
+	})
+
 })
 
 var _ = AfterSuite(func() {

diff --git a/test/testenv/rancher.go b/test/testenv/rancher.go
@@ -22,20 +22,27 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	turtlesframework "github.com/rancher-sandbox/rancher-turtles/test/framework"
 
 	"github.com/drone/envsubst/v2"
+	"github.com/rancher-sandbox/rancher-turtles/test/e2e"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	opframework "sigs.k8s.io/cluster-api-operator/test/framework"
 	"sigs.k8s.io/cluster-api/test/framework"
+	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/envtest/komega"
 )
 
 type DeployRancherInput struct {
 	BootstrapClusterProxy  framework.ClusterProxy
+	E2EConfig              *clusterctl.E2EConfig
 	HelmBinaryPath         string
+	CertManagerChartPath   string
+	CertManagerUrl         string
+	CertManagerRepoName    string
 	RancherChartRepoName   string
 	RancherChartURL        string
 	RancherChartPath       string
@@ -45,7 +52,7 @@ type DeployRancherInput struct {
 	RancherHost            string
 	RancherPassword        string
 	RancherFeatures        string
-	RancherSettingsPatch   []byte
+	RancherPatches         [][]byte
 	RancherWaitInterval    []interface{}
 	ControllerWaitInterval []interface{}
 	IsolatedMode           bool
@@ -59,6 +66,9 @@ func DeployRancher(ctx context.Context, input DeployRancherInput) {
 	Expect(ctx).NotTo(BeNil(), "ctx is required for DeployRancher")
 	Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "BootstrapClusterProxy is required for DeployRancher")
 	Expect(input.HelmBinaryPath).ToNot(BeEmpty(), "HelmBinaryPath is required for DeployRancher")
+	Expect(input.CertManagerRepoName).ToNot(BeEmpty(), "CertManagerRepoName is required for DeployRancher")
+	Expect(input.CertManagerUrl).ToNot(BeEmpty(), "CertManagerUrl is required for DeployRancher")
+	Expect(input.CertManagerChartPath).ToNot(BeEmpty(), "CertManagerChartPath is required for DeployRancher")
 	Expect(input.RancherChartRepoName).ToNot(BeEmpty(), "RancherChartRepoName is required for DeployRancher")
 	Expect(input.RancherChartURL).ToNot(BeEmpty(), "RancherChartURL is required for DeployRancher")
 	Expect(input.RancherChartPath).ToNot(BeEmpty(), "RancherChartPath is required for DeployRancher")
@@ -75,16 +85,28 @@ func DeployRancher(ctx context.Context, input DeployRancherInput) {
 		Fail("Only one of RancherVersion or RancherImageTag cen be used")
 	}
 
-	By("Adding Rancher chart repo")
+	By("Add cert manager chart repo")
 	addChart := &opframework.HelmChart{
+		BinaryPath:      input.HelmBinaryPath,
+		Name:            input.CertManagerRepoName,
+		Path:            input.CertManagerUrl,
+		Commands:        opframework.Commands(opframework.Repo, opframework.Add),
+		AdditionalFlags: opframework.Flags("--force-update"),
+		Kubeconfig:      input.BootstrapClusterProxy.GetKubeconfigPath(),
+	}
+	_, err := addChart.Run(nil)
+	Expect(err).ToNot(HaveOccurred())
+
+	By("Adding Rancher chart repo")
+	addChart = &opframework.HelmChart{
 		BinaryPath:      input.HelmBinaryPath,
 		Name:            input.RancherChartRepoName,
 		Path:            input.RancherChartURL,
 		Commands:        opframework.Commands(opframework.Repo, opframework.Add),
 		AdditionalFlags: opframework.Flags("--force-update"),
 		Kubeconfig:      input.BootstrapClusterProxy.GetKubeconfigPath(),
 	}
-	_, err := addChart.Run(nil)
+	_, err = addChart.Run(nil)
 	Expect(err).ToNot(HaveOccurred())
 
 	updateChart := &opframework.HelmChart{
@@ -95,11 +117,28 @@ func DeployRancher(ctx context.Context, input DeployRancherInput) {
 	_, err = updateChart.Run(nil)
 	Expect(err).ToNot(HaveOccurred())
 
+	By("Installing cert-manager")
+	chart := &opframework.HelmChart{
+		BinaryPath: input.HelmBinaryPath,
+		Path:       input.CertManagerChartPath,
+		Name:       "cert-manager",
+		Kubeconfig: input.BootstrapClusterProxy.GetKubeconfigPath(),
+		AdditionalFlags: opframework.Flags(
+			"--namespace", "cert-manager",
+			"--version", "v1.12.0",
+			"--create-namespace",
+		),
+		Wait: true,
+	}
+	_, err = chart.Run(map[string]string{
+		"installCRDs": "true",
+	})
+	Expect(err).ToNot(HaveOccurred())
+
 	By("Installing Rancher")
 	installFlags := opframework.Flags(
 		"--namespace", input.RancherNamespace,
 		"--create-namespace",
-		"--wait",
 	)
 	if input.RancherVersion != "" {
 		installFlags = append(installFlags, "--version", input.RancherVersion)
@@ -108,12 +147,13 @@ func DeployRancher(ctx context.Context, input DeployRancherInput) {
 		installFlags = append(installFlags, "--devel")
 	}
 
-	chart := &opframework.HelmChart{
+	chart = &opframework.HelmChart{
 		BinaryPath:      input.HelmBinaryPath,
 		Path:            input.RancherChartPath,
 		Name:            "rancher",
 		Kubeconfig:      input.BootstrapClusterProxy.GetKubeconfigPath(),
 		AdditionalFlags: installFlags,
+		Wait:            true,
 	}
 	values := map[string]string{
 		"bootstrapPassword":         input.RancherPassword,
@@ -131,18 +171,16 @@ func DeployRancher(ctx context.Context, input DeployRancherInput) {
 	_, err = chart.Run(values)
 	Expect(err).ToNot(HaveOccurred())
 
-	if len(input.RancherSettingsPatch) > 0 {
-		By("Updating rancher settings")
-		settingPatch, err := envsubst.Eval(string(input.RancherSettingsPatch), func(s string) string {
-			switch s {
-			case "RANCHER_HOSTNAME":
-				return input.RancherHost
-			default:
-				return os.Getenv(s)
-			}
-		})
-		Expect(err).ToNot(HaveOccurred())
-		Expect(input.BootstrapClusterProxy.Apply(ctx, []byte(settingPatch))).To(Succeed())
+	By("Updating rancher configuration")
+	for _, patch := range input.RancherPatches {
+		Expect(turtlesframework.ApplyFromTemplate(ctx, turtlesframework.ApplyFromTemplateInput{
+			Proxy:    input.BootstrapClusterProxy,
+			Template: patch,
+			Getter:   input.E2EConfig.GetVariable,
+			AddtionalEnvironmentVariables: map[string]string{
+				e2e.RancherHostnameVar: input.RancherHost,
+			},
+		})).To(Succeed())
 	}
 
 	if !input.IsolatedMode {