E2E: Add custom ingress for gitea

Signed-off-by: Alexandr Demicev <alexandr.demicev@suse.com>

Makefile

@@ -76,6 +76,7 @@ SKIP_RESOURCE_CLEANUP ?= false
 USE_EXISTING_CLUSTER ?= false
 USE_EKS ?= true
 ISOLATED_MODE ?= false
+GITEA_CUSTOM_INGRESS ?= false
 GINKGO_NOCOLOR ?= false
 GINKGO_LABEL_FILTER ?= short
 GINKGO_TESTS ?= $(ROOT_DIR)/$(TEST_DIR)/e2e/suites/...
@@ -535,7 +536,8 @@ test-e2e: $(GINKGO) $(HELM) $(CLUSTERCTL) kubectl e2e-image ## Run the end-to-en
 	    -e2e.skip-resource-cleanup=$(SKIP_RESOURCE_CLEANUP) \
 		-e2e.use-existing-cluster=$(USE_EXISTING_CLUSTER) \
 		-e2e.isolated-mode=$(ISOLATED_MODE) \
-		-e2e.use-eks=$(USE_EKS)
+		-e2e.use-eks=$(USE_EKS) \
+		-e2e.gitea-custom-ingress=$(GITEA_CUSTOM_INGRESS)
 
 .PHONY: e2e-image
 e2e-image: ## Build the image for e2e tests

test/e2e/const.go

@@ -80,6 +80,9 @@ var (
 
 	//go:embed data/cluster-templates/vsphere-kubeadm.yaml
 	CAPIvSphereKubeadm []byte
+
+	//go:embed data/gitea/ingress.yaml
+	GiteaIngress []byte
 )
 
 const (

test/e2e/data/gitea/ingress.yaml

@@ -0,0 +1,18 @@
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: gitea-http
+  namespace: default
+spec:
+  ingressClassName: ngrok # This ingress in intended to be used only with ngrok
+  rules:
+  - host: gitea.${RANCHER_HOSTNAME}
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: gitea-http
+            port:
+              number: 3000

test/e2e/flags.go

@@ -54,6 +54,9 @@ type FlagValues struct {
 
 	// ClusterctlBinaryPath is the path to the clusterctl binary to use.
 	ClusterctlBinaryPath string
+
+	// GiteaCustomIngress is the flag to enable custom ingress for Gitea.
+	GiteaCustomIngress bool
 }
 
 // InitFlags is used to specify the standard flags for the e2e tests.
@@ -68,4 +71,5 @@ func InitFlags(values *FlagValues) {
 	flag.StringVar(&values.ClusterctlBinaryPath, "e2e.clusterctl-binary-path", "helm", "path to the clusterctl binary")
 	flag.StringVar(&values.ChartPath, "e2e.chart-path", "", "path to the operator chart")
 	flag.BoolVar(&values.IsolatedMode, "e2e.isolated-mode", false, "if true, the test will run without ngrok and exposing the cluster to the internet. This setup will only work with CAPD or other providers that run in the same network as the bootstrap cluster.")
+	flag.BoolVar(&values.GiteaCustomIngress, "e2e.gitea-custom-ingress", false, "if true, the test will use a custom ingress for Gitea")
 }

test/e2e/helpers.go

@@ -41,6 +41,7 @@ import (
 	managementv3 "github.com/rancher/turtles/internal/rancher/management/v3"
 	provisioningv1 "github.com/rancher/turtles/internal/rancher/provisioning/v1"
 	turtlesframework "github.com/rancher/turtles/test/framework"
+	networkingv1 "k8s.io/api/networking/v1"
 )
 
 func SetupSpecNamespace(ctx context.Context, specName string, clusterProxy framework.ClusterProxy, artifactFolder string) (*corev1.Namespace, context.CancelFunc) {
@@ -86,6 +87,7 @@ func InitScheme() *runtime.Scheme {
 	Expect(clusterv1.AddToScheme(scheme)).To(Succeed())
 	Expect(provisioningv1.AddToScheme(scheme)).To(Succeed())
 	Expect(managementv3.AddToScheme(scheme)).To(Succeed())
+	Expect(networkingv1.AddToScheme(scheme)).To(Succeed())
 	return scheme
 }
 

test/e2e/suites/embedded-capi-disabled/suite_test.go

@@ -29,6 +29,7 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/klog/v2"
 	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -243,10 +244,15 @@ var _ = BeforeSuite(func() {
 	giteaValues := map[string]string{
 		"gitea.admin.username": e2eConfig.GetVariable(e2e.GiteaUserNameVar),
 		"gitea.admin.password": e2eConfig.GetVariable(e2e.GiteaUserPasswordVar),
-		"service.http.type":    "NodePort",
 	}
+
+	giteaServiceType := corev1.ServiceTypeNodePort
 	if flagVals.UseEKS {
-		giteaValues["service.http.type"] = "LoadBalancer"
+		giteaServiceType = corev1.ServiceTypeLoadBalancer
+	}
+
+	if flagVals.GiteaCustomIngress {
+		giteaServiceType = corev1.ServiceTypeClusterIP
 	}
 
 	giteaResult = testenv.DeployGitea(ctx, testenv.DeployGiteaInput{
@@ -263,6 +269,9 @@ var _ = BeforeSuite(func() {
 		AuthSecretName:        e2e.AuthSecretName,
 		Username:              e2eConfig.GetVariable(e2e.GiteaUserNameVar),
 		Password:              e2eConfig.GetVariable(e2e.GiteaUserPasswordVar),
+		ServiceType:           giteaServiceType,
+		CustomIngressConfig:   e2e.GiteaIngress,
+		Variables:             e2eConfig.Variables,
 	})
 })
 

test/e2e/suites/import-gitops-v3/suite_test.go

@@ -29,14 +29,14 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-	"k8s.io/klog/v2"
-	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
-	ctrl "sigs.k8s.io/controller-runtime"
-
 	"github.com/rancher/turtles/test/e2e"
 	"github.com/rancher/turtles/test/framework"
 	turtlesframework "github.com/rancher/turtles/test/framework"
 	"github.com/rancher/turtles/test/testenv"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
+	ctrl "sigs.k8s.io/controller-runtime"
 )
 
 // Test suite flags.
@@ -237,10 +237,15 @@ var _ = BeforeSuite(func() {
 	giteaValues := map[string]string{
 		"gitea.admin.username": e2eConfig.GetVariable(e2e.GiteaUserNameVar),
 		"gitea.admin.password": e2eConfig.GetVariable(e2e.GiteaUserPasswordVar),
-		"service.http.type":    "NodePort",
 	}
+
+	giteaServiceType := corev1.ServiceTypeNodePort
 	if flagVals.UseEKS {
-		giteaValues["service.http.type"] = "LoadBalancer"
+		giteaServiceType = corev1.ServiceTypeLoadBalancer
+	}
+
+	if flagVals.GiteaCustomIngress {
+		giteaServiceType = corev1.ServiceTypeClusterIP
 	}
 
 	giteaResult = testenv.DeployGitea(ctx, testenv.DeployGiteaInput{
@@ -257,6 +262,9 @@ var _ = BeforeSuite(func() {
 		AuthSecretName:        e2e.AuthSecretName,
 		Username:              e2eConfig.GetVariable(e2e.GiteaUserNameVar),
 		Password:              e2eConfig.GetVariable(e2e.GiteaUserPasswordVar),
+		ServiceType:           giteaServiceType,
+		CustomIngressConfig:   e2e.GiteaIngress,
+		Variables:             e2eConfig.Variables,
 	})
 })
 

test/e2e/suites/import-gitops/suite_test.go

@@ -29,14 +29,14 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-	"k8s.io/klog/v2"
-	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
-	ctrl "sigs.k8s.io/controller-runtime"
-
 	"github.com/rancher/turtles/test/e2e"
 	"github.com/rancher/turtles/test/framework"
 	turtlesframework "github.com/rancher/turtles/test/framework"
 	"github.com/rancher/turtles/test/testenv"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
+	ctrl "sigs.k8s.io/controller-runtime"
 )
 
 // Test suite flags.
@@ -301,10 +301,15 @@ var _ = BeforeSuite(func() {
 	giteaValues := map[string]string{
 		"gitea.admin.username": e2eConfig.GetVariable(e2e.GiteaUserNameVar),
 		"gitea.admin.password": e2eConfig.GetVariable(e2e.GiteaUserPasswordVar),
-		"service.http.type":    "NodePort",
 	}
+
+	giteaServiceType := corev1.ServiceTypeNodePort
 	if flagVals.UseEKS {
-		giteaValues["service.http.type"] = "LoadBalancer"
+		giteaServiceType = corev1.ServiceTypeLoadBalancer
+	}
+
+	if flagVals.GiteaCustomIngress {
+		giteaServiceType = corev1.ServiceTypeClusterIP
 	}
 
 	giteaResult = testenv.DeployGitea(ctx, testenv.DeployGiteaInput{
@@ -321,6 +326,9 @@ var _ = BeforeSuite(func() {
 		AuthSecretName:        e2e.AuthSecretName,
 		Username:              e2eConfig.GetVariable(e2e.GiteaUserNameVar),
 		Password:              e2eConfig.GetVariable(e2e.GiteaUserPasswordVar),
+		ServiceType:           giteaServiceType,
+		CustomIngressConfig:   e2e.GiteaIngress,
+		Variables:             e2eConfig.Variables,
 	})
 })
 

test/e2e/suites/migrate-gitops/suite_test.go

@@ -29,13 +29,13 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-	"k8s.io/klog/v2"
-	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
-	ctrl "sigs.k8s.io/controller-runtime"
-
 	"github.com/rancher/turtles/test/e2e"
 	turtlesframework "github.com/rancher/turtles/test/framework"
 	"github.com/rancher/turtles/test/testenv"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
+	ctrl "sigs.k8s.io/controller-runtime"
 )
 
 // Test suite flags.
@@ -188,7 +188,15 @@ var _ = BeforeSuite(func() {
 	giteaValues := map[string]string{
 		"gitea.admin.username": e2eConfig.GetVariable(e2e.GiteaUserNameVar),
 		"gitea.admin.password": e2eConfig.GetVariable(e2e.GiteaUserPasswordVar),
-		"service.http.type":    "NodePort",
+	}
+
+	giteaServiceType := corev1.ServiceTypeNodePort
+	if flagVals.UseEKS {
+		giteaServiceType = corev1.ServiceTypeLoadBalancer
+	}
+
+	if flagVals.GiteaCustomIngress {
+		giteaServiceType = corev1.ServiceTypeClusterIP
 	}
 
 	giteaResult = testenv.DeployGitea(ctx, testenv.DeployGiteaInput{
@@ -205,6 +213,9 @@ var _ = BeforeSuite(func() {
 		AuthSecretName:        e2e.AuthSecretName,
 		Username:              e2eConfig.GetVariable(e2e.GiteaUserNameVar),
 		Password:              e2eConfig.GetVariable(e2e.GiteaUserPasswordVar),
+		ServiceType:           giteaServiceType,
+		CustomIngressConfig:   e2e.GiteaIngress,
+		Variables:             e2eConfig.Variables,
 	})
 })
 

test/framework/helper.go

@@ -38,7 +38,7 @@ type VariableLookupFunc func(key string) string
 // GetVariable is used to get the value for a variable. The expectation is that the variable exists in one of
 // the sources. Assertion will fail if its not found. The order of precedence when checking for variables is:
 // 1. Environment variables
-// 2. Base variablesß
+// 2. Base variables
 // This is a re-implementation of the CAPI function to add additional logging.
 func GetVariable(vars VariableCollection) VariableLookupFunc {
 	Expect(vars).ToNot(BeNil(), "Variable should not be nil")

test/framework/kube_helper.go

@@ -23,6 +23,7 @@ import (
 	. "github.com/onsi/gomega"
 
 	corev1 "k8s.io/api/core/v1"
+	networkingv1 "k8s.io/api/networking/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/klog/v2"
@@ -233,3 +234,28 @@ func CreateDockerRegistrySecret(ctx context.Context, input CreateDockerRegistryS
 	Expect(cmdCreateSecret.Error).NotTo(HaveOccurred(), "Failed creating docker registry k8s secret")
 	Expect(cmdCreateSecret.ExitCode).To(Equal(0), "Creating secret return non-zero exit code")
 }
+
+// GetIngressHostInput is the input to GetIngressHost.
+type GetIngressHostInput struct {
+	GetLister        framework.GetLister
+	IngressName      string
+	IngressNamespace string
+	IngressRuleIndex int
+}
+
+// GetIngressHost gets the host from an ingress object.
+func GetIngressHost(ctx context.Context, input GetIngressHostInput) string {
+	Expect(ctx).NotTo(BeNil(), "ctx is required for GetNodeAddress")
+	Expect(input.GetLister).ToNot(BeNil(), "Invalid argument. input.GetLister can't be nil when calling GetIngressHost")
+
+	ingress := &networkingv1.Ingress{}
+	Eventually(func() error {
+		return input.GetLister.Get(ctx, client.ObjectKey{Namespace: input.IngressNamespace, Name: input.IngressName}, ingress)
+	}).Should(Succeed(), "Failed to get ingress")
+
+	Expect(ingress.Spec.Rules).NotTo(HaveLen(0), "Expected ingress to have at least 1 rule")
+	Expect(len(ingress.Spec.Rules) >= input.IngressRuleIndex).To(BeTrue(), "Ingress rule index is greater than number of rules")
+
+	rule := ingress.Spec.Rules[input.IngressRuleIndex]
+	return rule.Host
+}

test/testenv/gitea.go

@@ -21,6 +21,7 @@ import (
 	"fmt"
 	"net/http"
 
+	"github.com/drone/envsubst/v2"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 
@@ -47,6 +48,9 @@ type DeployGiteaInput struct {
 	Username              string
 	Password              string
 	AuthSecretName        string
+	CustomIngressConfig   []byte
+	ServiceType           corev1.ServiceType
+	Variables             turtlesframework.VariableCollection
 }
 
 type DeployGiteaResult struct {
@@ -63,12 +67,21 @@ func DeployGitea(ctx context.Context, input DeployGiteaInput) *DeployGiteaResult
 	Expect(input.ChartVersion).ToNot(BeEmpty(), "Chartversion is required for DeployGitea")
 	Expect(input.RolloutWaitInterval).ToNot(BeNil(), "RolloutWaitInterval is required for DeployGitea")
 	Expect(input.ServiceWaitInterval).ToNot(BeNil(), "ServiceWaitInterval is required for DeployGitea")
+	Expect(input.ServiceType).ToNot(BeEmpty(), "ServiceType is required for DeployGitea")
 
 	if input.Username != "" {
 		Expect(input.Password).ToNot(BeEmpty(), "Password is required for DeployGitea if a username is supplied")
 		Expect(input.AuthSecretName).ToNot(BeEmpty(), "AuthSecretName is required for DeployGitea if a username is supplied")
 	}
 
+	if input.ServiceType == corev1.ServiceTypeClusterIP {
+		Expect(input.CustomIngressConfig).ToNot(BeEmpty(), "CustomIngressConfig is required for DeployGitea if service type is ClusterIP")
+	}
+
+	if input.Values["service.http.type"] == "" {
+		input.Values["service.http.type"] = string(input.ServiceType)
+	}
+
 	result := &DeployGiteaResult{}
 
 	By("Installing gitea chart")
@@ -126,7 +139,8 @@ func DeployGitea(ctx context.Context, input DeployGiteaInput) *DeployGiteaResult
 	}, input.ServiceWaitInterval...)
 	Expect(port.NodePort).ToNot(Equal(0), "Node port for Gitea service is not set")
 
-	if input.Values["service.http.type"] == "NodePort" {
+	switch input.Values["service.http.type"] {
+	case string(corev1.ServiceTypeNodePort):
 		By("Get Git server node port")
 		addr := turtlesframework.GetNodeAddress(ctx, turtlesframework.GetNodeAddressInput{
 			Lister:       input.BootstrapClusterProxy.GetClient(),
@@ -135,7 +149,7 @@ func DeployGitea(ctx context.Context, input DeployGiteaInput) *DeployGiteaResult
 		})
 
 		result.GitAddress = fmt.Sprintf("http://%s:%d", addr, port.NodePort)
-	} else {
+	case string(corev1.ServiceTypeLoadBalancer):
 		By("Getting git server ingress address")
 		svcRes := &WaitForServiceIngressHostnameResult{}
 		WaitForServiceIngressHostname(ctx, WaitForServiceIngressHostnameInput{
@@ -145,6 +159,22 @@ func DeployGitea(ctx context.Context, input DeployGiteaInput) *DeployGiteaResult
 			IngressWaitInterval:   input.ServiceWaitInterval,
 		}, svcRes)
 		result.GitAddress = fmt.Sprintf("http://%s:%d", svcRes.Hostname, port.Port)
+	case string(corev1.ServiceTypeClusterIP):
+		By("Creating custom ingress for gitea")
+		variableGetter := turtlesframework.GetVariable(input.Variables)
+		ingress, err := envsubst.Eval(string(input.CustomIngressConfig), variableGetter)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(input.BootstrapClusterProxy.Apply(ctx, []byte(ingress))).To(Succeed())
+
+		By("Getting git server ingress address")
+		host := turtlesframework.GetIngressHost(ctx, turtlesframework.GetIngressHostInput{
+			GetLister:        input.BootstrapClusterProxy.GetClient(),
+			IngressRuleIndex: 0,
+			IngressName:      "gitea-http",
+			IngressNamespace: "default",
+		})
+
+		result.GitAddress = fmt.Sprintf("https://%s", host)
 	}
 
 	if input.Username == "" {