Dedicated job to debug TestMutationSecondMasterSetDown

Run in a loop the E2E test `TestMutationSecondMasterSetDown` with
`--skip-cleanup` enabled to prevent to run cleanup actions in the k8s
cluster.
The post action to delete the k8s cluster at the end of the job is also
removed.

.ci/jobs/e2e-tests-testmutationsecondmastersetdown.yml

@@ -0,0 +1,17 @@
+---
+- job:
+    description: Run ECK E2E tests on GKE, on every merge in master
+    name: cloud-on-k8s-e2e-tests-testmutationsecondmastersetdown
+    project-type: pipeline
+    triggers:
+      - timed: '@daily'
+    concurrent: true
+    pipeline-scm:
+      scm:
+        - git:
+            url: https://github.com/elastic/cloud-on-k8s
+            branches:
+              - master
+            credentials-id: 'f6c7695a-671e-4f4f-a331-acdce44ff9ba'
+      script-path: .ci/pipelines/e2e-tests-testmutationsecondmastersetdown.Jenkinsfile
+      lightweight-checkout: true

.ci/pipelines/e2e-tests-testmutationsecondmastersetdown.Jenkinsfile

@@ -0,0 +1,46 @@
+// This library overrides the default checkout behavior to enable sleep+retries if there are errors
+// Added to help overcome some recurring github connection issues
+@Library('apm@current') _
+
+pipeline {
+
+    agent {
+        label 'linux'
+    }
+
+    options {
+        timeout(time: 24, unit: 'HOURS')
+    }
+
+    environment {
+        VAULT_ADDR = credentials('vault-addr')
+        VAULT_ROLE_ID = credentials('vault-role-id')
+        VAULT_SECRET_ID = credentials('vault-secret-id')
+        GCLOUD_PROJECT = credentials('k8s-operators-gcloud-project')
+    }
+
+    stages {
+        stage('Checkout from GitHub') {
+            steps {
+                checkout scm
+            }
+        }
+        stage("E2E tests") {
+            steps {
+                sh '.ci/setenvconfig e2e/master'
+
+                sh 'echo TESTS_MATCH = TestMutationSecondMasterSetDown >> .env'
+                sh 'echo E2E_SKIP_CLEANUP = true >> .env'
+
+                sh 'make -C .ci get-test-artifacts TARGET=ci-build-operator-e2e-run-indefinitely ci'
+            }
+        }
+    }
+
+    post {
+        cleanup {
+            cleanWs()
+        }
+        // K8s cluster will not be cleanup.
+    }
+}

Makefile

@@ -363,6 +363,7 @@ E2E_IMG ?= $(BASE_IMG)-e2e-tests:$(TAG)
 STACK_VERSION ?= 7.6.0
 E2E_JSON ?= false
 TEST_TIMEOUT ?= 5m
+E2E_SKIP_CLEANUP ?= false
 
 # clean to remove irrelevant/build-breaking generated public keys
 e2e-docker-build: clean
@@ -396,7 +397,8 @@ e2e-run:
 		--provider=$(E2E_PROVIDER) \
 		--clusterName=$(CLUSTER_NAME) \
 		--kubernetes-version=$(KUBERNETES_VERSION) \
-		--monitoring-secrets=$(MONITORING_SECRETS)
+		--monitoring-secrets=$(MONITORING_SECRETS) \
+		--skip-cleanup=$(E2E_SKIP_CLEANUP)
 
 e2e-generate-xml:
 	@ gotestsum --junitfile e2e-tests.xml --raw-command cat e2e-tests.json
@@ -438,6 +440,10 @@ ci-e2e: setup-e2e e2e-run
 ci-build-operator-e2e-run: E2E_JSON := true
 ci-build-operator-e2e-run: setup-e2e build-operator-image e2e-run
 
+ci-build-operator-e2e-run-indefinitely: E2E_JSON := true
+ci-build-operator-e2e-run-indefinitely: setup-e2e build-operator-image
+	while true; do echo make e2e-run; done
+
 run-deployer: build-deployer
 	./hack/deployer/deployer execute --plans-file hack/deployer/config/plans.yml --config-file deployer-config.yml