Dedicated job to debug TestMutationSecondMasterSetDown (#2849)

Run the E2E test `TestMutationSecondMasterSetDown` with
`--skip-cleanup` enabled to prevent to run cleanup actions in the k8s
cluster every 30 minutes.
The post action to delete the k8s cluster at the end of the job is run
only if there is no failed tests.

.ci/jobs/e2e-tests-testmutationsecondmastersetdown.yml

@@ -0,0 +1,17 @@
+---
+- job:
+    description: Run a particular ECK E2E tests on GKE
+    name: cloud-on-k8s-e2e-tests-testmutationsecondmastersetdown
+    project-type: pipeline
+    triggers:
+      - timed: 'H/30 * * * *'
+    concurrent: true
+    pipeline-scm:
+      scm:
+        - git:
+            url: https://github.com/elastic/cloud-on-k8s
+            branches:
+              - master
+            credentials-id: 'f6c7695a-671e-4f4f-a331-acdce44ff9ba'
+      script-path: .ci/pipelines/e2e-tests-testmutationsecondmastersetdown.Jenkinsfile
+      lightweight-checkout: true

.ci/pipelines/e2e-tests-testmutationsecondmastersetdown.Jenkinsfile

@@ -0,0 +1,112 @@
+// This library overrides the default checkout behavior to enable sleep+retries if there are errors
+// Added to help overcome some recurring github connection issues
+@Library('apm@current') _
+
+def failedTests = []
+def lib
+
+pipeline {
+
+    agent {
+        label 'linux'
+    }
+
+    options {
+        timeout(time: 300, unit: 'MINUTES')
+    }
+
+    environment {
+        VAULT_ADDR = credentials('vault-addr')
+        VAULT_ROLE_ID = credentials('vault-role-id')
+        VAULT_SECRET_ID = credentials('vault-secret-id')
+        GCLOUD_PROJECT = credentials('k8s-operators-gcloud-project')
+    }
+
+    stages {
+        stage('Checkout from GitHub') {
+            steps {
+                checkout scm
+            }
+        }
+        stage('Load common scripts') {
+            steps {
+                script {
+                    lib = load ".ci/common/tests.groovy"
+                }
+            }
+        }
+       /* stage('Run Checks') {
+            when {
+                expression {
+                    notOnlyDocs()
+                }
+            }
+            steps {
+                sh 'make -C .ci TARGET=ci-check ci'
+            }
+        }*/
+        stage("E2E tests") {
+            when {
+                expression {
+                    notOnlyDocs()
+                }
+            }
+            steps {
+                sh '.ci/setenvconfig e2e/master'
+
+                sh 'echo TESTS_MATCH = TestMutationSecondMasterSetDown >> .env'
+                sh 'echo E2E_SKIP_CLEANUP = true >> .env'
+                sh 'sed "s/CLUSTER_NAME=eck-e2e/CLUSTER_NAME=eck-debug-e2e/" -i .env'
+
+                script {
+                    env.SHELL_EXIT_CODE = sh(returnStatus: true, script: 'make -C .ci get-test-artifacts TARGET=ci-build-operator-e2e-run ci')
+
+                    sh 'make -C .ci TARGET=e2e-generate-xml ci'
+                    junit "e2e-tests.xml"
+
+                    if (env.SHELL_EXIT_CODE != 0) {
+                        failedTests = lib.getListOfFailedTests()
+                    }
+
+                    sh 'exit $SHELL_EXIT_CODE'
+                }
+            }
+        }
+    }
+
+    post {
+        unsuccessful {
+            script {
+                def msg = lib.generateSlackMessage("E2E tests failed!", env.BUILD_URL, failedTests)
+
+                slackSend(
+                      channel: '#cloud-k8s',
+                      color: 'danger',
+                      message: msg,
+                    tokenCredentialId: 'cloud-ci-slack-integration-token',
+                    botUser: true,
+                    failOnError: true
+                )
+            }
+        }
+        cleanup {
+            script {
+                if (notOnlyDocs() && failedTests.size == 0) {
+                    build job: 'cloud-on-k8s-e2e-cleanup',
+                        parameters: [string(name: 'JKS_PARAM_GKE_CLUSTER', value: "eck-e2e-${BUILD_NUMBER}")],
+                        wait: false
+                }
+            }
+
+            cleanWs()
+        }
+    }
+}
+
+def notOnlyDocs() {
+    // grep succeeds if there is at least one line without docs/
+    return sh (
+        script: "git diff --name-status HEAD~1 HEAD | grep -v docs/",
+        returnStatus: true
+    ) == 0
+}

Makefile

@@ -363,6 +363,7 @@ E2E_IMG ?= $(BASE_IMG)-e2e-tests:$(TAG)
 STACK_VERSION ?= 7.6.0
 E2E_JSON ?= false
 TEST_TIMEOUT ?= 5m
+E2E_SKIP_CLEANUP ?= false
 
 # clean to remove irrelevant/build-breaking generated public keys
 e2e-docker-build: clean
@@ -396,7 +397,8 @@ e2e-run:
 		--provider=$(E2E_PROVIDER) \
 		--clusterName=$(CLUSTER_NAME) \
 		--kubernetes-version=$(KUBERNETES_VERSION) \
-		--monitoring-secrets=$(MONITORING_SECRETS)
+		--monitoring-secrets=$(MONITORING_SECRETS) \
+		--skip-cleanup=$(E2E_SKIP_CLEANUP)
 
 e2e-generate-xml:
 	@ gotestsum --junitfile e2e-tests.xml --raw-command cat e2e-tests.json