From 2554f4aeae8c9501f05b3f160c15645111ef0877 Mon Sep 17 00:00:00 2001
From: Yecheng Fu <fuyecheng@pingcap.com>
Date: Wed, 19 Feb 2020 20:52:16 +0800
Subject: [PATCH 01/13] support eks provider in e2e

---
 hack/e2e.sh                 | 117 +++++++++++++++++++++++++-----------
 hack/lib.sh                 |  32 ++++++++--
 hack/run-e2e.sh             |  66 +++++++++++++++++++-
 tests/images/e2e/Dockerfile |   8 ++-
 4 files changed, 182 insertions(+), 41 deletions(-)

diff --git a/hack/e2e.sh b/hack/e2e.sh
index 7411a8133e..b30cf0ff60 100755
--- a/hack/e2e.sh
+++ b/hack/e2e.sh
@@ -46,31 +46,34 @@ Usage: hack/e2e.sh [-h] -- [extra test args]
 
 Environments:
 
-    PROVIDER            Kubernetes provider, e.g. kind, gke, defaults: kind
-    DOCKER_REGISTRY     image docker registry
-    IMAGE_TAG           image tag
-    CLUSTER             the name of e2e cluster, defaults: tidb-operator
-    KUBECONFIG          path to the kubeconfig file, defaults: ~/.kube/config
-    SKIP_BUILD          skip building binaries
-    SKIP_IMAGE_BUILD    skip build and push images
-    SKIP_UP             skip starting the cluster
-    SKIP_DOWN           skip shutting down the cluster
-    SKIP_TEST           skip running the test
-    KUBE_VERSION        the version of Kubernetes to test against
-    KUBE_WORKERS        the number of worker nodes (excludes master nodes), defaults: 3
-    DOCKER_IO_MIRROR    configure mirror for docker.io
-    GCR_IO_MIRROR       configure mirror for gcr.io
-    QUAY_IO_MIRROR      configure mirror for quay.io
-    KIND_DATA_HOSTPATH  (kind only) the host path of data directory for kind cluster, defaults: none
-    GCP_PROJECT         (gke only) the GCP project to run in
-    GCP_SERVICE_ACCOUNT (gke only) the GCP service account to use
-    GCP_REGION          (gke only) the GCP region, if specified a regional cluster is creaetd
-    GCP_ZONE            (gke only) the GCP zone, if specified a zonal cluster is created
-    GCP_SSH_PRIVATE_KEY (gke only) the path to the private ssh key
-    GCP_SSH_PUBLIC_KEY  (gke only) the path to the public ssh key
-    GINKGO_NODES        ginkgo nodes to run specs, defaults: 1
-    GINKGO_PARALLEL     if set to `y`, will run specs in parallel, the number of nodes will be the number of cpus
-    GINKGO_NO_COLOR     if set to `y`, suppress color output in default reporter
+    PROVIDER              Kubernetes provider, e.g. kind, gke, eks, defaults: kind
+    DOCKER_REGISTRY       image docker registry
+    IMAGE_TAG             image tag
+    CLUSTER               the name of e2e cluster, defaults: tidb-operator
+    KUBECONFIG            path to the kubeconfig file, defaults: ~/.kube/config
+    SKIP_BUILD            skip building binaries
+    SKIP_IMAGE_BUILD      skip build and push images
+    SKIP_UP               skip starting the cluster
+    SKIP_DOWN             skip shutting down the cluster
+    SKIP_TEST             skip running the test
+    KUBE_VERSION          the version of Kubernetes to test against
+    KUBE_WORKERS          the number of worker nodes (excludes master nodes), defaults: 3
+    DOCKER_IO_MIRROR      configure mirror for docker.io
+    GCR_IO_MIRROR         configure mirror for gcr.io
+    QUAY_IO_MIRROR        configure mirror for quay.io
+    KIND_DATA_HOSTPATH    (kind only) the host path of data directory for kind cluster, defaults: none
+    GCP_PROJECT           (gke only) the GCP project to run in
+    GCP_SERVICE_ACCOUNT   (gke only) the GCP service account to use
+    GCP_REGION            (gke only) the GCP region, if specified a regional cluster is creaetd
+    GCP_ZONE              (gke only) the GCP zone, if specified a zonal cluster is created
+    GCP_SSH_PRIVATE_KEY   (gke only) the path to the private ssh key
+    GCP_SSH_PUBLIC_KEY    (gke only) the path to the public ssh key
+    AWS_ACCESS_KEY_ID     (eks only) the aws access key id
+    AWS_SECRET_ACCESS_KEY (eks only) the aws secret access key
+    AWS_REGION            (eks only) the aws region
+    GINKGO_NODES          ginkgo nodes to run specs, defaults: 1
+    GINKGO_PARALLEL       if set to `y`, will run specs in parallel, the number of nodes will be the number of cpus
+    GINKGO_NO_COLOR       if set to `y`, suppress color output in default reporter
 
 Examples:
 
@@ -103,10 +106,7 @@ Examples:
 
 5) run e2e with gke provider locally
 
-    You need install Google Cloud SDK first, then prepare GCP servie account
-    and configure ssh key pairs
-
-    GCP service account must be created with following permissions:
+    You need prepare GCP service account with the following permissions:
 
         - Compute Network Admin
         - Kubernetes Engine Admin
@@ -121,11 +121,32 @@ Examples:
 
     Then run with following additional GCP-specific environments:
 
-    export GCP_PROJECT=<project>
-    export GCP_SERVICE_ACCOUNT=<path-to-gcp-service-account>
-    export GCP_ZONE=us-central1-b
+        export GCP_PROJECT=<project>
+        export GCP_SERVICE_ACCOUNT=<path-to-gcp-service-account>
+        export GCP_ZONE=us-central1-b
+
+        PROVIDER=gke ./hack/e2e.sh -- <e2e args>
+
+    If you run the outside of the dev containter started by
+    ./hack/run-in-container.sh, Google Cloud SDK must be installed on you
+    machine.
+
+6) run e2e with eks provider locally
+
+    You need configure your aws credential and region or set it via following
+    environments:
+
+        export AWS_ACCESS_KEY_ID=<your-aws-access-key-id>
+        export AWS_SECRET_ACCESS_KEY=<your-aws-secret-key-id>
+        export AWS_REGION=<your-aws-region>
+
+    then run e2e with eks provider:
 
-    ./hack/e2e.sh -- <e2e args>
+        PROVIDER=eks ./hack/e2e.sh -- <e2e args>
+
+    If you run the outside of the dev containter started by
+    ./hack/run-in-container.sh, AWS CLI must be installed on you
+    machine.
 
 EOF
 
@@ -166,6 +187,9 @@ GCP_REGION=${GCP_REGION:-}
 GCP_ZONE=${GCP_ZONE:-}
 GCP_SSH_PRIVATE_KEY=${GCP_SSH_PRIVATE_KEY:-}
 GCP_SSH_PUBLIC_KEY=${GCP_SSH_PUBLIC_KEY:-}
+AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-}
+AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-}
+AWS_REGION=${AWS_REGION:-}
 KUBE_VERSION=${KUBE_VERSION:-v1.12.10}
 KUBE_WORKERS=${KUBE_WORKERS:-3}
 DOCKER_IO_MIRROR=${DOCKER_IO_MIRROR:-}
@@ -186,6 +210,9 @@ echo "GCP_PROJECT: $GCP_PROJECT"
 echo "GCP_SERVICE_ACCOUNT: $GCP_SERVICE_ACCOUNT"
 echo "GCP_REGION: $GCP_REGION"
 echo "GCP_ZONE: $GCP_ZONE"
+echo "AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID"
+echo "AWS_SECRET_ACCESS_KEY: $AWS_SECRET_ACCESS_KEY"
+echo "AWS_REGION: $AWS_REGION"
 echo "KUBE_VERSION: $KUBE_VERSION"
 echo "KUBE_WORKERS: $KUBE_WORKERS"
 echo "DOCKER_IO_MIRROR: $DOCKER_IO_MIRROR"
@@ -384,7 +411,7 @@ elif [ "$PROVIDER" == "gke" ]; then
         echo "error: GCP_REGION or GCP_ZONE cannot be both set"
         exit 1
     fi
-	echo "info: preparing ssh keypairs for GCP"
+    echo "info: preparing ssh keypairs for GCP"
     if [ ! -d ~/.ssh ]; then
         mkdir ~/.ssh
     fi
@@ -414,6 +441,23 @@ elif [ "$PROVIDER" == "gke" ]; then
             --zone "$GCP_ZONE"
         )
     fi
+elif [ "$PROVIDER" == "eks" ]; then
+    hack::ensure_aws_k8s_tester
+    if [ -n "$AWS_REGION" ]; then
+        aws configure set default.region "$AWS_REGION"
+    fi
+    if [ -n "$AWS_ACCESS_KEY_ID" ]; then
+        aws configure set aws_access_key_id "$AWS_ACCESS_KEY_ID"
+    fi
+    if [ -n "$AWS_SECRET_ACCESS_KEY" ]; then
+        aws configure set aws_secret_access_key "$AWS_SECRET_ACCESS_KEY"
+    fi
+    export AWS_K8S_TESTER_EKS_NAME=$CLUSTER
+    export AWS_K8S_TESTER_EKS_CONFIG_PATH=/tmp/kubetest2.eks.$CLUSTER
+    export AWS_K8S_TESTER_EKS_ADD_ON_NLB_HELLO_WORLD_ENABLE="false"
+    export AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_MNGS='{"aws-k8s-tester-tidb-operator-mng":{"name":"aws-k8s-tester-tidb-operator-mng","ami-type":"AL2_x86_64","asg-min-size":3,"asg-max-size":3,"asg-desired-capacity":3,"instance-types":["c5.xlarge"],"volume-size":40}}'
+    # override KUBECONFIG
+    KUBECONFIG=$AWS_K8S_TESTER_EKS_CONFIG_PATH.kubeconfig.yaml
 else
     echo "error: unsupported provider '$PROVIDER'"
     exit 1
@@ -429,6 +473,11 @@ export TIDB_OPERATOR_IMAGE=$DOCKER_REGISTRY/pingcap/tidb-operator:${IMAGE_TAG}
 export E2E_IMAGE=$DOCKER_REGISTRY/pingcap/tidb-operator-e2e:${IMAGE_TAG}
 export PATH=$PATH:$OUTPUT_BIN
 
+# Environments for kubetest2
+if [ -n "${REPORT_DIR:-}" ]; then
+    export ARTIFACTS=${REPORT_DIR:-}
+fi
+
 hack::ensure_kubetest2
 echo "info: run 'kubetest2 ${kubetest2_args[@]} -- hack/run-e2e.sh $@'"
 $KUBETSTS2_BIN ${kubetest2_args[@]} -- hack/run-e2e.sh "$@"
diff --git a/hack/lib.sh b/hack/lib.sh
index d753775a94..c8e3f5eb58 100644
--- a/hack/lib.sh
+++ b/hack/lib.sh
@@ -34,10 +34,13 @@ HELM_BIN=$OUTPUT_BIN/helm
 HELM_VERSION=${HELM_VERSION:-2.9.1}
 KIND_VERSION=${KIND_VERSION:-0.7.0}
 KIND_BIN=$OUTPUT_BIN/kind
-KUBETEST2_VERSION=v0.0.1+a810685993a3e100f4c51bc346cdc05eaf753922
-KUBETEST2_GKE_VERSION=v0.0.1+a3755779de7f745733de10f9bf63e01cf0864f9d
-KUBETEST2_KIND_VERSION=v0.0.1+d8d70a33d2cc5df85786b7724ac61c221bad3e18
+KUBETEST2_VERSION=v0.0.2+8e0a95176a86e6bdbc0b5aa18c40fa5542828d15
+KUBETEST2_GKE_VERSION=v0.0.2+5b16ede1983db0dfc384145f4c559db15f80c14f
+KUBETEST2_EKS_VERSION=v0.0.2+ddbcc8482c65d8f230511a73b1188791af96ed6b
+KUBETEST2_KIND_VERSION=v0.0.2+a81b87d6155611e5561c9642d25a14d4911fb8a7
 KUBETSTS2_BIN=$OUTPUT_BIN/kubetest2
+AWS_K8S_TESTER_VERSION=v0.6.2
+AWS_K8S_TESTER_BIN=$OUTPUT_BIN/aws-k8s-tester
 
 test -d "$OUTPUT_BIN" || mkdir -p "$OUTPUT_BIN"
 
@@ -168,7 +171,7 @@ function hack::__ensure_kubetest2() {
     if hack::__verify_kubetest2 $n $h; then
         return 0
     fi
-    tmpfile=$(mktemp)
+    local tmpfile=$(mktemp)
     trap "test -f $tmpfile && rm $tmpfile" RETURN
     curl --retry 10 -L -o - https://github.com/cofyc/kubetest2/releases/download/$v/$n.gz | gunzip > $tmpfile
     mv $tmpfile $OUTPUT_BIN/$n
@@ -179,4 +182,25 @@ function hack::ensure_kubetest2() {
     hack::__ensure_kubetest2 kubetest2 $KUBETEST2_VERSION
     hack::__ensure_kubetest2 kubetest2-gke $KUBETEST2_GKE_VERSION
     hack::__ensure_kubetest2 kubetest2-kind $KUBETEST2_KIND_VERSION
+    hack::__ensure_kubetest2 kubetest2-eks $KUBETEST2_EKS_VERSION
+}
+
+function hack::verify_aws_k8s_tester() {
+    if test -x $AWS_K8S_TESTER_BIN; then
+        [[ "$($AWS_K8S_TESTER_BIN version | awk '/ReleaseVersion/ {print $2}')" == "$AWS_K8S_TESTER_VERSION" ]]
+        return
+    fi
+    return 1
+}
+
+function hack::ensure_aws_k8s_tester() {
+    if hack::verify_aws_k8s_tester; then
+        return
+    fi
+	local DOWNLOAD_URL=https://github.com/aws/aws-k8s-tester/releases/download
+    local tmpfile=$(mktemp)
+    trap "test -f $tmpfile && rm $tmpfile" RETURN
+    curl --retry 10 -L -o $tmpfile https://github.com/aws/aws-k8s-tester/releases/download/$AWS_K8S_TESTER_VERSION/aws-k8s-tester-$AWS_K8S_TESTER_VERSION-$OS-$ARCH
+	mv $tmpfile $AWS_K8S_TESTER_BIN
+	chmod +x $AWS_K8S_TESTER_BIN
 }
diff --git a/hack/run-e2e.sh b/hack/run-e2e.sh
index c4b056a77e..fee9cb3dfd 100755
--- a/hack/run-e2e.sh
+++ b/hack/run-e2e.sh
@@ -114,7 +114,7 @@ for ((i = 1; i <= 32; i++)) {
 }
 EOF
         done
-    else
+    elif [ "$PROVIDER" == "gke" ]; then
         # disks are created under /mnt/stateful_partition directory
         # https://cloud.google.com/container-optimized-os/docs/concepts/disks-and-filesystem
         for n in $($KUBECTL_BIN --context $KUBECONTEXT get nodes -ojsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
@@ -136,12 +136,47 @@ for ((i = 1; i <= 32; i++)) {
 }
 '"'"
         done
+    elif [ "$PROVIDER" == "eks" ]; then
+        while IFS=$'\n' read -r line; do
+            read -r id dns <<< $line
+            echo "info: prepare disks on $dns"
+            ssh -T -o "StrictHostKeyChecking no" -i ~/.ssh/kube_aws_rsa ec2-user@$dns <<'EOF'
+sudo bash -c '
+test -d /mnt/disks || mkdir -p /mnt/disks
+df -h /mnt/disks
+if mountpoint /mnt/disks &>/dev/null; then
+    echo "info: /mnt/disks is a mountpoint"
+else
+    echo "info: /mnt/disks is not a mountpoint, creating local volumes on the rootfs"
+fi
+cd /mnt/disks
+for ((i = 1; i <= 32; i++)) {
+    if [ ! -d vol$i ]; then
+        mkdir vol$i
+    fi
+    if ! mountpoint vol$i &>/dev/null; then
+        mount --bind vol$i vol$i
+    fi
+}
+'
+EOF
+        done <<< "$(e2e::__eks_instances)"
     fi
     echo "info: installing local-volume-provisioner"
     $KUBECTL_BIN --context $KUBECONTEXT apply -f ${ROOT}/manifests/local-dind/local-volume-provisioner.yaml
     e2e::__wait_for_ds kube-system local-volume-provisioner
 }
 
+function e2e::__eks_instances() {
+    aws ec2 describe-instances --filter Name=tag:eks:cluster-name,Values=$CLUSTER --query 'Reservations[*].Instances[*].{InstanceId:InstanceId,PublicDnsName:PublicDnsName}' --output text
+}
+
+function e2e::__ecr_url() {
+    local account_id=$(aws sts get-caller-identity | awk '/Account/ { gsub("\x27", "", $2); print $2}')
+    local region=$(aws configure get region)
+    echo "${account_id}.dkr.ecr.${region}.amazonaws.com"
+}
+
 function e2e::get_kube_version() {
     $KUBECTL_BIN --context $KUBECONTEXT version --short | awk '/Server Version:/ {print $3}'
 }
@@ -186,6 +221,28 @@ function e2e::image_load() {
         docker push $GCP_E2E_IMAGE
         TIDB_OPERATOR_IMAGE=$GCP_TIDB_OPERATOR_IMAGE
         E2E_IMAGE=$GCP_E2E_IMAGE
+    elif [ "$PROVIDER" == "eks" ]; then
+        for repoName in e2e/tidb-operator e2e/tidb-operator-e2e; do
+            local ret=0
+            aws ecr describe-repositories --repository-names $repoName || ret=$?
+            if [ $ret -ne 0 ]; then
+                echo "info: creating repository $repoName"
+                aws ecr create-repository --repository-name $repoName
+            fi
+        done
+        local ecrURL=$(e2e::__ecr_url)
+        echo "info: logging in $ecrURL"
+        aws ecr get-login-password | docker login --username AWS --password-stdin $ecrURL
+        AWS_TIDB_OPERATOR_IMAGE=$ecrURL/e2e/tidb-operator:$IMAGE_TAG
+        AWS_E2E_IMAGE=$ecrURL/e2e/tidb-operator-e2e:$IMAGE_TAG
+        docker tag $TIDB_OPERATOR_IMAGE $AWS_TIDB_OPERATOR_IMAGE
+        docker tag $E2E_IMAGE $AWS_E2E_IMAGE
+        echo "info: pushing $AWS_TIDB_OPERATOR_IMAGE"
+        docker push $AWS_TIDB_OPERATOR_IMAGE
+        echo "info: pushing $AWS_E2E_IMAGE"
+        docker push $AWS_E2E_IMAGE
+        TIDB_OPERATOR_IMAGE=$AWS_TIDB_OPERATOR_IMAGE
+        E2E_IMAGE=$AWS_E2E_IMAGE
     else
         echo "info: unsupported provider '$PROVIDER', skip loading images"
     fi
@@ -262,6 +319,13 @@ docker_args=(
     --env KUBECONTEXT=$KUBECONTEXT
 )
 
+if [ "$PROVIDER" == "eks" ]; then
+    # aws credential is required to get token for EKS
+    docker_args+=(
+        -v $HOME/.aws:/root/.aws
+    )
+fi
+
 if [ -n "$REPORT_DIR" ]; then
     docker_args+=(
         -v $REPORT_DIR:$REPORT_DIR
diff --git a/tests/images/e2e/Dockerfile b/tests/images/e2e/Dockerfile
index 260ec91cf9..ccce271038 100644
--- a/tests/images/e2e/Dockerfile
+++ b/tests/images/e2e/Dockerfile
@@ -1,9 +1,10 @@
-FROM alpine:3.10
+FROM debian:buster-slim
 
 ENV KUBECTL_VERSION=v1.12.2
 ENV HELM_VERSION=v2.9.1
 
-RUN apk update && apk add --no-cache ca-certificates curl git openssl bash mysql-client
+RUN apt-get update && \
+    apt-get install -y ca-certificates curl git openssl default-mysql-client unzip
 RUN curl https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl \
     -o /usr/local/bin/kubectl && \
     chmod +x /usr/local/bin/kubectl && \
@@ -13,6 +14,9 @@ RUN curl https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VER
     mv linux-amd64/helm /usr/local/bin/helm && \
     rm -rf linux-amd64 && \
     rm helm-${HELM_VERSION}-linux-amd64.tar.gz
+RUN curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" && \
+    unzip awscliv2.zip && \
+    ./aws/install
 
 ADD tidb-operator /charts/e2e/tidb-operator
 ADD tidb-cluster /charts/e2e/tidb-cluster

From cb1183e108723731e408be9738c2d35ec2d4d72e Mon Sep 17 00:00:00 2001
From: Yecheng Fu <fuyecheng@pingcap.com>
Date: Thu, 20 Feb 2020 14:44:39 +0800
Subject: [PATCH 02/13] upgrade to use kubetest v0.0.3

---
 hack/e2e.sh | 4 ++++
 hack/lib.sh | 8 ++++----
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/hack/e2e.sh b/hack/e2e.sh
index b30cf0ff60..c922450e33 100755
--- a/hack/e2e.sh
+++ b/hack/e2e.sh
@@ -458,6 +458,10 @@ elif [ "$PROVIDER" == "eks" ]; then
     export AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_MNGS='{"aws-k8s-tester-tidb-operator-mng":{"name":"aws-k8s-tester-tidb-operator-mng","ami-type":"AL2_x86_64","asg-min-size":3,"asg-max-size":3,"asg-desired-capacity":3,"instance-types":["c5.xlarge"],"volume-size":40}}'
     # override KUBECONFIG
     KUBECONFIG=$AWS_K8S_TESTER_EKS_CONFIG_PATH.kubeconfig.yaml
+    # clear previous created private key to work around permission issue on this file
+    if test -f $HOME/.ssh/kube_aws_rsa; then
+        rm -f $HOME/.ssh/kube_aws_rsa
+    fi
 else
     echo "error: unsupported provider '$PROVIDER'"
     exit 1
diff --git a/hack/lib.sh b/hack/lib.sh
index c8e3f5eb58..fad41694b3 100644
--- a/hack/lib.sh
+++ b/hack/lib.sh
@@ -34,10 +34,10 @@ HELM_BIN=$OUTPUT_BIN/helm
 HELM_VERSION=${HELM_VERSION:-2.9.1}
 KIND_VERSION=${KIND_VERSION:-0.7.0}
 KIND_BIN=$OUTPUT_BIN/kind
-KUBETEST2_VERSION=v0.0.2+8e0a95176a86e6bdbc0b5aa18c40fa5542828d15
-KUBETEST2_GKE_VERSION=v0.0.2+5b16ede1983db0dfc384145f4c559db15f80c14f
-KUBETEST2_EKS_VERSION=v0.0.2+ddbcc8482c65d8f230511a73b1188791af96ed6b
-KUBETEST2_KIND_VERSION=v0.0.2+a81b87d6155611e5561c9642d25a14d4911fb8a7
+KUBETEST2_VERSION=v0.0.3+c3df273d31e01450e53d23b7482ae5a834a0a682
+KUBETEST2_EKS_VERSION=v0.0.3+63a212b4833fa8528253506f1b05e2ceccbc7b88
+KUBETEST2_GKE_VERSION=v0.0.3+2ed37632a7070fddeb93955dadc8984462ee6a5d
+KUBETEST2_KIND_VERSION=v0.0.3+f22b63a132e0c0f144940558c58cf87159cfad0e
 KUBETSTS2_BIN=$OUTPUT_BIN/kubetest2
 AWS_K8S_TESTER_VERSION=v0.6.2
 AWS_K8S_TESTER_BIN=$OUTPUT_BIN/aws-k8s-tester

From 9a3c332bf228fa37474537275e2bca279004baee Mon Sep 17 00:00:00 2001
From: Yecheng Fu <fuyecheng@pingcap.com>
Date: Thu, 20 Feb 2020 15:05:36 +0800
Subject: [PATCH 03/13] prefix image tag with CLUSTER, then multiple clusters
 can be started in same project/account

---
 hack/run-e2e.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/hack/run-e2e.sh b/hack/run-e2e.sh
index fee9cb3dfd..1b5c5e440d 100755
--- a/hack/run-e2e.sh
+++ b/hack/run-e2e.sh
@@ -211,8 +211,8 @@ function e2e::image_load() {
     elif [ "$PROVIDER" == "gke" ]; then
         unset DOCKER_CONFIG # We don't need this and it may be read-only and fail the command to fail
         gcloud auth configure-docker
-        GCP_TIDB_OPERATOR_IMAGE=gcr.io/$GCP_PROJECT/tidb-operator:$IMAGE_TAG
-        GCP_E2E_IMAGE=gcr.io/$GCP_PROJECT/tidb-operator-e2e:$IMAGE_TAG
+        GCP_TIDB_OPERATOR_IMAGE=gcr.io/$GCP_PROJECT/tidb-operator:$CLUSTER-$IMAGE_TAG
+        GCP_E2E_IMAGE=gcr.io/$GCP_PROJECT/tidb-operator-e2e:$CLUSTER-$IMAGE_TAG
         docker tag $TIDB_OPERATOR_IMAGE $GCP_TIDB_OPERATOR_IMAGE
         docker tag $E2E_IMAGE $GCP_E2E_IMAGE
         echo "info: pushing $GCP_TIDB_OPERATOR_IMAGE"
@@ -233,8 +233,8 @@ function e2e::image_load() {
         local ecrURL=$(e2e::__ecr_url)
         echo "info: logging in $ecrURL"
         aws ecr get-login-password | docker login --username AWS --password-stdin $ecrURL
-        AWS_TIDB_OPERATOR_IMAGE=$ecrURL/e2e/tidb-operator:$IMAGE_TAG
-        AWS_E2E_IMAGE=$ecrURL/e2e/tidb-operator-e2e:$IMAGE_TAG
+        AWS_TIDB_OPERATOR_IMAGE=$ecrURL/e2e/tidb-operator:$CLUSTER-$IMAGE_TAG
+        AWS_E2E_IMAGE=$ecrURL/e2e/tidb-operator-e2e:$CLUSTER-$IMAGE_TAG
         docker tag $TIDB_OPERATOR_IMAGE $AWS_TIDB_OPERATOR_IMAGE
         docker tag $E2E_IMAGE $AWS_E2E_IMAGE
         echo "info: pushing $AWS_TIDB_OPERATOR_IMAGE"

From d18c7e7d949db64a3867fb3458f11453cd6741de Mon Sep 17 00:00:00 2001
From: Yecheng Fu <fuyecheng@pingcap.com>
Date: Thu, 20 Feb 2020 16:52:00 +0800
Subject: [PATCH 04/13] - upgrade kubetest2-eks to v0.0.4 - use unique node
 group name

---
 hack/e2e.sh | 3 ++-
 hack/lib.sh | 8 ++++----
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/hack/e2e.sh b/hack/e2e.sh
index c922450e33..f21348ad1f 100755
--- a/hack/e2e.sh
+++ b/hack/e2e.sh
@@ -452,10 +452,11 @@ elif [ "$PROVIDER" == "eks" ]; then
     if [ -n "$AWS_SECRET_ACCESS_KEY" ]; then
         aws configure set aws_secret_access_key "$AWS_SECRET_ACCESS_KEY"
     fi
+    mngName=kubetest.eks.$CLUSTER.mng.$(xxd -l5  -ps /dev/urandom)
     export AWS_K8S_TESTER_EKS_NAME=$CLUSTER
     export AWS_K8S_TESTER_EKS_CONFIG_PATH=/tmp/kubetest2.eks.$CLUSTER
     export AWS_K8S_TESTER_EKS_ADD_ON_NLB_HELLO_WORLD_ENABLE="false"
-    export AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_MNGS='{"aws-k8s-tester-tidb-operator-mng":{"name":"aws-k8s-tester-tidb-operator-mng","ami-type":"AL2_x86_64","asg-min-size":3,"asg-max-size":3,"asg-desired-capacity":3,"instance-types":["c5.xlarge"],"volume-size":40}}'
+    export AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_MNGS=$(printf '{"%s":{"name":"%s","ami-type":"AL2_x86_64","asg-min-size":3,"asg-max-size":3,"asg-desired-capacity":3,"instance-types":["c5.xlarge"],"volume-size":40}}' "$mngName" "$mngName")
     # override KUBECONFIG
     KUBECONFIG=$AWS_K8S_TESTER_EKS_CONFIG_PATH.kubeconfig.yaml
     # clear previous created private key to work around permission issue on this file
diff --git a/hack/lib.sh b/hack/lib.sh
index fad41694b3..21cc271362 100644
--- a/hack/lib.sh
+++ b/hack/lib.sh
@@ -34,10 +34,10 @@ HELM_BIN=$OUTPUT_BIN/helm
 HELM_VERSION=${HELM_VERSION:-2.9.1}
 KIND_VERSION=${KIND_VERSION:-0.7.0}
 KIND_BIN=$OUTPUT_BIN/kind
-KUBETEST2_VERSION=v0.0.3+c3df273d31e01450e53d23b7482ae5a834a0a682
-KUBETEST2_EKS_VERSION=v0.0.3+63a212b4833fa8528253506f1b05e2ceccbc7b88
-KUBETEST2_GKE_VERSION=v0.0.3+2ed37632a7070fddeb93955dadc8984462ee6a5d
-KUBETEST2_KIND_VERSION=v0.0.3+f22b63a132e0c0f144940558c58cf87159cfad0e
+KUBETEST2_VERSION=v0.0.4+c23999f66bf004c79cad2113098d06fa3c1f5e3f
+KUBETEST2_EKS_VERSION=v0.0.4+c2ff633f7690e04bfd44f074093635c62fceff0e
+KUBETEST2_GKE_VERSION=v0.0.4+21574e224e7285844ce0e3a3335c2ecb57c72290
+KUBETEST2_KIND_VERSION=v0.0.4+befc59a4eb11506c0078540c9ee7b1a493cc1f1b
 KUBETSTS2_BIN=$OUTPUT_BIN/kubetest2
 AWS_K8S_TESTER_VERSION=v0.6.2
 AWS_K8S_TESTER_BIN=$OUTPUT_BIN/aws-k8s-tester

From fcf771249a1a19f8bae5121a086b7b8d1457c59f Mon Sep 17 00:00:00 2001
From: Yecheng Fu <fuyecheng@pingcap.com>
Date: Thu, 20 Feb 2020 17:05:18 +0800
Subject: [PATCH 05/13] $RANDOM should be enough

---
 hack/e2e.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hack/e2e.sh b/hack/e2e.sh
index f21348ad1f..9958a0d1ec 100755
--- a/hack/e2e.sh
+++ b/hack/e2e.sh
@@ -452,7 +452,7 @@ elif [ "$PROVIDER" == "eks" ]; then
     if [ -n "$AWS_SECRET_ACCESS_KEY" ]; then
         aws configure set aws_secret_access_key "$AWS_SECRET_ACCESS_KEY"
     fi
-    mngName=kubetest.eks.$CLUSTER.mng.$(xxd -l5  -ps /dev/urandom)
+    mngName=kubetest.eks.$CLUSTER.mng.$RANDOM
     export AWS_K8S_TESTER_EKS_NAME=$CLUSTER
     export AWS_K8S_TESTER_EKS_CONFIG_PATH=/tmp/kubetest2.eks.$CLUSTER
     export AWS_K8S_TESTER_EKS_ADD_ON_NLB_HELLO_WORLD_ENABLE="false"

From 908331abb4da946fbb86e8c5d26788e1b54e1979 Mon Sep 17 00:00:00 2001
From: Yecheng Fu <fuyecheng@pingcap.com>
Date: Thu, 20 Feb 2020 20:04:30 +0800
Subject: [PATCH 06/13] support KUBE_WORKERS

---
 hack/e2e.sh | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/hack/e2e.sh b/hack/e2e.sh
index 9958a0d1ec..2592ecad3f 100755
--- a/hack/e2e.sh
+++ b/hack/e2e.sh
@@ -456,12 +456,14 @@ elif [ "$PROVIDER" == "eks" ]; then
     export AWS_K8S_TESTER_EKS_NAME=$CLUSTER
     export AWS_K8S_TESTER_EKS_CONFIG_PATH=/tmp/kubetest2.eks.$CLUSTER
     export AWS_K8S_TESTER_EKS_ADD_ON_NLB_HELLO_WORLD_ENABLE="false"
-    export AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_MNGS=$(printf '{"%s":{"name":"%s","ami-type":"AL2_x86_64","asg-min-size":3,"asg-max-size":3,"asg-desired-capacity":3,"instance-types":["c5.xlarge"],"volume-size":40}}' "$mngName" "$mngName")
+    export AWS_K8S_TESTER_EKS_ADD_ON_MANAGED_NODE_GROUPS_MNGS=$(printf '{"%s":{"name":"%s","ami-type":"AL2_x86_64","asg-min-size":%d,"asg-max-size":%d,"asg-desired-capacity":%d,"instance-types":["c5.xlarge"],"volume-size":40}}' "$mngName" "$mngName" "$KUBE_WORKERS" "$KUBE_WORKERS" "$KUBE_WORKERS")
     # override KUBECONFIG
     KUBECONFIG=$AWS_K8S_TESTER_EKS_CONFIG_PATH.kubeconfig.yaml
-    # clear previous created private key to work around permission issue on this file
-    if test -f $HOME/.ssh/kube_aws_rsa; then
-        rm -f $HOME/.ssh/kube_aws_rsa
+    if [ -z "$SKIP_UP" ]; then
+        # clear previous created private key to work around permission issue on this file
+        if test -f $HOME/.ssh/kube_aws_rsa; then
+            rm -f $HOME/.ssh/kube_aws_rsa
+        fi
     fi
 else
     echo "error: unsupported provider '$PROVIDER'"

From 590b08517575ba71000bf777387fc70db2c2278c Mon Sep 17 00:00:00 2001
From: Yecheng Fu <fuyecheng@pingcap.com>
Date: Thu, 20 Feb 2020 22:12:42 +0800
Subject: [PATCH 07/13] fix mngName

---
 hack/e2e.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/hack/e2e.sh b/hack/e2e.sh
index 2592ecad3f..30121b91a4 100755
--- a/hack/e2e.sh
+++ b/hack/e2e.sh
@@ -452,7 +452,7 @@ elif [ "$PROVIDER" == "eks" ]; then
     if [ -n "$AWS_SECRET_ACCESS_KEY" ]; then
         aws configure set aws_secret_access_key "$AWS_SECRET_ACCESS_KEY"
     fi
-    mngName=kubetest.eks.$CLUSTER.mng.$RANDOM
+    mngName=$CLUSTER-mng-$RANDOM
     export AWS_K8S_TESTER_EKS_NAME=$CLUSTER
     export AWS_K8S_TESTER_EKS_CONFIG_PATH=/tmp/kubetest2.eks.$CLUSTER
     export AWS_K8S_TESTER_EKS_ADD_ON_NLB_HELLO_WORLD_ENABLE="false"

From ff08d6de73b87776e057dd41d50d143f669faf8b Mon Sep 17 00:00:00 2001
From: Yecheng Fu <fuyecheng@pingcap.com>
Date: Fri, 21 Feb 2020 11:12:48 +0800
Subject: [PATCH 08/13] fix e2e bug

---
 tests/actions.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/actions.go b/tests/actions.go
index ce21dc51d2..95639de4d9 100644
--- a/tests/actions.go
+++ b/tests/actions.go
@@ -2297,7 +2297,7 @@ func (oa *operatorActions) DeployAdHocBackup(info *TidbClusterConfig) error {
 		)
 		glog.Info(getTSCmd)
 
-		res, err := exec.Command("/bin/sh", "-c", getTSCmd).CombinedOutput()
+		res, err := exec.Command("/bin/bash", "-c", getTSCmd).CombinedOutput()
 		if err != nil {
 			glog.Errorf("failed to get ts %v, %s", err, string(res))
 			return false, nil

From 8107ea3e2940311719aa713fde42404b02412447 Mon Sep 17 00:00:00 2001
From: Yecheng Fu <fuyecheng@pingcap.com>
Date: Fri, 21 Feb 2020 14:07:54 +0800
Subject: [PATCH 09/13] specify runner suite name

---
 ci/pingcap_tidb_operator_build_kind.groovy | 8 ++++----
 hack/e2e.sh                                | 8 ++++++++
 hack/lib.sh                                | 8 ++++----
 3 files changed, 16 insertions(+), 8 deletions(-)

diff --git a/ci/pingcap_tidb_operator_build_kind.groovy b/ci/pingcap_tidb_operator_build_kind.groovy
index 45ec44403d..9b7a046eda 100644
--- a/ci/pingcap_tidb_operator_build_kind.groovy
+++ b/ci/pingcap_tidb_operator_build_kind.groovy
@@ -238,16 +238,16 @@ def call(BUILD_BRANCH, CREDENTIALS_ID, CODECOV_CREDENTIALS_ID) {
 		def MIRRORS = "DOCKER_IO_MIRROR=http://172.16.4.143:5000 QUAY_IO_MIRROR=http://172.16.4.143:5001"
 		def builds = [:]
 		builds["E2E v1.12.10"] = {
-			build("${MIRRORS} IMAGE_TAG=${GITHASH} SKIP_BUILD=y GINKGO_NODES=8 KUBE_VERSION=v1.12.10 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.12.10_ ./hack/e2e.sh -- --preload-images --ginkgo.skip='\\[Serial\\]'", artifacts)
+			build("${MIRRORS} RUNNER_SUITE_NAME=e2e-v1.12 IMAGE_TAG=${GITHASH} SKIP_BUILD=y GINKGO_NODES=8 KUBE_VERSION=v1.12.10 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.12.10_ ./hack/e2e.sh -- --preload-images --ginkgo.skip='\\[Serial\\]'", artifacts)
 		}
 		builds["E2E v1.12.10 AdvancedStatefulSet"] = {
-			build("${MIRRORS} IMAGE_TAG=${GITHASH} SKIP_BUILD=y GINKGO_NODES=8 KUBE_VERSION=v1.12.10 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.12.10_advanced_statefulset ./hack/e2e.sh -- --preload-images --ginkgo.skip='\\[Serial\\]' --operator-features AdvancedStatefulSet=true", artifacts)
+			build("${MIRRORS} RUNNER_SUITE_NAME=e2e-v1.12-advanced-statefulset IMAGE_TAG=${GITHASH} SKIP_BUILD=y GINKGO_NODES=8 KUBE_VERSION=v1.12.10 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.12.10_advanced_statefulset ./hack/e2e.sh -- --preload-images --ginkgo.skip='\\[Serial\\]' --operator-features AdvancedStatefulSet=true", artifacts)
 		}
 		builds["E2E v1.17.0"] = {
-			build("${MIRRORS} IMAGE_TAG=${GITHASH} SKIP_BUILD=y GINKGO_NODES=8 KUBE_VERSION=v1.17.0 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.17.0_ ./hack/e2e.sh -- -preload-images --ginkgo.skip='\\[Serial\\]'", artifacts)
+			build("${MIRRORS} RUNNER_SUITE_NAME=e2e-v1.17 IMAGE_TAG=${GITHASH} SKIP_BUILD=y GINKGO_NODES=8 KUBE_VERSION=v1.17.0 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.17.0_ ./hack/e2e.sh -- -preload-images --ginkgo.skip='\\[Serial\\]'", artifacts)
 		}
 		builds["E2E v1.12.10 Serial"] = {
-			build("${MIRRORS} IMAGE_TAG=${GITHASH} SKIP_BUILD=y KUBE_VERSION=v1.12.10 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.12.10_serial_ ./hack/e2e.sh -- --preload-images --ginkgo.focus='\\[Serial\\]' --install-operator=false", artifacts)
+			build("${MIRRORS} RUNNER_SUITE_NAME=e2e-v1.12-serial IMAGE_TAG=${GITHASH} SKIP_BUILD=y KUBE_VERSION=v1.12.10 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.12.10_serial_ ./hack/e2e.sh -- --preload-images --ginkgo.focus='\\[Serial\\]' --install-operator=false", artifacts)
 		}
 		builds.failFast = false
 		parallel builds
diff --git a/hack/e2e.sh b/hack/e2e.sh
index 30121b91a4..c97adf86f5 100755
--- a/hack/e2e.sh
+++ b/hack/e2e.sh
@@ -74,6 +74,7 @@ Environments:
     GINKGO_NODES          ginkgo nodes to run specs, defaults: 1
     GINKGO_PARALLEL       if set to `y`, will run specs in parallel, the number of nodes will be the number of cpus
     GINKGO_NO_COLOR       if set to `y`, suppress color output in default reporter
+    RUNNER_SUITE_NAME     the suite name of runner
 
 Examples:
 
@@ -195,6 +196,7 @@ KUBE_WORKERS=${KUBE_WORKERS:-3}
 DOCKER_IO_MIRROR=${DOCKER_IO_MIRROR:-}
 GCR_IO_MIRROR=${GCR_IO_MIRROR:-}
 QUAY_IO_MIRROR=${QUAY_IO_MIRROR:-}
+RUNNER_SUITE_NAME=${RUNNER_SUITE_NAME:-}
 
 echo "PROVIDER: $PROVIDER"
 echo "DOCKER_REGISTRY: $DOCKER_REGISTRY"
@@ -360,6 +362,12 @@ kubetest2_args=(
     $PROVIDER
 )
 
+if [ -n "$RUNNER_SUITE_NAME" ]; then
+    kubetest2_args+=(
+        --suite-name "$RUNNER_SUITE_NAME"
+    )
+fi
+
 if [ -z "$SKIP_UP" ]; then
     kubetest2_args+=(--up)
 fi
diff --git a/hack/lib.sh b/hack/lib.sh
index 21cc271362..3a9d345968 100644
--- a/hack/lib.sh
+++ b/hack/lib.sh
@@ -34,10 +34,10 @@ HELM_BIN=$OUTPUT_BIN/helm
 HELM_VERSION=${HELM_VERSION:-2.9.1}
 KIND_VERSION=${KIND_VERSION:-0.7.0}
 KIND_BIN=$OUTPUT_BIN/kind
-KUBETEST2_VERSION=v0.0.4+c23999f66bf004c79cad2113098d06fa3c1f5e3f
-KUBETEST2_EKS_VERSION=v0.0.4+c2ff633f7690e04bfd44f074093635c62fceff0e
-KUBETEST2_GKE_VERSION=v0.0.4+21574e224e7285844ce0e3a3335c2ecb57c72290
-KUBETEST2_KIND_VERSION=v0.0.4+befc59a4eb11506c0078540c9ee7b1a493cc1f1b
+KUBETEST2_VERSION=v0.0.5+8b0d44d839810c125ee4ab12004d1ede2dc88004
+KUBETEST2_EKS_VERSION=v0.0.5+b5182f351791f085f1ff6c51f8c370887c5d99ec
+KUBETEST2_GKE_VERSION=v0.0.5+bc2913cac967200792f00479bc2e6ac57bcfdb3b
+KUBETEST2_KIND_VERSION=v0.0.5+8b81fb1be19fd0419b5b0f3dff497042385f6b72
 KUBETSTS2_BIN=$OUTPUT_BIN/kubetest2
 AWS_K8S_TESTER_VERSION=v0.6.2
 AWS_K8S_TESTER_BIN=$OUTPUT_BIN/aws-k8s-tester

From 01ea234fa8f76126397c4089b958b2570167b17d Mon Sep 17 00:00:00 2001
From: Yecheng Fu <fuyecheng@pingcap.com>
Date: Fri, 21 Feb 2020 16:06:24 +0800
Subject: [PATCH 10/13] increase open files for containers automatically

---
 hack/run-e2e.sh | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/hack/run-e2e.sh b/hack/run-e2e.sh
index 1b5c5e440d..d1b5341250 100755
--- a/hack/run-e2e.sh
+++ b/hack/run-e2e.sh
@@ -158,6 +158,11 @@ for ((i = 1; i <= 32; i++)) {
         mount --bind vol$i vol$i
     fi
 }
+echo "info: increase max open files for containers"
+if ! grep -qF "OPTIONS" /etc/sysconfig/docker; then
+    echo 'OPTIONS="--default-ulimit nofile=1024000:1024000"' >> /etc/sysconfig/docker
+fi
+systemctl restart docker
 '
 EOF
         done <<< "$(e2e::__eks_instances)"

From 3fe07207162c9056cd85fd39ea396323fc9fa167 Mon Sep 17 00:00:00 2001
From: Yecheng Fu <fuyecheng@pingcap.com>
Date: Sat, 22 Feb 2020 16:34:26 +0800
Subject: [PATCH 11/13] use kubetest2 v0.0.6

---
 hack/lib.sh | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/hack/lib.sh b/hack/lib.sh
index 3a9d345968..c4314b59ef 100644
--- a/hack/lib.sh
+++ b/hack/lib.sh
@@ -34,10 +34,10 @@ HELM_BIN=$OUTPUT_BIN/helm
 HELM_VERSION=${HELM_VERSION:-2.9.1}
 KIND_VERSION=${KIND_VERSION:-0.7.0}
 KIND_BIN=$OUTPUT_BIN/kind
-KUBETEST2_VERSION=v0.0.5+8b0d44d839810c125ee4ab12004d1ede2dc88004
-KUBETEST2_EKS_VERSION=v0.0.5+b5182f351791f085f1ff6c51f8c370887c5d99ec
-KUBETEST2_GKE_VERSION=v0.0.5+bc2913cac967200792f00479bc2e6ac57bcfdb3b
-KUBETEST2_KIND_VERSION=v0.0.5+8b81fb1be19fd0419b5b0f3dff497042385f6b72
+KUBETEST2_VERSION=v0.0.6+81d814748ab990ecd893cd1313edfb82400752bd
+KUBETEST2_EKS_VERSION=v0.0.6+d6afb853359f35999c6aa3c06ec96cb8ebcbd032
+KUBETEST2_GKE_VERSION=v0.0.6+12f40220e086ff4d4aa86b98d05cfc62f17d9cf9
+KUBETEST2_KIND_VERSION=v0.0.6+b4be23daed89152e595dc3ad4826d104107edc62
 KUBETSTS2_BIN=$OUTPUT_BIN/kubetest2
 AWS_K8S_TESTER_VERSION=v0.6.2
 AWS_K8S_TESTER_BIN=$OUTPUT_BIN/aws-k8s-tester
@@ -173,6 +173,7 @@ function hack::__ensure_kubetest2() {
     fi
     local tmpfile=$(mktemp)
     trap "test -f $tmpfile && rm $tmpfile" RETURN
+    echo "info: downloading $n $v"
     curl --retry 10 -L -o - https://github.com/cofyc/kubetest2/releases/download/$v/$n.gz | gunzip > $tmpfile
     mv $tmpfile $OUTPUT_BIN/$n
     chmod +x $OUTPUT_BIN/$n

From ce2e1154e3f9043a9c70781cb65c3a1ddf9a41c4 Mon Sep 17 00:00:00 2001
From: Yecheng Fu <fuyecheng@pingcap.com>
Date: Sat, 22 Feb 2020 17:17:45 +0800
Subject: [PATCH 12/13] --up-retries

---
 hack/e2e.sh | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/hack/e2e.sh b/hack/e2e.sh
index c97adf86f5..2c7ba89233 100755
--- a/hack/e2e.sh
+++ b/hack/e2e.sh
@@ -399,6 +399,9 @@ if [ "$PROVIDER" == "kind" ]; then
     fi
     kubetest2_args+=(--image-name $image)
     kubetest2_args+=(
+        # add some retires because kind may fail to start the cluster when the
+        # load is high
+        --up-retries 3
         --cluster-name "$CLUSTER"
         --config "$tmpfile"
         --verbosity 4

From 57f0e12938eea36b2ff900055154568a4bb0284f Mon Sep 17 00:00:00 2001
From: Yecheng Fu <fuyecheng@pingcap.com>
Date: Sat, 22 Feb 2020 21:41:21 +0800
Subject: [PATCH 13/13] decrease concurrency because in each node we will start
 a lots of pod

---
 ci/pingcap_tidb_operator_build_kind.groovy | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/ci/pingcap_tidb_operator_build_kind.groovy b/ci/pingcap_tidb_operator_build_kind.groovy
index 9b7a046eda..b919be05d8 100644
--- a/ci/pingcap_tidb_operator_build_kind.groovy
+++ b/ci/pingcap_tidb_operator_build_kind.groovy
@@ -238,13 +238,13 @@ def call(BUILD_BRANCH, CREDENTIALS_ID, CODECOV_CREDENTIALS_ID) {
 		def MIRRORS = "DOCKER_IO_MIRROR=http://172.16.4.143:5000 QUAY_IO_MIRROR=http://172.16.4.143:5001"
 		def builds = [:]
 		builds["E2E v1.12.10"] = {
-			build("${MIRRORS} RUNNER_SUITE_NAME=e2e-v1.12 IMAGE_TAG=${GITHASH} SKIP_BUILD=y GINKGO_NODES=8 KUBE_VERSION=v1.12.10 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.12.10_ ./hack/e2e.sh -- --preload-images --ginkgo.skip='\\[Serial\\]'", artifacts)
+			build("${MIRRORS} RUNNER_SUITE_NAME=e2e-v1.12 IMAGE_TAG=${GITHASH} SKIP_BUILD=y GINKGO_NODES=6 KUBE_VERSION=v1.12.10 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.12.10_ ./hack/e2e.sh -- --preload-images --ginkgo.skip='\\[Serial\\]'", artifacts)
 		}
 		builds["E2E v1.12.10 AdvancedStatefulSet"] = {
-			build("${MIRRORS} RUNNER_SUITE_NAME=e2e-v1.12-advanced-statefulset IMAGE_TAG=${GITHASH} SKIP_BUILD=y GINKGO_NODES=8 KUBE_VERSION=v1.12.10 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.12.10_advanced_statefulset ./hack/e2e.sh -- --preload-images --ginkgo.skip='\\[Serial\\]' --operator-features AdvancedStatefulSet=true", artifacts)
+			build("${MIRRORS} RUNNER_SUITE_NAME=e2e-v1.12-advanced-statefulset IMAGE_TAG=${GITHASH} SKIP_BUILD=y GINKGO_NODES=6 KUBE_VERSION=v1.12.10 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.12.10_advanced_statefulset ./hack/e2e.sh -- --preload-images --ginkgo.skip='\\[Serial\\]' --operator-features AdvancedStatefulSet=true", artifacts)
 		}
 		builds["E2E v1.17.0"] = {
-			build("${MIRRORS} RUNNER_SUITE_NAME=e2e-v1.17 IMAGE_TAG=${GITHASH} SKIP_BUILD=y GINKGO_NODES=8 KUBE_VERSION=v1.17.0 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.17.0_ ./hack/e2e.sh -- -preload-images --ginkgo.skip='\\[Serial\\]'", artifacts)
+			build("${MIRRORS} RUNNER_SUITE_NAME=e2e-v1.17 IMAGE_TAG=${GITHASH} SKIP_BUILD=y GINKGO_NODES=6 KUBE_VERSION=v1.17.0 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.17.0_ ./hack/e2e.sh -- -preload-images --ginkgo.skip='\\[Serial\\]'", artifacts)
 		}
 		builds["E2E v1.12.10 Serial"] = {
 			build("${MIRRORS} RUNNER_SUITE_NAME=e2e-v1.12-serial IMAGE_TAG=${GITHASH} SKIP_BUILD=y KUBE_VERSION=v1.12.10 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.12.10_serial_ ./hack/e2e.sh -- --preload-images --ginkgo.focus='\\[Serial\\]' --install-operator=false", artifacts)