diff --git a/ci/e2e_kind.groovy b/ci/e2e_kind.groovy new file mode 100644 index 00000000000..245b75a3b8b --- /dev/null +++ b/ci/e2e_kind.groovy @@ -0,0 +1,220 @@ +// +// Jenkins pipeline for Kind e2e job. +// +// This script is written in declarative syntax. Refer to +// https://jenkins.io/doc/book/pipeline/syntax/ for more details. +// +// Note that parameters of the job is configured in this script. +// + +import groovy.transform.Field + +@Field +def podYAML = ''' +apiVersion: v1 +kind: Pod +metadata: + labels: + app: tidb-operator-e2e +spec: + containers: + - name: main + image: gcr.io/k8s-testimages/kubekins-e2e:v20200311-1e25827-master + command: + - runner.sh + # Clean containers on TERM signal in root process to avoid cgroup leaking. + # https://github.com/pingcap/tidb-operator/issues/1603#issuecomment-582402196 + - exec + - bash + - -c + - | + function clean() { + echo "info: clean all containers to avoid cgroup leaking" + docker kill $(docker ps -q) || true + docker system prune -af || true + } + trap clean TERM + sleep 1d & wait + # we need privileged mode in order to do docker in docker + securityContext: + privileged: true + env: + - name: DOCKER_IN_DOCKER_ENABLED + value: "true" + resources: + requests: + memory: "8000Mi" + cpu: 8000m + ephemeral-storage: "50Gi" + limits: + memory: "8000Mi" + cpu: 8000m + ephemeral-storage: "50Gi" + # kind needs /lib/modules and cgroups from the host + volumeMounts: + - mountPath: /lib/modules + name: modules + readOnly: true + - mountPath: /sys/fs/cgroup + name: cgroup + # dind expects /var/lib/docker to be volume + - name: docker-root + mountPath: /var/lib/docker + # legacy docker path for cr.io/k8s-testimages/kubekins-e2e + - name: docker-graph + mountPath: /docker-graph + volumes: + - name: modules + hostPath: + path: /lib/modules + type: Directory + - name: cgroup + hostPath: + path: /sys/fs/cgroup + type: Directory + - name: docker-root + emptyDir: {} + - name: docker-graph + emptyDir: {} + tolerations: + - effect: NoSchedule + key: tidb-operator + operator: Exists + affinity: + # running on nodes for tidb-operator only + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: ci.pingcap.com + operator: In + values: + - tidb-operator + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - tidb-operator-e2e + topologyKey: kubernetes.io/hostname +''' + +// Able to override default values in Jenkins job via environment variables. +if (!env.DEFAULT_GIT_REF) { + env.DEFAULT_GIT_REF = "master" +} + +if (!env.DEFAULT_GINKGO_NODES) { + env.DEFAULT_GINKGO_NODES = "8" +} + +if (!env.DEFAULT_E2E_ARGS) { + env.DEFAULT_E2E_ARGS = "" +} + +if (!env.DEFAULT_DOCKER_IO_MIRROR) { + env.DEFAULT_DOCKER_IO_MIRROR = "" +} + +if (!env.DEFAULT_QUAY_IO_MIRROR) { + env.DEFAULT_QUAY_IO_MIRROR = "" +} + +if (!env.DEFAULT_GCR_IO_MIRROR) { + env.DEFAULT_GCR_IO_MIRROR = "" +} + +pipeline { + agent { + kubernetes { + yaml podYAML + defaultContainer "main" + customWorkspace "/home/jenkins/agent/workspace/go/src/github.com/pingcap/tidb-operator" + } + } + + options { + timeout(time: 3, unit: 'HOURS') + } + + parameters { + string(name: 'GIT_URL', defaultValue: 'git@github.com:pingcap/tidb-operator.git', description: 'git repo url') + string(name: 'GIT_REF', defaultValue: env.DEFAULT_GIT_REF, description: 'git ref spec to checkout, e.g. master, release-1.1') + string(name: 'PR_ID', defaultValue: '', description: 'pull request ID, this will override GIT_REF if set, e.g. 1889') + string(name: 'GINKGO_NODES', defaultValue: env.DEFAULT_GINKGO_NODES, description: 'the number of ginkgo nodes') + string(name: 'E2E_ARGS', defaultValue: env.DEFAULT_E2E_ARGS, description: "e2e args, e.g. --ginkgo.focus='\\[Stability\\]'") + string(name: 'DOCKER_IO_MIRROR', defaultValue: env.DEFAULT_DOCKER_IO_MIRROR, description: "docker mirror for docker.io") + string(name: 'QUAY_IO_MIRROR', defaultValue: env.DEFAULT_QUAY_IO_MIRROR, description: "mirror for quay.io") + string(name: 'GCR_IO_MIRROR', defaultValue: env.DEFAULT_GCR_IO_MIRROR, description: "mirror for gcr.io") + } + + environment { + GIT_REF = '' + ARTIFACTS = "${env.WORKSPACE}/artifacts" + } + + stages { + stage("Prepare") { + steps { + // The declarative model for Jenkins Pipelines has a restricted + // subset of syntax that it allows in the stage blocks. We use + // script step to bypass the restriction. + // https://jenkins.io/doc/book/pipeline/syntax/#script + script { + GIT_REF = params.GIT_REF + if (params.PR_ID != "") { + GIT_REF = "refs/remotes/origin/pr/${params.PR_ID}/head" + } + } + echo "env.NODE_NAME: ${env.NODE_NAME}" + echo "env.WORKSPACE: ${env.WORKSPACE}" + echo "GIT_REF: ${GIT_REF}" + echo "ARTIFACTS: ${ARTIFACTS}" + } + } + + stage("Checkout") { + steps { + checkout scm: [ + $class: 'GitSCM', + branches: [[name: GIT_REF]], + userRemoteConfigs: [[ + credentialsId: 'github-sre-bot-ssh', + refspec: '+refs/heads/*:refs/remotes/origin/* +refs/pull/*:refs/remotes/origin/pr/*', + url: "${params.GIT_URL}", + ]] + ] + } + } + + stage("Run") { + steps { + sh """ + #!/bin/bash + export GINKGO_NODES=${params.GINKGO_NODES} + export REPORT_DIR=${ARTIFACTS} + export DOCKER_IO_MIRROR=${params.DOCKER_IO_MIRROR} + export QUAY_IO_MIRROR=${params.QUAY_IO_MIRROR} + export GCR_IO_MIRROR=${params.GCR_IO_MIRROR} + echo "info: begin to run e2e" + ./hack/e2e.sh -- ${params.E2E_ARGS} + """ + } + } + } + + post { + always { + dir(ARTIFACTS) { + archiveArtifacts artifacts: "**", allowEmptyArchive: true + junit testResults: "*.xml", allowEmptyResults: true + } + } + } +} + +// vim: et sw=4 ts=4 diff --git a/ci/pingcap_tidb_operator_build_kind.groovy b/ci/pingcap_tidb_operator_build_kind.groovy index aaa97ed6ff8..70ab0486bba 100644 --- a/ci/pingcap_tidb_operator_build_kind.groovy +++ b/ci/pingcap_tidb_operator_build_kind.groovy @@ -238,13 +238,13 @@ def call(BUILD_BRANCH, CREDENTIALS_ID, CODECOV_CREDENTIALS_ID) { def MIRRORS = "DOCKER_IO_MIRROR=http://172.16.4.143:5000 QUAY_IO_MIRROR=http://172.16.4.143:5001" def builds = [:] builds["E2E v1.12.10"] = { - build("${MIRRORS} RUNNER_SUITE_NAME=e2e-v1.12 IMAGE_TAG=${GITHASH} SKIP_BUILD=y GINKGO_NODES=6 KUBE_VERSION=v1.12.10 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.12.10_ ./hack/e2e.sh -- --preload-images --ginkgo.skip='\\[Serial\\]'", artifacts) + build("${MIRRORS} RUNNER_SUITE_NAME=e2e-v1.12 IMAGE_TAG=${GITHASH} SKIP_BUILD=y GINKGO_NODES=6 KUBE_VERSION=v1.12.10 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.12.10_ ./hack/e2e.sh -- --preload-images", artifacts) } builds["E2E v1.12.10 AdvancedStatefulSet"] = { - build("${MIRRORS} RUNNER_SUITE_NAME=e2e-v1.12-advanced-statefulset IMAGE_TAG=${GITHASH} SKIP_BUILD=y GINKGO_NODES=6 KUBE_VERSION=v1.12.10 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.12.10_advanced_statefulset ./hack/e2e.sh -- --preload-images --ginkgo.skip='\\[Serial\\]' --operator-features AdvancedStatefulSet=true", artifacts) + build("${MIRRORS} RUNNER_SUITE_NAME=e2e-v1.12-advanced-statefulset IMAGE_TAG=${GITHASH} SKIP_BUILD=y GINKGO_NODES=6 KUBE_VERSION=v1.12.10 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.12.10_advanced_statefulset ./hack/e2e.sh -- --preload-images --operator-features AdvancedStatefulSet=true", artifacts) } builds["E2E v1.17.0"] = { - build("${MIRRORS} RUNNER_SUITE_NAME=e2e-v1.17 IMAGE_TAG=${GITHASH} SKIP_BUILD=y GINKGO_NODES=6 KUBE_VERSION=v1.17.0 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.17.0_ ./hack/e2e.sh -- -preload-images --ginkgo.skip='\\[Serial\\]'", artifacts) + build("${MIRRORS} RUNNER_SUITE_NAME=e2e-v1.17 IMAGE_TAG=${GITHASH} SKIP_BUILD=y GINKGO_NODES=6 KUBE_VERSION=v1.17.0 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.17.0_ ./hack/e2e.sh -- -preload-images", artifacts) } builds["E2E v1.12.10 Serial"] = { build("${MIRRORS} RUNNER_SUITE_NAME=e2e-v1.12-serial IMAGE_TAG=${GITHASH} SKIP_BUILD=y KUBE_VERSION=v1.12.10 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.12.10_serial_ ./hack/e2e.sh -- --preload-images --ginkgo.focus='\\[Serial\\]' --install-operator=false", artifacts) diff --git a/cmd/backup-manager/app/backup/backup.go b/cmd/backup-manager/app/backup/backup.go index 5e9ab4210d1..2becef8bf6d 100644 --- a/cmd/backup-manager/app/backup/backup.go +++ b/cmd/backup-manager/app/backup/backup.go @@ -23,15 +23,16 @@ import ( "github.com/gogo/protobuf/proto" kvbackup "github.com/pingcap/kvproto/pkg/backup" "github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants" - "github.com/pingcap/tidb-operator/cmd/backup-manager/app/util" + backupUtil "github.com/pingcap/tidb-operator/cmd/backup-manager/app/util" "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" + "github.com/pingcap/tidb-operator/pkg/util" corev1 "k8s.io/api/core/v1" "k8s.io/klog" ) // Options contains the input arguments to the backup command type Options struct { - util.GenericOptions + backupUtil.GenericOptions } func (bo *Options) backupData(backup *v1alpha1.Backup) (string, error) { @@ -44,10 +45,10 @@ func (bo *Options) backupData(backup *v1alpha1.Backup) (string, error) { return "", err } args = append(args, fmt.Sprintf("--pd=%s-pd.%s:2379", backup.Spec.BR.Cluster, clusterNamespace)) - if backup.Spec.BR.EnableTLSClient { - args = append(args, fmt.Sprintf("--ca=%s", constants.ServiceAccountCAPath)) - args = append(args, fmt.Sprintf("--cert=%s", path.Join(constants.BRCertPath, corev1.TLSCertKey))) - args = append(args, fmt.Sprintf("--key=%s", path.Join(constants.BRCertPath, corev1.TLSPrivateKeyKey))) + if backup.Spec.BR.TLSCluster != nil && backup.Spec.BR.TLSCluster.Enabled { + args = append(args, fmt.Sprintf("--ca=%s", path.Join(util.ClusterClientTLSPath, corev1.ServiceAccountRootCAKey))) + args = append(args, fmt.Sprintf("--cert=%s", path.Join(util.ClusterClientTLSPath, corev1.TLSCertKey))) + args = append(args, fmt.Sprintf("--key=%s", path.Join(util.ClusterClientTLSPath, corev1.TLSPrivateKeyKey))) } var btype string @@ -73,7 +74,7 @@ func (bo *Options) backupData(backup *v1alpha1.Backup) (string, error) { // getCommitTs get backup position from `EndVersion` in BR backup meta func getCommitTs(backup *v1alpha1.Backup) (uint64, error) { var commitTs uint64 - s, err := util.NewRemoteStorage(backup) + s, err := backupUtil.NewRemoteStorage(backup) if err != nil { return commitTs, err } @@ -101,7 +102,7 @@ func getCommitTs(backup *v1alpha1.Backup) (uint64, error) { // constructOptions constructs options for BR and also return the remote path func constructOptions(backup *v1alpha1.Backup) ([]string, string, error) { - args, remotePath, err := util.ConstructBRGlobalOptionsForBackup(backup) + args, remotePath, err := backupUtil.ConstructBRGlobalOptionsForBackup(backup) if err != nil { return args, remotePath, err } @@ -124,7 +125,7 @@ func constructOptions(backup *v1alpha1.Backup) ([]string, string, error) { // getBackupSize get the backup data size from remote func getBackupSize(backup *v1alpha1.Backup) (int64, error) { var size int64 - s, err := util.NewRemoteStorage(backup) + s, err := backupUtil.NewRemoteStorage(backup) if err != nil { return size, err } diff --git a/cmd/backup-manager/app/restore/restore.go b/cmd/backup-manager/app/restore/restore.go index 90d0667ee09..d2e5a643d87 100644 --- a/cmd/backup-manager/app/restore/restore.go +++ b/cmd/backup-manager/app/restore/restore.go @@ -18,15 +18,15 @@ import ( "os/exec" "path" - "github.com/pingcap/tidb-operator/cmd/backup-manager/app/constants" - "github.com/pingcap/tidb-operator/cmd/backup-manager/app/util" + backupUtil "github.com/pingcap/tidb-operator/cmd/backup-manager/app/util" "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" + "github.com/pingcap/tidb-operator/pkg/util" corev1 "k8s.io/api/core/v1" "k8s.io/klog" ) type Options struct { - util.GenericOptions + backupUtil.GenericOptions } func (ro *Options) restoreData(restore *v1alpha1.Restore) error { @@ -39,10 +39,10 @@ func (ro *Options) restoreData(restore *v1alpha1.Restore) error { return err } args = append(args, fmt.Sprintf("--pd=%s-pd.%s:2379", restore.Spec.BR.Cluster, clusterNamespace)) - if restore.Spec.BR.EnableTLSClient { - args = append(args, fmt.Sprintf("--ca=%s", constants.ServiceAccountCAPath)) - args = append(args, fmt.Sprintf("--cert=%s", path.Join(constants.BRCertPath, corev1.TLSCertKey))) - args = append(args, fmt.Sprintf("--key=%s", path.Join(constants.BRCertPath, corev1.TLSPrivateKeyKey))) + if restore.Spec.BR.TLSCluster != nil && restore.Spec.BR.TLSCluster.Enabled { + args = append(args, fmt.Sprintf("--ca=%s", path.Join(util.ClusterClientTLSPath, corev1.ServiceAccountRootCAKey))) + args = append(args, fmt.Sprintf("--cert=%s", path.Join(util.ClusterClientTLSPath, corev1.TLSCertKey))) + args = append(args, fmt.Sprintf("--key=%s", path.Join(util.ClusterClientTLSPath, corev1.TLSPrivateKeyKey))) } var restoreType string @@ -66,7 +66,7 @@ func (ro *Options) restoreData(restore *v1alpha1.Restore) error { } func constructBROptions(restore *v1alpha1.Restore) ([]string, error) { - args, err := util.ConstructBRGlobalOptionsForRestore(restore) + args, err := backupUtil.ConstructBRGlobalOptionsForRestore(restore) if err != nil { return nil, err } diff --git a/docs/api-references/docs.html b/docs/api-references/docs.html index 41c5be1b109..286be033d2c 100644 --- a/docs/api-references/docs.html +++ b/docs/api-references/docs.html @@ -1546,13 +1546,17 @@

BRConfig -enableTLSClient
+tlsCluster
-bool + +TLSCluster + -

Whether enable TLS in TiDBCluster

+(Optional) +

Whether enable the TLS connection between TiDB server components +Optional: Defaults to nil

@@ -6652,6 +6656,7 @@

TLSCluster

(Appears on: +BRConfig, TidbClusterSpec)

diff --git a/hack/lib.sh b/hack/lib.sh index 3330541ad6f..3e46051598d 100644 --- a/hack/lib.sh +++ b/hack/lib.sh @@ -154,7 +154,7 @@ function hack::wait_for_success() { } # -# Concatenates the elements with an separator between them. +# Concatenates the elements with a separator between them. # # Usage: hack::join ',' a b c # diff --git a/hack/run-e2e.sh b/hack/run-e2e.sh index a2be877399f..b83846cb9ec 100755 --- a/hack/run-e2e.sh +++ b/hack/run-e2e.sh @@ -28,6 +28,7 @@ GCP_PROJECT=${GCP_PROJECT:-} GCP_REGION=${GCP_REGION:-} GCP_ZONE=${GCP_ZONE:-} GCP_CREDENTIALS=${GCP_CREDENTIALS:-} +GCP_SDK=${GCP_SDK:-/google-cloud-sdk} IMAGE_TAG=${IMAGE_TAG:-} SKIP_IMAGE_LOAD=${SKIP_IMAGE_LOAD:-} TIDB_OPERATOR_IMAGE=${TIDB_OPERATOR_IMAGE:-localhost:5000/pingcap/tidb-operator:latest} @@ -370,6 +371,15 @@ elif [ "$PROVIDER" == "gke" ]; then -v ${GCP_CREDENTIALS}:${GCP_CREDENTIALS} --env GOOGLE_APPLICATION_CREDENTIALS=${GCP_CREDENTIALS} ) + # google-cloud-sdk is very large, we didn't pack it into our e2e image. + # instead, we use the sdk installed in CI image. + if [ ! -e "${GCP_SDK}/bin/gcloud" ]; then + echo "error: ${GCP_SDK} is not google cloud sdk, please install it here or specify correct path via GCP_SDK env" + exit 1 + fi + docker_args+=( + -v ${GCP_SDK}:/google-cloud-sdk + ) else e2e_args+=( --provider=${PROVIDER} diff --git a/manifests/crd.yaml b/manifests/crd.yaml index d69a3a1d498..0caffeaec26 100644 --- a/manifests/crd.yaml +++ b/manifests/crd.yaml @@ -6766,9 +6766,6 @@ spec: db: description: DB is the specific DB which will be backed-up or restored type: string - enableTLSClient: - description: Whether enable TLS in TiDBCluster - type: boolean logLevel: description: LogLevel is the log level type: string @@ -6796,6 +6793,7 @@ spec: description: TimeAgo is the history version of the backup task, e.g. 1m, 1h type: string + tlsCluster: {} required: - cluster type: object @@ -7610,9 +7608,6 @@ spec: db: description: DB is the specific DB which will be backed-up or restored type: string - enableTLSClient: - description: Whether enable TLS in TiDBCluster - type: boolean logLevel: description: LogLevel is the log level type: string @@ -7640,6 +7635,7 @@ spec: description: TimeAgo is the history version of the backup task, e.g. 1m, 1h type: string + tlsCluster: {} required: - cluster type: object @@ -8497,9 +8493,6 @@ spec: description: DB is the specific DB which will be backed-up or restored type: string - enableTLSClient: - description: Whether enable TLS in TiDBCluster - type: boolean logLevel: description: LogLevel is the log level type: string @@ -8527,6 +8520,7 @@ spec: description: TimeAgo is the history version of the backup task, e.g. 1m, 1h type: string + tlsCluster: {} required: - cluster type: object diff --git a/pkg/apis/pingcap/v1alpha1/openapi_generated.go b/pkg/apis/pingcap/v1alpha1/openapi_generated.go index 077d8ddad4f..ae26ed35e2e 100644 --- a/pkg/apis/pingcap/v1alpha1/openapi_generated.go +++ b/pkg/apis/pingcap/v1alpha1/openapi_generated.go @@ -375,11 +375,10 @@ func schema_pkg_apis_pingcap_v1alpha1_BRConfig(ref common.ReferenceCallback) com Description: "BRConfig contains config for BR", Type: []string{"object"}, Properties: map[string]spec.Schema{ - "enableTLSClient": { + "tlsCluster": { SchemaProps: spec.SchemaProps{ - Description: "Whether enable TLS in TiDBCluster", - Type: []string{"boolean"}, - Format: "", + Description: "Whether enable the TLS connection between TiDB server components Optional: Defaults to nil", + Ref: ref("github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TLSCluster"), }, }, "cluster": { @@ -470,6 +469,8 @@ func schema_pkg_apis_pingcap_v1alpha1_BRConfig(ref common.ReferenceCallback) com Required: []string{"cluster"}, }, }, + Dependencies: []string{ + "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1.TLSCluster"}, } } diff --git a/pkg/apis/pingcap/v1alpha1/types.go b/pkg/apis/pingcap/v1alpha1/types.go index ee2f3813a84..499bb65e303 100644 --- a/pkg/apis/pingcap/v1alpha1/types.go +++ b/pkg/apis/pingcap/v1alpha1/types.go @@ -819,8 +819,10 @@ type BackupSpec struct { // +k8s:openapi-gen=true // BRConfig contains config for BR type BRConfig struct { - // Whether enable TLS in TiDBCluster - EnableTLSClient bool `json:"enableTLSClient,omitempty"` + // Whether enable the TLS connection between TiDB server components + // Optional: Defaults to nil + // +optional + TLSCluster *TLSCluster `json:"tlsCluster,omitempty"` // ClusterName of backup/restore cluster Cluster string `json:"cluster"` // Namespace of backup/restore cluster diff --git a/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go index e8ed1ac39cb..88e7aed5d1a 100644 --- a/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/pingcap/v1alpha1/zz_generated.deepcopy.go @@ -28,6 +28,11 @@ import ( // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BRConfig) DeepCopyInto(out *BRConfig) { *out = *in + if in.TLSCluster != nil { + in, out := &in.TLSCluster, &out.TLSCluster + *out = new(TLSCluster) + **out = **in + } if in.Concurrency != nil { in, out := &in.Concurrency, &out.Concurrency *out = new(uint32) diff --git a/pkg/backup/backup/backup_manager.go b/pkg/backup/backup/backup_manager.go index cf8d94e5f54..fa162fba9cf 100644 --- a/pkg/backup/backup/backup_manager.go +++ b/pkg/backup/backup/backup_manager.go @@ -22,6 +22,7 @@ import ( backuputil "github.com/pingcap/tidb-operator/pkg/backup/util" "github.com/pingcap/tidb-operator/pkg/controller" "github.com/pingcap/tidb-operator/pkg/label" + "github.com/pingcap/tidb-operator/pkg/util" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -276,14 +277,17 @@ func (bm *backupManager) makeBackupJob(backup *v1alpha1.Backup) (*batchv1.Job, s backupLabel := label.NewBackup().Instance(backup.GetInstanceName()).BackupJob().Backup(name) volumeMounts := []corev1.VolumeMount{} volumes := []corev1.Volume{} - if backup.Spec.BR.EnableTLSClient { + if backup.Spec.BR.TLSCluster != nil && backup.Spec.BR.TLSCluster.Enabled { volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: "br-tls", ReadOnly: true, MountPath: constants.BRCertPath, + Name: "cluster-client-tls", + ReadOnly: true, + MountPath: util.ClusterClientTLSPath, }) volumes = append(volumes, corev1.Volume{ - Name: "br-tls", VolumeSource: corev1.VolumeSource{ + Name: "cluster-client-tls", + VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: fmt.Sprintf("%s-client", controller.PDMemberName(backup.Spec.BR.Cluster)), + SecretName: util.ClusterClientTLSSecretName(backup.Spec.BR.Cluster), }, }, }) diff --git a/pkg/backup/restore/restore_manager.go b/pkg/backup/restore/restore_manager.go index cdbcec11e8c..1a38b7489fb 100644 --- a/pkg/backup/restore/restore_manager.go +++ b/pkg/backup/restore/restore_manager.go @@ -23,6 +23,7 @@ import ( listers "github.com/pingcap/tidb-operator/pkg/client/listers/pingcap/v1alpha1" "github.com/pingcap/tidb-operator/pkg/controller" "github.com/pingcap/tidb-operator/pkg/label" + "github.com/pingcap/tidb-operator/pkg/util" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" @@ -260,14 +261,17 @@ func (rm *restoreManager) makeRestoreJob(restore *v1alpha1.Restore) (*batchv1.Jo restoreLabel := label.NewBackup().Instance(restore.GetInstanceName()).RestoreJob().Restore(name) volumeMounts := []corev1.VolumeMount{} volumes := []corev1.Volume{} - if restore.Spec.BR.EnableTLSClient { + if restore.Spec.BR.TLSCluster != nil && restore.Spec.BR.TLSCluster.Enabled { volumeMounts = append(volumeMounts, corev1.VolumeMount{ - Name: "br-tls", ReadOnly: true, MountPath: constants.BRCertPath, + Name: "cluster-client-tls", + ReadOnly: true, + MountPath: util.ClusterClientTLSPath, }) volumes = append(volumes, corev1.Volume{ - Name: "br-tls", VolumeSource: corev1.VolumeSource{ + Name: "cluster-client-tls", + VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ - SecretName: fmt.Sprintf("%s-client", controller.PDMemberName(restore.Spec.BR.Cluster)), + SecretName: util.ClusterClientTLSSecretName(restore.Spec.BR.Cluster), }, }, }) diff --git a/tests/e2e/e2e.go b/tests/e2e/e2e.go index a7058225aa9..35831e83150 100644 --- a/tests/e2e/e2e.go +++ b/tests/e2e/e2e.go @@ -274,6 +274,11 @@ func RunE2ETests(t *testing.T) { gomega.RegisterFailHandler(e2elog.Fail) + // Disable serial and stability tests by default unless they are explicitly requested. + if config.GinkgoConfig.FocusString == "" && config.GinkgoConfig.SkipString == "" { + config.GinkgoConfig.SkipString = `\[Stability\]|\[Serial\]` + } + // Run tests through the Ginkgo runner with output to console + JUnit for Jenkins var r []ginkgo.Reporter if framework.TestContext.ReportDir != "" { diff --git a/tests/e2e/tidbcluster/serial.go b/tests/e2e/tidbcluster/serial.go index 7bfffd8fa0b..f476791b54b 100644 --- a/tests/e2e/tidbcluster/serial.go +++ b/tests/e2e/tidbcluster/serial.go @@ -62,6 +62,7 @@ func mustToString(set sets.Int32) string { return string(b) } +// Serial specs describe tests which cannot run in parallel. var _ = ginkgo.Describe("[tidb-operator][Serial]", func() { f := framework.NewDefaultFramework("serial") @@ -490,8 +491,8 @@ var _ = ginkgo.Describe("[tidb-operator][Serial]", func() { }) framework.ExpectNoError(err) - ginkgo.By("Make sure pods are not affected") - err = utilpod.WaitForPodsAreNotAffected(c, podListBeforeUpgrade.Items, time.Minute*3) + ginkgo.By("Make sure pods are not changed") + err = utilpod.WaitForPodsAreChanged(c, podListBeforeUpgrade.Items, time.Minute*3) framework.ExpectEqual(err, wait.ErrWaitTimeout, "Pods was not affeteced after the operator is upgraded") }) diff --git a/tests/e2e/tidbcluster/stability.go b/tests/e2e/tidbcluster/stability.go new file mode 100644 index 00000000000..88d2d8fb051 --- /dev/null +++ b/tests/e2e/tidbcluster/stability.go @@ -0,0 +1,189 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package tidbcluster + +import ( + "context" + "fmt" + _ "net/http/pprof" + "time" + + "github.com/onsi/ginkgo" + asclientset "github.com/pingcap/advanced-statefulset/pkg/client/clientset/versioned" + "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned" + "github.com/pingcap/tidb-operator/pkg/label" + "github.com/pingcap/tidb-operator/pkg/scheme" + "github.com/pingcap/tidb-operator/tests" + e2econfig "github.com/pingcap/tidb-operator/tests/e2e/config" + utilimage "github.com/pingcap/tidb-operator/tests/e2e/util/image" + utilpod "github.com/pingcap/tidb-operator/tests/e2e/util/pod" + "github.com/pingcap/tidb-operator/tests/e2e/util/portforward" + utiltidb "github.com/pingcap/tidb-operator/tests/e2e/util/tidb" + "github.com/pingcap/tidb-operator/tests/pkg/fixture" + v1 "k8s.io/api/core/v1" + apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" + aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" + "k8s.io/kubernetes/test/e2e/framework" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// Stability specs describe tests which involve disruptive operations, e.g. +// stop kubelet, kill nodes, empty pd/tikv data. +// Like serial tests, they cannot run in parallel too. +var _ = ginkgo.Describe("[tidb-operator][Stability]", func() { + f := framework.NewDefaultFramework("stability") + + var ns string + var c clientset.Interface + var cli versioned.Interface + var asCli asclientset.Interface + var aggrCli aggregatorclient.Interface + var apiExtCli apiextensionsclientset.Interface + var cfg *tests.Config + var config *restclient.Config + var fw portforward.PortForward + var fwCancel context.CancelFunc + + ginkgo.BeforeEach(func() { + ns = f.Namespace.Name + c = f.ClientSet + var err error + config, err = framework.LoadConfig() + framework.ExpectNoError(err, "failed to load config") + cli, err = versioned.NewForConfig(config) + framework.ExpectNoError(err, "failed to create clientset") + asCli, err = asclientset.NewForConfig(config) + framework.ExpectNoError(err, "failed to create clientset") + aggrCli, err = aggregatorclient.NewForConfig(config) + framework.ExpectNoError(err, "failed to create clientset") + apiExtCli, err = apiextensionsclientset.NewForConfig(config) + framework.ExpectNoError(err, "failed to create clientset") + clientRawConfig, err := e2econfig.LoadClientRawConfig() + framework.ExpectNoError(err, "failed to load raw config") + ctx, cancel := context.WithCancel(context.Background()) + fw, err = portforward.NewPortForwarder(ctx, e2econfig.NewSimpleRESTClientGetter(clientRawConfig)) + framework.ExpectNoError(err, "failed to create port forwarder") + fwCancel = cancel + cfg = e2econfig.TestConfig + }) + + ginkgo.AfterEach(func() { + if fwCancel != nil { + fwCancel() + } + }) + + // TODO generate more contexts for different operator values + ginkgo.Context("operator with default values", func() { + var ocfg *tests.OperatorConfig + var oa tests.OperatorActions + var genericCli client.Client + + ginkgo.BeforeEach(func() { + ocfg = &tests.OperatorConfig{ + Namespace: "pingcap", + ReleaseName: "operator", + Image: cfg.OperatorImage, + Tag: cfg.OperatorTag, + LogLevel: "4", + TestMode: true, + } + oa = tests.NewOperatorActions(cli, c, asCli, aggrCli, apiExtCli, tests.DefaultPollInterval, ocfg, e2econfig.TestConfig, nil, fw, f) + ginkgo.By("Installing CRDs") + oa.CleanCRDOrDie() + oa.InstallCRDOrDie(ocfg) + ginkgo.By("Installing tidb-operator") + oa.CleanOperatorOrDie(ocfg) + oa.DeployOperatorOrDie(ocfg) + var err error + genericCli, err = client.New(config, client.Options{Scheme: scheme.Scheme}) + framework.ExpectNoError(err, "failed to create clientset") + }) + + ginkgo.AfterEach(func() { + ginkgo.By("Uninstall tidb-operator") + oa.CleanOperatorOrDie(ocfg) + ginkgo.By("Uninstalling CRDs") + oa.CleanCRDOrDie() + }) + + testCases := []struct { + name string + fn func() + }{ + { + name: "tidb-operator does not exist", + fn: func() { + ginkgo.By("Uninstall tidb-operator") + oa.CleanOperatorOrDie(ocfg) + }, + }, + } + + for _, test := range testCases { + ginkgo.It("tidb cluster should not be affected while "+test.name, func() { + clusterName := "test" + tc := fixture.GetTidbCluster(ns, clusterName, utilimage.TiDBV3Version) + err := genericCli.Create(context.TODO(), tc) + framework.ExpectNoError(err) + err = oa.WaitForTidbClusterReady(tc, 30*time.Minute, 15*time.Second) + framework.ExpectNoError(err) + + test.fn() + + ginkgo.By("Check tidb cluster is not affected") + listOptions := metav1.ListOptions{ + LabelSelector: labels.SelectorFromSet(label.New().Instance(clusterName).Labels()).String(), + } + podList, err := c.CoreV1().Pods(ns).List(listOptions) + framework.ExpectNoError(err) + err = wait.PollImmediate(time.Second*30, time.Minute*5, func() (bool, error) { + var ok bool + var err error + framework.Logf("check whether pods of cluster %q are changed", clusterName) + ok, err = utilpod.PodsAreChanged(c, podList.Items)() + if ok || err != nil { + // pod changed or some error happened + return true, err + } + framework.Logf("check whether pods of cluster %q are running", clusterName) + newPodList, err := c.CoreV1().Pods(ns).List(listOptions) + if err != nil { + return false, err + } + for _, pod := range newPodList.Items { + if pod.Status.Phase != v1.PodRunning { + return false, fmt.Errorf("pod %s/%s is not running", pod.Namespace, pod.Name) + } + } + framework.Logf("check whehter tidb cluster %q is connectable", clusterName) + ok, err = utiltidb.TiDBIsConnectable(fw, ns, clusterName, "root", "")() + if !ok || err != nil { + // not connectable or some error happened + return true, err + } + return false, nil + }) + framework.ExpectEqual(err, wait.ErrWaitTimeout, "TiDB cluster is not affeteced") + }) + } + + }) + +}) diff --git a/tests/e2e/util/pod/pod.go b/tests/e2e/util/pod/pod.go index b0a95ab8b25..f84abd0ce76 100644 --- a/tests/e2e/util/pod/pod.go +++ b/tests/e2e/util/pod/pod.go @@ -26,10 +26,9 @@ import ( testutils "k8s.io/kubernetes/test/utils" ) -// WaitForPodsAreNotAffected waits for given pods are not affected. -// It returns wait.ErrWaitTimeout if the given pods are not affected in specified timeout. -func WaitForPodsAreNotAffected(c kubernetes.Interface, pods []v1.Pod, timeout time.Duration) error { - return wait.PollImmediate(time.Second*5, timeout, func() (bool, error) { +// PodsAreChanged checks the given pods are changed or not (recreate, update). +func PodsAreChanged(c kubernetes.Interface, pods []v1.Pod) wait.ConditionFunc { + return func() (bool, error) { for _, pod := range pods { podNew, err := c.CoreV1().Pods(pod.Namespace).Get(pod.Name, metav1.GetOptions{}) if err != nil { @@ -46,5 +45,11 @@ func WaitForPodsAreNotAffected(c kubernetes.Interface, pods []v1.Pod, timeout ti } } return false, nil - }) + } +} + +// WaitForPodsAreChanged waits for given pods are changed. +// It returns wait.ErrWaitTimeout if the given pods are not changed in specified timeout. +func WaitForPodsAreChanged(c kubernetes.Interface, pods []v1.Pod, timeout time.Duration) error { + return wait.PollImmediate(time.Second*5, timeout, PodsAreChanged(c, pods)) } diff --git a/tests/e2e/util/tidb/tidb.go b/tests/e2e/util/tidb/tidb.go new file mode 100644 index 00000000000..aba1656dadb --- /dev/null +++ b/tests/e2e/util/tidb/tidb.go @@ -0,0 +1,57 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package tidb + +import ( + "context" + "database/sql" + "fmt" + + // To register MySQL driver + _ "github.com/go-sql-driver/mysql" + "github.com/pingcap/tidb-operator/pkg/controller" + "github.com/pingcap/tidb-operator/tests/e2e/util/portforward" + "k8s.io/apimachinery/pkg/util/wait" +) + +var dummyCancel = func() {} + +// GetTiDBDSN returns a DSN to use +func GetTiDBDSN(fw portforward.PortForward, ns, tc, user, password, database string) (string, context.CancelFunc, error) { + localHost, localPort, cancel, err := portforward.ForwardOnePort(fw, ns, fmt.Sprintf("svc/%s", controller.TiDBMemberName(tc)), 4000) + if err != nil { + return "", dummyCancel, err + } + return fmt.Sprintf("%s:%s@(%s:%d)/%s?charset=utf8", user, password, localHost, localPort, database), cancel, nil +} + +// TiDBIsConnectable checks whether the tidb cluster is connectable. +func TiDBIsConnectable(fw portforward.PortForward, ns, tc, user, password string) wait.ConditionFunc { + return func() (bool, error) { + var db *sql.DB + dsn, cancel, err := GetTiDBDSN(fw, ns, tc, "root", password, "test") + if err != nil { + return false, err + } + defer cancel() + if db, err = sql.Open("mysql", dsn); err != nil { + return false, err + } + defer db.Close() + if err := db.Ping(); err != nil { + return false, err + } + return true, nil + } +} diff --git a/tests/images/e2e/Dockerfile b/tests/images/e2e/Dockerfile index ccce271038c..2d78d69f51d 100644 --- a/tests/images/e2e/Dockerfile +++ b/tests/images/e2e/Dockerfile @@ -28,3 +28,6 @@ ADD bin/e2e.test /usr/local/bin/ ADD bin/webhook /usr/local/bin/ ADD bin/blockwriter /usr/local/bin/ ADD bin/apiserver /usr/local/bin/ + +ADD entrypoint.sh /usr/local/bin +ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] diff --git a/tests/images/e2e/entrypoint.sh b/tests/images/e2e/entrypoint.sh new file mode 100755 index 00000000000..820c594a794 --- /dev/null +++ b/tests/images/e2e/entrypoint.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -e + +# Add default command if no command provided or the first argument is an +# option. +if [ $# -lt 1 -o "${1:0:1}" = '-' ]; then + set -- /usr/local/bin/ginkgo "$@" +fi + +# If google-cloud-sdk is detected, install it. +if [ -d /google-cloud-sdk ]; then + source /google-cloud-sdk/path.bash.inc + export CLOUDSDK_CORE_DISABLE_PROMPTS=1 +fi + +exec "$@"