add AdvancedStatefulSet feature

pingcap · Nov 14, 2019 · b2cbef1 · b2cbef1
1 parent 3692c17
commit b2cbef1
Show file tree

Hide file tree

Showing 21 changed files with 411 additions and 57 deletions.
diff --git a/charts/tidb-operator/templates/advanced-statefulset-deployment.yaml b/charts/tidb-operator/templates/advanced-statefulset-deployment.yaml
@@ -0,0 +1,55 @@
+{{- if .Values.advancedStatefulset.create }}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: advanced-statefulset-controller
+  labels:
+    app.kubernetes.io/name: {{ template "chart.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: advanced-statefulset-controller
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
+spec:
+  replicas: {{ .Values.advancedStatefulset.replicas }}
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: {{ template "chart.name" . }}
+      app.kubernetes.io/instance: {{ .Release.Name }}
+      app.kubernetes.io/component: advanced-statefulset-controller
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: {{ template "chart.name" . }}
+        app.kubernetes.io/instance: {{ .Release.Name }}
+        app.kubernetes.io/component: advanced-statefulset-controller
+    spec:
+      serviceAccountName: {{ .Values.advancedStatefulset.serviceAccount }}
+      containers:
+      - name: advanced-statefulset-controller
+        image: {{ .Values.advancedStatefulset.image }}
+        imagePullPolicy: {{ .Values.advancedStatefulset.imagePullPolicy | default "IfNotPresent" }}
+        args:
+        - --v={{ .Values.advancedStatefulset.logLevel }}
+        - --leader-elect
+        - --leader-elect-resource-name=advanced-statefulset-controller
+        - --leader-elect-resource-namespace=$(POD_NAMESPACE)
+        env:
+        - name: POD_NAMESPACE
+          valueFrom:
+            fieldRef:
+              fieldPath: metadata.namespace
+        resources:
+{{ toYaml .Values.advancedStatefulset.resources | indent 12 }}
+    {{- with .Values.advancedStatefulset.nodeSelector }}
+      nodeSelector:
+{{ toYaml . | indent 8 }}
+    {{- end }}
+    {{- with .Values.advancedStatefulset.affinity }}
+      affinity:
+{{ toYaml . | indent 8 }}
+    {{- end }}
+    {{- with .Values.advancedStatefulset.tolerations }}
+      tolerations:
+{{ toYaml . | indent 8 }}
+    {{- end }}
+{{- end }}
diff --git a/charts/tidb-operator/templates/advanced-statefulset-rbac.yaml b/charts/tidb-operator/templates/advanced-statefulset-rbac.yaml
@@ -0,0 +1,96 @@
+{{- if .Values.advancedStatefulset.create }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ .Values.advancedStatefulset.serviceAccount }}
+  labels:
+    app.kubernetes.io/name: {{ template "chart.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: advanced-statefulset-controller
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: advanced-statefulset-controller
+  labels:
+    app.kubernetes.io/name: {{ template "chart.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: advanced-statefulset-controller
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
+rules:
+- apiGroups:
+  - apps.pingcap.com
+  resources:
+  - '*'
+  verbs:
+  - '*'
+- apiGroups:
+  - 'apps'
+  resources:
+  - 'controllerrevisions'
+  verbs:
+  - '*'
+- apiGroups:
+  - ''
+  resources:
+  - 'pods'
+  - 'persistentvolumeclaims'
+  - 'persistentvolumes'
+  verbs:
+  - '*'
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: advanced-statefulset-controller
+  labels:
+    app.kubernetes.io/name: {{ template "chart.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: advanced-statefulset-controller
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: advanced-statefulset-controller
+subjects:
+- kind: ServiceAccount
+  name: advanced-statefulset-controller
+  namespace: {{ .Release.Namespace }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: advanced-statefulset-controller
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ template "chart.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: advanced-statefulset-controller
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
+rules:
+- apiGroups:
+  - ''
+  resources:
+  - 'endpoints'
+  verbs:
+  - '*'
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: advanced-statefulset-controller
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: advanced-statefulset-controller
+subjects:
+- kind: ServiceAccount
+  name: advanced-statefulset-controller
+  namespace: {{ .Release.Namespace }}
+{{- end }}
diff --git a/charts/tidb-operator/templates/controller-manager-deployment.yaml b/charts/tidb-operator/templates/controller-manager-deployment.yaml
@@ -49,6 +49,9 @@ spec:
           {{- if .Values.testMode }}
           - -test-mode={{ .Values.testMode }}
           {{- end}}
+          {{- if .Values.features }}
+          - -features={{ join "," .Values.features }}
+          {{- end }}
         env:
           - name: NAMESPACE
             valueFrom:

diff --git a/charts/tidb-operator/templates/controller-manager-rbac.yaml b/charts/tidb-operator/templates/controller-manager-rbac.yaml
@@ -56,6 +56,14 @@ rules:
   - restores
   - restores/finalizers
   verbs: ["*"]
+{{- if .Values.features | has "AdvancedStatefulSet=true" }}
+- apiGroups:
+  - apps.pingcap.com
+  resources:
+  - statefulsets
+  verbs:
+  - '*'
+{{- end }}
 {{- end }}
 - apiGroups: [""]
   resources: ["nodes"]
@@ -129,6 +137,14 @@ rules:
   - restores
   - restores/finalizers
   verbs: ["*"]
+{{- if .Values.features | has "AdvancedStatefulSet=true" }}
+- apiGroups:
+  - apps.pingcap.com
+  resources:
+  - statefulsets
+  verbs:
+  - '*'
+{{- end }}
 ---
 kind: RoleBinding
 apiVersion: rbac.authorization.k8s.io/v1beta1

diff --git a/charts/tidb-operator/templates/scheduler-deployment.yaml b/charts/tidb-operator/templates/scheduler-deployment.yaml
@@ -35,8 +35,8 @@ spec:
           - /usr/local/bin/tidb-scheduler
           - -v={{ .Values.scheduler.logLevel }}
           - -port=10262
-        {{- if .Values.scheduler.features }}
-          - -features={{ join "," .Values.scheduler.features }}
+        {{- if .Values.features }}
+          - -features={{ join "," .Values.features }}
         {{- end }}
       {{- if and (ne .Values.timezone "UTC") (ne .Values.timezone "") }}
         env:

diff --git a/charts/tidb-operator/values.yaml b/charts/tidb-operator/values.yaml
@@ -21,6 +21,23 @@ defaultStorageClassName: local-storage
 # tidbBackupManagerImage: pingcap/tidb-backup-manager:latest
 # defaultBackupStorageClassName: local-storage
 
+#
+# Enable or disable tidb-operator features:
+#
+#   StableScheduling (default: true)
+#     Enable stable scheduling of tidb servers.
+#
+#   AdvancedStatefulSet (default: false)
+#     If enabled, tidb-operator will use AdvancedStatefulSet to manage pods
+#     instead of Kubernetes StatefulSet.
+#     It's ok to turn it on if this feature is not enabled. However it's not ok
+#     to turn it off when the tidb-operator already uses AdvancedStatefulSet to
+#     manage pods. This is in alpha phase.
+#
+features: []
+# - AdvancedStatefulSet=false
+# - StableScheduling=true
+
 controllerManager:
   # With rbac.create=false, the user is responsible for creating this account
   # With rbac.create=true, this service account will be created
@@ -66,8 +83,6 @@ scheduler:
   logLevel: 2
   replicas: 1
   schedulerName: tidb-scheduler
-  # features:
-  # - StableScheduling=true
   resources:
     limits:
       cpu: 250m
@@ -127,3 +142,36 @@ apiserver:
   #   value: tidb-operator
   #   effect: "NoSchedule"
 
+# When AdvancedStatefulSet feature is enabled, you must install
+# AdvancedStatefulSet controller.
+# Note that AdvancedStatefulSet CRD must be installed manually via the following
+# command:
+#   kubectl apply -f manifests/advanced-statefulset-crd.v1beta1.yaml
+advancedStatefulset:
+  create: false
+  image: pingcap/advanced-statefulset:v0.1.0
+  imagePullPolicy: IfNotPresent
+  serviceAccount: advanced-statefulset-controller
+  logLevel: 2
+  replicas: 1
+  resources:
+    limits:
+      cpu: 500m
+      memory: 300Mi
+    requests:
+      cpu: 200m
+      memory: 50Mi
+  ## affinity defines pod scheduling rules,affinity default settings is empty.
+  ## please read the affinity document before set your scheduling rule:
+  ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
+  affinity: {}
+  ## nodeSelector ensure pods only assigning to nodes which have each of the indicated key-value pairs as labels
+  ## ref:https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
+  nodeSelector: {}
+  ## Tolerations are applied to pods, and allow pods to schedule onto nodes with matching taints.
+  ## refer to https://kubernetes.io/docs/concepts/configuration/taint-and-toleration
+  tolerations: []
+  # - key: node-role
+  #   operator: Equal
+  #   value: tidb-operator
+  #   effect: "NoSchedule"
diff --git a/cmd/admission-controller/main.go b/cmd/admission-controller/main.go
@@ -20,9 +20,12 @@ import (
 	"os/signal"
 	"syscall"
 
+	"github.com/pingcap/advanced-statefulset/pkg/apis/apps/v1alpha1/helper"
+	asclientset "github.com/pingcap/advanced-statefulset/pkg/client/clientset/versioned"
 	"github.com/pingcap/tidb-operator/pkg/client/clientset/versioned"
 	informers "github.com/pingcap/tidb-operator/pkg/client/informers/externalversions"
 	"github.com/pingcap/tidb-operator/pkg/controller"
+	"github.com/pingcap/tidb-operator/pkg/features"
 	"github.com/pingcap/tidb-operator/pkg/version"
 	"github.com/pingcap/tidb-operator/pkg/webhook"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -44,6 +47,7 @@ func init() {
 	flag.BoolVar(&printVersion, "version", false, "Show version and quit")
 	flag.StringVar(&certFile, "tlsCertFile", "/etc/webhook/certs/cert.pem", "File containing the x509 Certificate for HTTPS.")
 	flag.StringVar(&keyFile, "tlsKeyFile", "/etc/webhook/certs/key.pem", "File containing the x509 private key to --tlsCertFile.")
+	features.DefaultFeatureGate.AddFlag(flag.CommandLine)
 	flag.Parse()
 }
 
@@ -68,10 +72,22 @@ func main() {
 		glog.Fatalf("failed to create Clientset: %v", err)
 	}
 
-	kubeCli, err := kubernetes.NewForConfig(cfg)
+	var kubeCli kubernetes.Interface
+	kubeCli, err = kubernetes.NewForConfig(cfg)
 	if err != nil {
 		glog.Fatalf("failed to get kubernetes Clientset: %v", err)
 	}
+	asCli, err := asclientset.NewForConfig(cfg)
+	if err != nil {
+		glog.Fatalf("failed to get advanced-statefulset Clientset: %v", err)
+	}
+
+	if features.DefaultFeatureGate.Enabled(features.AdvancedStatefulSet) {
+		// If AdvancedStatefulSet is enabled, we hijack the Kubernetes client to use
+		// AdvancedStatefulSet.
+		kubeCli = helper.NewHijackClient(kubeCli, asCli)
+	}
+
 	informerFactory := informers.NewSharedInformerFactory(cli, controller.ResyncDuration)
 	kubeInformerFactory := kubeinformers.NewSharedInformerFactory(kubeCli, controller.ResyncDuration)
 

diff --git a/cmd/controller-manager/main.go b/cmd/controller-manager/main.go
@@ -21,13 +21,16 @@ import (
 	"os"
 	"time"
 
+	"github.com/pingcap/advanced-statefulset/pkg/apis/apps/v1alpha1/helper"
+	asclientset "github.com/pingcap/advanced-statefulset/pkg/client/clientset/versioned"
 	"github.com/pingcap/tidb-operator/pkg/client/clientset/versioned"
 	informers "github.com/pingcap/tidb-operator/pkg/client/informers/externalversions"
 	"github.com/pingcap/tidb-operator/pkg/controller"
 	"github.com/pingcap/tidb-operator/pkg/controller/backup"
 	"github.com/pingcap/tidb-operator/pkg/controller/backupschedule"
 	"github.com/pingcap/tidb-operator/pkg/controller/restore"
 	"github.com/pingcap/tidb-operator/pkg/controller/tidbcluster"
+	"github.com/pingcap/tidb-operator/pkg/features"
 	"github.com/pingcap/tidb-operator/pkg/version"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -68,6 +71,7 @@ func init() {
 	flag.DurationVar(&controller.ResyncDuration, "resync-duration", time.Duration(30*time.Second), "Resync time of informer")
 	flag.BoolVar(&controller.TestMode, "test-mode", false, "whether tidb-operator run in test mode")
 	flag.StringVar(&controller.TidbBackupManagerImage, "tidb-backup-manager-image", "pingcap/tidb-backup-manager:latest", "The image of backup manager tool")
+	features.DefaultFeatureGate.AddFlag(flag.CommandLine)
 
 	flag.Parse()
 }
@@ -101,10 +105,21 @@ func main() {
 	if err != nil {
 		glog.Fatalf("failed to create Clientset: %v", err)
 	}
-	kubeCli, err := kubernetes.NewForConfig(cfg)
+	var kubeCli kubernetes.Interface
+	kubeCli, err = kubernetes.NewForConfig(cfg)
 	if err != nil {
 		glog.Fatalf("failed to get kubernetes Clientset: %v", err)
 	}
+	asCli, err := asclientset.NewForConfig(cfg)
+	if err != nil {
+		glog.Fatalf("failed to get advanced-statefulset Clientset: %v", err)
+	}
+
+	if features.DefaultFeatureGate.Enabled(features.AdvancedStatefulSet) {
+		// If AdvancedStatefulSet is enabled, we hijack the Kubernetes client to use
+		// AdvancedStatefulSet.
+		kubeCli = helper.NewHijackClient(kubeCli, asCli)
+	}
 
 	var informerFactory informers.SharedInformerFactory
 	var kubeInformerFactory kubeinformers.SharedInformerFactory

diff --git a/go.mod b/go.mod
@@ -45,6 +45,7 @@ require (
 	github.com/onsi/gomega v1.5.0
 	github.com/opentracing/opentracing-go v1.1.0 // indirect
 	github.com/pierrec/lz4 v2.0.5+incompatible // indirect
+	github.com/pingcap/advanced-statefulset v0.1.0
 	github.com/pingcap/errors v0.11.0
 	github.com/pingcap/kvproto v0.0.0-20190516013202-4cf58ad90b6c
 	github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd

diff --git a/go.sum b/go.sum
@@ -569,6 +569,8 @@ github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+v
 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
 github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
+github.com/pingcap/advanced-statefulset v0.1.0 h1:rjUx6Tc90YwlRrgIXWTvjr8D97fx0GP4E49dObmBqj0=
+github.com/pingcap/advanced-statefulset v0.1.0/go.mod h1:rg2p1v6AGsKhvEZi6Sm0YNYJCmdXdZZhQ6Sviei7Ivs=
 github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 h1:USx2/E1bX46VG32FIw034Au6seQ2fY9NEILmNh/UlQg=
 github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ=
 github.com/pingcap/errors v0.11.0 h1:DCJQB8jrHbQ1VVlMFIrbj2ApScNNotVmkSNplu2yUt4=
@@ -816,6 +818,8 @@ golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 h1:xQwXv67TxFo9nC1GJFyab5eq/5B590r6RlnL/G8Sz7w=
+golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20170824195420-5d2fd3ccab98/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181003024731-2f84ea8ef872/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=