diff --git a/charts/tidb-operator/templates/controller-manager-rbac.yaml b/charts/tidb-operator/templates/controller-manager-rbac.yaml index 3f20c9a9cf..2d6b598f69 100644 --- a/charts/tidb-operator/templates/controller-manager-rbac.yaml +++ b/charts/tidb-operator/templates/controller-manager-rbac.yaml @@ -28,7 +28,7 @@ rules: - events verbs: ["*"] - apiGroups: [""] - resources: ["endpoints"] + resources: ["endpoints","configmaps"] verbs: ["create", "get", "list", "watch", "update"] - apiGroups: ["batch"] resources: ["jobs"] diff --git a/pkg/controller/configmap_control.go b/pkg/controller/configmap_control.go new file mode 100644 index 0000000000..a50f6421d2 --- /dev/null +++ b/pkg/controller/configmap_control.go @@ -0,0 +1,182 @@ +// Copyright 2019. PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "fmt" + "strings" + + "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" + corev1 "k8s.io/api/core/v1" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + coreinformers "k8s.io/client-go/informers/core/v1" + "k8s.io/client-go/kubernetes" + corelisters "k8s.io/client-go/listers/core/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/record" + "k8s.io/client-go/util/retry" + "k8s.io/klog" +) + +// ConfigMapControlInterface manages configmaps used by TiDB clusters +type ConfigMapControlInterface interface { + CreateConfigMap(*v1alpha1.TidbCluster, *corev1.ConfigMap) error + UpdateConfigMap(*v1alpha1.TidbCluster, *corev1.ConfigMap) (*corev1.ConfigMap, error) + DeleteConfigMap(*v1alpha1.TidbCluster, *corev1.ConfigMap) error +} + +type realConfigMapControl struct { + kubeCli kubernetes.Interface + cmLister corelisters.ConfigMapLister + recorder record.EventRecorder +} + +// NewRealSecretControl creates a new SecretControlInterface +func NewRealConfigMapControl( + kubeCli kubernetes.Interface, + cmLister corelisters.ConfigMapLister, + recorder record.EventRecorder, +) ConfigMapControlInterface { + return &realConfigMapControl{ + kubeCli: kubeCli, + cmLister: cmLister, + recorder: recorder, + } +} + +func (cc *realConfigMapControl) CreateConfigMap(tc *v1alpha1.TidbCluster, cm *corev1.ConfigMap) error { + _, err := cc.kubeCli.CoreV1().ConfigMaps(tc.Namespace).Create(cm) + cc.recordConfigMapEvent("create", tc, cm, err) + return err +} + +func (cc *realConfigMapControl) UpdateConfigMap(tc *v1alpha1.TidbCluster, cm *corev1.ConfigMap) (*corev1.ConfigMap, error) { + ns := tc.GetNamespace() + tcName := tc.GetName() + cmName := cm.GetName() + cmData := cm.Data + + var updatedCm *corev1.ConfigMap + err := retry.RetryOnConflict(retry.DefaultBackoff, func() error { + var updateErr error + updatedCm, updateErr = cc.kubeCli.CoreV1().ConfigMaps(ns).Update(cm) + if updateErr == nil { + klog.Infof("update ConfigMap: [%s/%s] successfully, TidbCluster: %s", ns, cmName, tcName) + return nil + } + + if updated, err := cc.cmLister.ConfigMaps(tc.Namespace).Get(cmName); err != nil { + utilruntime.HandleError(fmt.Errorf("error getting updated ConfigMap %s/%s from lister: %v", ns, cmName, err)) + } else { + cm = updated.DeepCopy() + cm.Data = cmData + } + + return updateErr + }) + cc.recordConfigMapEvent("update", tc, cm, err) + return updatedCm, err +} + +func (cc *realConfigMapControl) DeleteConfigMap(tc *v1alpha1.TidbCluster, cm *corev1.ConfigMap) error { + err := cc.kubeCli.CoreV1().ConfigMaps(tc.Namespace).Delete(cm.Name, nil) + cc.recordConfigMapEvent("delete", tc, cm, err) + return err +} + +func (cc *realConfigMapControl) recordConfigMapEvent(verb string, tc *v1alpha1.TidbCluster, cm *corev1.ConfigMap, err error) { + tcName := tc.GetName() + cmName := cm.GetName() + if err == nil { + reason := fmt.Sprintf("Successful%s", strings.Title(verb)) + msg := fmt.Sprintf("%s ConfigMap %s in TidbCluster %s successful", + strings.ToLower(verb), cmName, tcName) + cc.recorder.Event(tc, corev1.EventTypeNormal, reason, msg) + } else { + reason := fmt.Sprintf("Failed%s", strings.Title(verb)) + msg := fmt.Sprintf("%s ConfigMap %s in TidbCluster %s failed error: %s", + strings.ToLower(verb), cmName, tcName, err) + cc.recorder.Event(tc, corev1.EventTypeWarning, reason, msg) + } +} + +var _ ConfigMapControlInterface = &realConfigMapControl{} + +// NewFakeConfigMapControl returns a FakeConfigMapControl +func NewFakeConfigMapControl(cmInformer coreinformers.ConfigMapInformer) *FakeConfigMapControl { + return &FakeConfigMapControl{ + cmInformer.Lister(), + cmInformer.Informer().GetIndexer(), + RequestTracker{}, + RequestTracker{}, + RequestTracker{}, + } +} + +// FakeConfigMapControl is a fake ConfigMapControlInterface +type FakeConfigMapControl struct { + CmLister corelisters.ConfigMapLister + CmIndexer cache.Indexer + createConfigMapTracker RequestTracker + updateConfigMapTracker RequestTracker + deleteConfigMapTracker RequestTracker +} + +// SetCreateConfigMapError sets the error attributes of createConfigMapTracker +func (cc *FakeConfigMapControl) SetCreateConfigMapError(err error, after int) { + cc.createConfigMapTracker.SetError(err).SetAfter(after) +} + +// SetUpdateConfigMapError sets the error attributes of updateConfigMapTracker +func (cc *FakeConfigMapControl) SetUpdateConfigMapError(err error, after int) { + cc.updateConfigMapTracker.SetError(err).SetAfter(after) +} + +// SetDeleteConfigMapError sets the error attributes of deleteConfigMapTracker +func (cc *FakeConfigMapControl) SetDeleteConfigMapError(err error, after int) { + cc.deleteConfigMapTracker.SetError(err).SetAfter(after) +} + +// CreateConfigMap adds the ConfigMap to ConfigMapIndexer +func (cc *FakeConfigMapControl) CreateConfigMap(_ *v1alpha1.TidbCluster, cm *corev1.ConfigMap) error { + defer cc.createConfigMapTracker.Inc() + if cc.createConfigMapTracker.ErrorReady() { + defer cc.createConfigMapTracker.Reset() + return cc.createConfigMapTracker.GetError() + } + + err := cc.CmIndexer.Add(cm) + if err != nil { + return err + } + return nil +} + +// UpdateConfigMap updates the ConfigMap of CmIndexer +func (cc *FakeConfigMapControl) UpdateConfigMap(_ *v1alpha1.TidbCluster, cm *corev1.ConfigMap) (*corev1.ConfigMap, error) { + defer cc.updateConfigMapTracker.Inc() + if cc.updateConfigMapTracker.ErrorReady() { + defer cc.updateConfigMapTracker.Reset() + return nil, cc.updateConfigMapTracker.GetError() + } + + return cm, cc.CmIndexer.Update(cm) +} + +// DeleteConfigMap deletes the ConfigMap of CmIndexer +func (cc *FakeConfigMapControl) DeleteConfigMap(_ *v1alpha1.TidbCluster, _ *corev1.ConfigMap) error { + return nil +} + +var _ ConfigMapControlInterface = &FakeConfigMapControl{} diff --git a/pkg/controller/configmap_control_test.go b/pkg/controller/configmap_control_test.go new file mode 100644 index 0000000000..00d4b975d2 --- /dev/null +++ b/pkg/controller/configmap_control_test.go @@ -0,0 +1,164 @@ +// Copyright 2019. PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "errors" + "testing" + + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + corelisters "k8s.io/client-go/listers/core/v1" + core "k8s.io/client-go/testing" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/tools/record" +) + +func TestConfigMapControlCreatesConfigMaps(t *testing.T) { + g := NewGomegaWithT(t) + recorder := record.NewFakeRecorder(10) + tc := newTidbCluster() + cm := newConfigMap() + fakeClient := &fake.Clientset{} + control := NewRealConfigMapControl(fakeClient, nil, recorder) + fakeClient.AddReactor("create", "configmaps", func(action core.Action) (bool, runtime.Object, error) { + create := action.(core.CreateAction) + return true, create.GetObject(), nil + }) + err := control.CreateConfigMap(tc, cm) + g.Expect(err).To(Succeed()) + + events := collectEvents(recorder.Events) + g.Expect(events).To(HaveLen(1)) + g.Expect(events[0]).To(ContainSubstring(corev1.EventTypeNormal)) +} + +func TestConfigMapControlCreatesConfigMapFailed(t *testing.T) { + g := NewGomegaWithT(t) + recorder := record.NewFakeRecorder(10) + tc := newTidbCluster() + cm := newConfigMap() + fakeClient := &fake.Clientset{} + control := NewRealConfigMapControl(fakeClient, nil, recorder) + fakeClient.AddReactor("create", "configmaps", func(action core.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewInternalError(errors.New("API server down")) + }) + err := control.CreateConfigMap(tc, cm) + g.Expect(err).To(HaveOccurred()) + + events := collectEvents(recorder.Events) + g.Expect(events).To(HaveLen(1)) + g.Expect(events[0]).To(ContainSubstring(corev1.EventTypeWarning)) +} + +func TestConfigMapControlUpdateConfigMap(t *testing.T) { + g := NewGomegaWithT(t) + recorder := record.NewFakeRecorder(10) + tc := newTidbCluster() + cm := newConfigMap() + cm.Data["file"] = "test" + fakeClient := &fake.Clientset{} + control := NewRealConfigMapControl(fakeClient, nil, recorder) + fakeClient.AddReactor("update", "configmaps", func(action core.Action) (bool, runtime.Object, error) { + update := action.(core.UpdateAction) + return true, update.GetObject(), nil + }) + updatecm, err := control.UpdateConfigMap(tc, cm) + g.Expect(err).To(Succeed()) + g.Expect(updatecm.Data["file"]).To(Equal("test")) + + events := collectEvents(recorder.Events) + g.Expect(events).To(HaveLen(1)) + g.Expect(events[0]).To(ContainSubstring(corev1.EventTypeNormal)) +} + +func TestConfigMapControlUpdateConfigMapConflictSuccess(t *testing.T) { + g := NewGomegaWithT(t) + recorder := record.NewFakeRecorder(10) + tc := newTidbCluster() + cm := newConfigMap() + cm.Data["file"] = "test" + fakeClient := &fake.Clientset{} + indexer := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + oldcm := newConfigMap() + oldcm.Data["file"] = "test2" + err := indexer.Add(oldcm) + g.Expect(err).To(Succeed()) + cmLister := corelisters.NewConfigMapLister(indexer) + control := NewRealConfigMapControl(fakeClient, cmLister, recorder) + conflict := false + fakeClient.AddReactor("update", "configmaps", func(action core.Action) (bool, runtime.Object, error) { + update := action.(core.UpdateAction) + if !conflict { + conflict = true + return true, oldcm, apierrors.NewConflict(action.GetResource().GroupResource(), cm.Name, errors.New("conflict")) + } + return true, update.GetObject(), nil + }) + updatecm, err := control.UpdateConfigMap(tc, cm) + g.Expect(err).To(Succeed()) + g.Expect(updatecm.Data["file"]).To(Equal("test")) + + events := collectEvents(recorder.Events) + g.Expect(events).To(HaveLen(1)) + g.Expect(events[0]).To(ContainSubstring(corev1.EventTypeNormal)) +} + +func TestConfigMapControlDeleteConfigMap(t *testing.T) { + g := NewGomegaWithT(t) + recorder := record.NewFakeRecorder(10) + tc := newTidbCluster() + cm := newConfigMap() + fakeClient := &fake.Clientset{} + control := NewRealConfigMapControl(fakeClient, nil, recorder) + fakeClient.AddReactor("delete", "configmaps", func(action core.Action) (bool, runtime.Object, error) { + return true, nil, nil + }) + err := control.DeleteConfigMap(tc, cm) + g.Expect(err).To(Succeed()) + events := collectEvents(recorder.Events) + g.Expect(events).To(HaveLen(1)) + g.Expect(events[0]).To(ContainSubstring(corev1.EventTypeNormal)) +} + +func TestConfigMapControlDeleteConfigMapFailed(t *testing.T) { + g := NewGomegaWithT(t) + recorder := record.NewFakeRecorder(10) + tc := newTidbCluster() + cm := newConfigMap() + fakeClient := &fake.Clientset{} + control := NewRealConfigMapControl(fakeClient, nil, recorder) + fakeClient.AddReactor("delete", "configmaps", func(action core.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewInternalError(errors.New("API server down")) + }) + err := control.DeleteConfigMap(tc, cm) + g.Expect(err).To(HaveOccurred()) + + events := collectEvents(recorder.Events) + g.Expect(events).To(HaveLen(1)) + g.Expect(events[0]).To(ContainSubstring(corev1.EventTypeWarning)) +} + +func newConfigMap() *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Data: map[string]string{}, + } +}