From 5d36a02b8018723b529301ff3e83bd6d67db5fce Mon Sep 17 00:00:00 2001 From: Brian McQueen Date: Mon, 13 Apr 2020 19:35:28 -0700 Subject: [PATCH 1/4] added initial support for PodMetaData, handling Annotations only --- pkg/apis/m3dboperator/v1alpha1/cluster.go | 4 +++ pkg/k8sops/annotations/annotations.go | 14 ++++++++++ pkg/k8sops/annotations/annotations_test.go | 31 ++++++++++++++++++++++ pkg/k8sops/m3db/generators_test.go | 28 +++++++++++++++++++ pkg/k8sops/m3db/statefulset.go | 2 +- 5 files changed, 78 insertions(+), 1 deletion(-) diff --git a/pkg/apis/m3dboperator/v1alpha1/cluster.go b/pkg/apis/m3dboperator/v1alpha1/cluster.go index 66b044f5..ea16bc0b 100644 --- a/pkg/apis/m3dboperator/v1alpha1/cluster.go +++ b/pkg/apis/m3dboperator/v1alpha1/cluster.go @@ -295,6 +295,10 @@ type ClusterSpec struct { // If the InitContainers require any storage volumes // Provide the complete specification for the required Volumes here InitVolumes []corev1.Volume `json:"initVolumes,omitempty"` + + // PodMetadata is for any Metadata that is unique to the pods, and does + // not belong on any other objects, such as Prometheus scrape tags + PodMetadata metav1.ObjectMeta `json:"podMetadata,omitempty"` } // NodeAffinityTerm represents a node label and a set of label values, any of diff --git a/pkg/k8sops/annotations/annotations.go b/pkg/k8sops/annotations/annotations.go index dc8964d9..b6dc6f4b 100644 --- a/pkg/k8sops/annotations/annotations.go +++ b/pkg/k8sops/annotations/annotations.go @@ -51,3 +51,17 @@ func BaseAnnotations(cluster *myspec.M3DBCluster) map[string]string { return base } + +// PodAnnotations is for specifying annotations that are only to be +// applied to the pods such as prometheus scrape tags +func PodAnnotations(cluster *myspec.M3DBCluster) map[string]string { + base := BaseAnnotations(cluster) + for k := range cluster.Spec.PodMetadata.Annotations { + // accept any user-specified annotations if its safe to do so + if _, found := base[k]; !found { + base[k] = cluster.Spec.PodMetadata.Annotations[k] + } + } + + return base +} diff --git a/pkg/k8sops/annotations/annotations_test.go b/pkg/k8sops/annotations/annotations_test.go index a479fb3e..ffa8923c 100644 --- a/pkg/k8sops/annotations/annotations_test.go +++ b/pkg/k8sops/annotations/annotations_test.go @@ -51,3 +51,34 @@ func TestGenerateBaseAnnotations(t *testing.T) { assert.Equal(t, expAnnotations, annotations) } + +func TestGeneratePodAnnotations(t *testing.T) { + cluster := &myspec.M3DBCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster-foo", + }, + Spec: myspec.ClusterSpec{ + PodMetadata: metav1.ObjectMeta{ + Annotations: map[string]string{ + "pod-annotation": "some-annotation", + }, + }, + }, + } + + annotations := PodAnnotations(cluster) + expAnnotations := map[string]string{ + "operator.m3db.io/app": "m3db", + "operator.m3db.io/cluster": "cluster-foo", + "pod-annotation": "some-annotation", + } + + assert.Equal(t, expAnnotations, annotations) + + cluster.Spec.Annotations = map[string]string{"foo": "bar"} + annotations = PodAnnotations(cluster) + expAnnotations["foo"] = "bar" + expAnnotations["pod-annotation"] = "some-annotation" + + assert.Equal(t, expAnnotations, annotations) +} diff --git a/pkg/k8sops/m3db/generators_test.go b/pkg/k8sops/m3db/generators_test.go index 771652db..1ef1d649 100644 --- a/pkg/k8sops/m3db/generators_test.go +++ b/pkg/k8sops/m3db/generators_test.go @@ -488,6 +488,28 @@ func TestGenerateStatefulSet(t *testing.T) { diff, _ := messagediff.PrettyDiff(ss, newSS) t.Log(diff) } + + // Test init containers + fixture = getFixture("testM3DBCluster.yaml", t) + fixture.Spec.InitContainers = []v1.Container{ + { + Name: "init0", + }, + } + + ss = baseSS.DeepCopy() + ss.Spec.Template.Spec.InitContainers = []v1.Container{ + { + Name: "init0", + }, + } + newSS, err = GenerateStatefulSet(fixture, isolationGroup, *instanceAmount) + assert.NoError(t, err) + assert.NotNil(t, newSS) + if !assert.Equal(t, ss, newSS) { + diff, _ := messagediff.PrettyDiff(ss, newSS) + t.Log(diff) + } } func TestGenerateM3DBService(t *testing.T) { @@ -526,6 +548,12 @@ func TestGenerateM3DBService(t *testing.T) { } assert.Equal(t, expSvc, svc) + + cluster.Spec.ExternalCoordinatorSelector = map[string]string{"foo": "bar"} + expSvc.Spec.Selector = map[string]string{"foo": "bar"} + svc, err = GenerateCoordinatorService(cluster) + assert.NoError(t, err) + assert.Equal(t, expSvc, svc) } func TestGenerateCoordinatorService(t *testing.T) { diff --git a/pkg/k8sops/m3db/statefulset.go b/pkg/k8sops/m3db/statefulset.go index 68a1f999..63c6725e 100644 --- a/pkg/k8sops/m3db/statefulset.go +++ b/pkg/k8sops/m3db/statefulset.go @@ -63,7 +63,7 @@ func NewBaseStatefulSet(ssName, isolationGroup string, cluster *myspec.M3DBClust objLabels[k] = v } - objAnnotations := annotations.BaseAnnotations(cluster) + objAnnotations := annotations.PodAnnotations(cluster) // TODO(schallert): we're currently using the health of the coordinator for // liveness probes until https://github.com/m3db/m3/issues/996 is fixed. Move From 76d8a67abe81517ee047595244671389b9d4ca69 Mon Sep 17 00:00:00 2001 From: Brian McQueen Date: Mon, 13 Apr 2020 19:46:35 -0700 Subject: [PATCH 2/4] fix a broken test --- pkg/k8sops/m3db/generators_test.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pkg/k8sops/m3db/generators_test.go b/pkg/k8sops/m3db/generators_test.go index 1ef1d649..d2368341 100644 --- a/pkg/k8sops/m3db/generators_test.go +++ b/pkg/k8sops/m3db/generators_test.go @@ -548,12 +548,6 @@ func TestGenerateM3DBService(t *testing.T) { } assert.Equal(t, expSvc, svc) - - cluster.Spec.ExternalCoordinatorSelector = map[string]string{"foo": "bar"} - expSvc.Spec.Selector = map[string]string{"foo": "bar"} - svc, err = GenerateCoordinatorService(cluster) - assert.NoError(t, err) - assert.Equal(t, expSvc, svc) } func TestGenerateCoordinatorService(t *testing.T) { From dbe486d9c58795d0b42d3eed195830196abbdb48 Mon Sep 17 00:00:00 2001 From: Brian McQueen Date: Mon, 13 Apr 2020 19:55:54 -0700 Subject: [PATCH 3/4] got rid of some lines that seem to have been copy/pasted --- pkg/k8sops/m3db/generators_test.go | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/pkg/k8sops/m3db/generators_test.go b/pkg/k8sops/m3db/generators_test.go index d2368341..771652db 100644 --- a/pkg/k8sops/m3db/generators_test.go +++ b/pkg/k8sops/m3db/generators_test.go @@ -488,28 +488,6 @@ func TestGenerateStatefulSet(t *testing.T) { diff, _ := messagediff.PrettyDiff(ss, newSS) t.Log(diff) } - - // Test init containers - fixture = getFixture("testM3DBCluster.yaml", t) - fixture.Spec.InitContainers = []v1.Container{ - { - Name: "init0", - }, - } - - ss = baseSS.DeepCopy() - ss.Spec.Template.Spec.InitContainers = []v1.Container{ - { - Name: "init0", - }, - } - newSS, err = GenerateStatefulSet(fixture, isolationGroup, *instanceAmount) - assert.NoError(t, err) - assert.NotNil(t, newSS) - if !assert.Equal(t, ss, newSS) { - diff, _ := messagediff.PrettyDiff(ss, newSS) - t.Log(diff) - } } func TestGenerateM3DBService(t *testing.T) { From ac05ef29daf571c984fe13cda1c318c269a01e79 Mon Sep 17 00:00:00 2001 From: Matt Schallert Date: Thu, 23 Apr 2020 00:20:21 -0400 Subject: [PATCH 4/4] gen docs --- docs/api.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/api.md b/docs/api.md index 08251968..6478fb27 100644 --- a/docs/api.md +++ b/docs/api.md @@ -65,6 +65,7 @@ ClusterSpec defines the desired state for a M3 cluster to be converge to. | externalCoordinatorSelector | Specify a \"controlling\" coordinator for the cluster It is expected that there is a separate standalone coordinator cluster It is externally managed - not managed by this operator It is expected to have a service endpoint Setup this db cluster, but do not assume a co-located coordinator Instead provide a selector here so we can point to a separate coordinator service Specify here the labels required for the selector | map[string]string | false | | initContainers | Custom setup for db nodes can be done via initContainers Provide the complete spec for the initContainer here If any storage volumes are needed in the initContainer see InitVolumes below | []corev1.Container | false | | initVolumes | If the InitContainers require any storage volumes Provide the complete specification for the required Volumes here | []corev1.Volume | false | +| podMetadata | PodMetadata is for any Metadata that is unique to the pods, and does not belong on any other objects, such as Prometheus scrape tags | [metav1.ObjectMeta](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.10/#objectmeta-v1-meta) | false | [Back to TOC](#table-of-contents)