From 3709f20dc151190dfcb9b0466fa615775a1a7a76 Mon Sep 17 00:00:00 2001 From: Rob Skillington Date: Wed, 19 Jun 2019 14:17:09 +1000 Subject: [PATCH 1/4] Add ability to specify and set base annotations for operator created resources --- pkg/apis/m3dboperator/v1alpha1/cluster.go | 4 ++ pkg/k8sops/annotations/annotations.go | 53 ++++++++++++++++++++++ pkg/k8sops/annotations/annotations_test.go | 53 ++++++++++++++++++++++ pkg/k8sops/generators.go | 9 ++-- pkg/k8sops/generators_test.go | 5 ++ pkg/k8sops/statefulset.go | 9 +++- 6 files changed, 128 insertions(+), 5 deletions(-) create mode 100644 pkg/k8sops/annotations/annotations.go create mode 100644 pkg/k8sops/annotations/annotations_test.go diff --git a/pkg/apis/m3dboperator/v1alpha1/cluster.go b/pkg/apis/m3dboperator/v1alpha1/cluster.go index aafd2df4..febf390c 100644 --- a/pkg/apis/m3dboperator/v1alpha1/cluster.go +++ b/pkg/apis/m3dboperator/v1alpha1/cluster.go @@ -234,6 +234,10 @@ type ClusterSpec struct { // the cluster. // TODO(schallert): design doc on labeling scheme. Labels map[string]string `json:"labels,omitempty"` + // Annotations sets the base annotations that will be applied to resources created by + // the cluster. + Annotations map[string]string `json:"annotations,omitempty" yaml:"annotations"` + // Tolerations sets the tolerations that will be applied to all M3DB pods. // +optional Tolerations []corev1.Toleration `json:"tolerations,omitempty"` diff --git a/pkg/k8sops/annotations/annotations.go b/pkg/k8sops/annotations/annotations.go new file mode 100644 index 00000000..dc8964d9 --- /dev/null +++ b/pkg/k8sops/annotations/annotations.go @@ -0,0 +1,53 @@ +// Copyright (c) 2019 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +// Package annotations defines constants and helpers for annotations used +// throughout the m3db operator. +package annotations + +import ( + myspec "github.com/m3db/m3db-operator/pkg/apis/m3dboperator/v1alpha1" + "github.com/m3db/m3db-operator/pkg/k8sops/labels" +) + +const ( + // App is the label used to identify the application. + App = labels.App + // AppM3DB is the value for "App" common to all operator-created clusters. + AppM3DB = labels.AppM3DB + // Cluster is the label identifying what m3db cluster an object is a part of. + Cluster = labels.Cluster +) + +// BaseAnnotations returns the base annotations we apply to all objects +// created by the operator for a given cluster. +func BaseAnnotations(cluster *myspec.M3DBCluster) map[string]string { + base := map[string]string{ + App: AppM3DB, + Cluster: cluster.Name, + } + if configured := cluster.Spec.Annotations; configured != nil { + for k, v := range configured { + base[k] = v + } + } + + return base +} diff --git a/pkg/k8sops/annotations/annotations_test.go b/pkg/k8sops/annotations/annotations_test.go new file mode 100644 index 00000000..a479fb3e --- /dev/null +++ b/pkg/k8sops/annotations/annotations_test.go @@ -0,0 +1,53 @@ +// Copyright (c) 2018 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package annotations + +import ( + "testing" + + myspec "github.com/m3db/m3db-operator/pkg/apis/m3dboperator/v1alpha1" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/stretchr/testify/assert" +) + +func TestGenerateBaseAnnotations(t *testing.T) { + cluster := &myspec.M3DBCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cluster-foo", + }, + } + + annotations := BaseAnnotations(cluster) + expAnnotations := map[string]string{ + "operator.m3db.io/app": "m3db", + "operator.m3db.io/cluster": "cluster-foo", + } + + assert.Equal(t, expAnnotations, annotations) + + cluster.Spec.Annotations = map[string]string{"foo": "bar"} + annotations = BaseAnnotations(cluster) + expAnnotations["foo"] = "bar" + + assert.Equal(t, expAnnotations, annotations) +} diff --git a/pkg/k8sops/generators.go b/pkg/k8sops/generators.go index 9b16a76d..9fed698a 100644 --- a/pkg/k8sops/generators.go +++ b/pkg/k8sops/generators.go @@ -24,6 +24,8 @@ import ( "errors" "fmt" + "github.com/m3db/m3db-operator/pkg/k8sops/annotations" + m3dboperator "github.com/m3db/m3db-operator/pkg/apis/m3dboperator" myspec "github.com/m3db/m3db-operator/pkg/apis/m3dboperator/v1alpha1" "github.com/m3db/m3db-operator/pkg/k8sops/labels" @@ -195,11 +197,12 @@ func GenerateM3DBService(cluster *myspec.M3DBCluster) (*v1.Service, error) { svcLabels := labels.BaseLabels(cluster) svcLabels[labels.Component] = labels.ComponentM3DBNode - + svcAnnotations := annotations.BaseAnnotations(cluster) return &v1.Service{ ObjectMeta: metav1.ObjectMeta{ - Name: HeadlessServiceName(cluster.Name), - Labels: svcLabels, + Name: HeadlessServiceName(cluster.Name), + Labels: svcLabels, + Annotations: svcAnnotations, }, Spec: v1.ServiceSpec{ Selector: svcLabels, diff --git a/pkg/k8sops/generators_test.go b/pkg/k8sops/generators_test.go index 35c40678..de75ab3e 100644 --- a/pkg/k8sops/generators_test.go +++ b/pkg/k8sops/generators_test.go @@ -25,6 +25,7 @@ import ( m3dboperator "github.com/m3db/m3db-operator/pkg/apis/m3dboperator" myspec "github.com/m3db/m3db-operator/pkg/apis/m3dboperator/v1alpha1" + "github.com/m3db/m3db-operator/pkg/k8sops/annotations" "github.com/m3db/m3db-operator/pkg/k8sops/labels" appsv1 "k8s.io/api/apps/v1" @@ -371,6 +372,10 @@ func TestGenerateM3DBService(t *testing.T) { ObjectMeta: metav1.ObjectMeta{ Name: "m3dbnode-cluster-a", Labels: baseLabels, + Annotations: map[string]string{ + annotations.App: annotations.AppM3DB, + annotations.Cluster: cluster.Name, + }, }, Spec: v1.ServiceSpec{ Selector: baseLabels, diff --git a/pkg/k8sops/statefulset.go b/pkg/k8sops/statefulset.go index 3e661636..7f0a73ea 100644 --- a/pkg/k8sops/statefulset.go +++ b/pkg/k8sops/statefulset.go @@ -24,6 +24,8 @@ import ( "errors" "fmt" + "github.com/m3db/m3db-operator/pkg/k8sops/annotations" + myspec "github.com/m3db/m3db-operator/pkg/apis/m3dboperator/v1alpha1" "github.com/m3db/m3db-operator/pkg/k8sops/labels" "github.com/m3db/m3db-operator/pkg/k8sops/podidentity" @@ -62,6 +64,8 @@ func NewBaseStatefulSet(ssName, isolationGroup string, cluster *myspec.M3DBClust objLabels[k] = v } + objAnnotations := annotations.BaseAnnotations(cluster) + // TODO(schallert): we're currently using the health of the coordinator for // liveness probes until https://github.com/m3db/m3/issues/996 is fixed. Move // to the dbnode's health endpoint once fixed. @@ -93,8 +97,9 @@ func NewBaseStatefulSet(ssName, isolationGroup string, cluster *myspec.M3DBClust return &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ - Name: ssName, - Labels: objLabels, + Name: ssName, + Labels: objLabels, + Annotations: objAnnotations, }, Spec: appsv1.StatefulSetSpec{ ServiceName: HeadlessServiceName(clusterName), From 96665f51d2c7bc5dee3ddaf461a3bf34f3c34e6f Mon Sep 17 00:00:00 2001 From: Rob Skillington Date: Wed, 19 Jun 2019 14:36:43 +1000 Subject: [PATCH 2/4] Update docs and arrange imports --- docs/api.md | 1 + pkg/k8sops/generators.go | 3 +-- pkg/k8sops/statefulset.go | 3 +-- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/api.md b/docs/api.md index 86ac47f1..b6be2e2c 100644 --- a/docs/api.md +++ b/docs/api.md @@ -53,6 +53,7 @@ ClusterSpec defines the desired state for a M3 cluster to be converge to. | podSecurityContext | PodSecurityContext allows the user to specify an optional security context for pods. | *corev1.PodSecurityContext | false | | securityContext | SecurityContext allows the user to specify a container-level security context. | *corev1.SecurityContext | false | | labels | Labels sets the base labels that will be applied to resources created by the cluster. // TODO(schallert): design doc on labeling scheme. | map[string]string | false | +| annotations | Annotations sets the base annotations that will be applied to resources created by the cluster. | map[string]string | false | | tolerations | Tolerations sets the tolerations that will be applied to all M3DB pods. | []corev1.Toleration | false | | priorityClassName | PriorityClassName sets the priority class for all M3DB pods. | string | false | diff --git a/pkg/k8sops/generators.go b/pkg/k8sops/generators.go index 9fed698a..edebd862 100644 --- a/pkg/k8sops/generators.go +++ b/pkg/k8sops/generators.go @@ -24,10 +24,9 @@ import ( "errors" "fmt" - "github.com/m3db/m3db-operator/pkg/k8sops/annotations" - m3dboperator "github.com/m3db/m3db-operator/pkg/apis/m3dboperator" myspec "github.com/m3db/m3db-operator/pkg/apis/m3dboperator/v1alpha1" + "github.com/m3db/m3db-operator/pkg/k8sops/annotations" "github.com/m3db/m3db-operator/pkg/k8sops/labels" appsv1 "k8s.io/api/apps/v1" diff --git a/pkg/k8sops/statefulset.go b/pkg/k8sops/statefulset.go index 7f0a73ea..e3d8679f 100644 --- a/pkg/k8sops/statefulset.go +++ b/pkg/k8sops/statefulset.go @@ -24,9 +24,8 @@ import ( "errors" "fmt" - "github.com/m3db/m3db-operator/pkg/k8sops/annotations" - myspec "github.com/m3db/m3db-operator/pkg/apis/m3dboperator/v1alpha1" + "github.com/m3db/m3db-operator/pkg/k8sops/annotations" "github.com/m3db/m3db-operator/pkg/k8sops/labels" "github.com/m3db/m3db-operator/pkg/k8sops/podidentity" From 3a7d2ebbaa53358e3abf023ccfd2f4c58322c48e Mon Sep 17 00:00:00 2001 From: Rob Skillington Date: Wed, 19 Jun 2019 16:38:23 +1000 Subject: [PATCH 3/4] Fix test --- pkg/k8sops/generators_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/k8sops/generators_test.go b/pkg/k8sops/generators_test.go index de75ab3e..f3b06b00 100644 --- a/pkg/k8sops/generators_test.go +++ b/pkg/k8sops/generators_test.go @@ -120,8 +120,9 @@ func TestGenerateStatefulSet(t *testing.T) { baseSS := &appsv1.StatefulSet{ ObjectMeta: metav1.ObjectMeta{ - Name: ssName, - Labels: labels, + Name: ssName, + Labels: labels, + Annotations: annotations.BaseAnnotations(fixture), OwnerReferences: []metav1.OwnerReference{ *metav1.NewControllerRef(fixture, schema.GroupVersionKind{ Group: myspec.SchemeGroupVersion.Group, From aab9e4e0f6e76e167f7882b857e34daf1610f3f0 Mon Sep 17 00:00:00 2001 From: Rob Skillington Date: Thu, 20 Jun 2019 09:59:21 +1000 Subject: [PATCH 4/4] Run kubernetes-gen --- .../m3dboperator/v1alpha1/openapi_generated.go | 15 +++++++++++++++ .../v1alpha1/zz_generated.deepcopy.go | 7 +++++++ 2 files changed, 22 insertions(+) diff --git a/pkg/apis/m3dboperator/v1alpha1/openapi_generated.go b/pkg/apis/m3dboperator/v1alpha1/openapi_generated.go index a08fb3d9..2eaf3ba1 100644 --- a/pkg/apis/m3dboperator/v1alpha1/openapi_generated.go +++ b/pkg/apis/m3dboperator/v1alpha1/openapi_generated.go @@ -459,6 +459,21 @@ func schema_pkg_apis_m3dboperator_v1alpha1_ClusterSpec(ref common.ReferenceCallb }, }, }, + "annotations": { + SchemaProps: spec.SchemaProps{ + Description: "Annotations sets the base annotations that will be applied to resources created by the cluster.", + Type: []string{"object"}, + AdditionalProperties: &spec.SchemaOrBool{ + Allows: true, + Schema: &spec.Schema{ + SchemaProps: spec.SchemaProps{ + Type: []string{"string"}, + Format: "", + }, + }, + }, + }, + }, "tolerations": { SchemaProps: spec.SchemaProps{ Description: "Tolerations sets the tolerations that will be applied to all M3DB pods.", diff --git a/pkg/apis/m3dboperator/v1alpha1/zz_generated.deepcopy.go b/pkg/apis/m3dboperator/v1alpha1/zz_generated.deepcopy.go index 6cc57816..da42fa4c 100644 --- a/pkg/apis/m3dboperator/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/apis/m3dboperator/v1alpha1/zz_generated.deepcopy.go @@ -100,6 +100,13 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { (*out)[key] = val } } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } if in.Tolerations != nil { in, out := &in.Tolerations, &out.Tolerations *out = make([]v1.Toleration, len(*in))