Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[controller] Add logic to update StatefulSets #236

Merged
merged 7 commits into from
Sep 29, 2020
Merged

[controller] Add logic to update StatefulSets #236

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
1beef78
[controller] Add logic to update StatefulSets
jeromefroe Sep 25, 2020
f1db6ed
Ensure go mod is up to date
jeromefroe Sep 25, 2020
22dde4d
Use DeepDerivative to determine if we should perform an update
jeromefroe Sep 28, 2020
e2a5428
Fix typo
jeromefroe Sep 28, 2020
3ef8892
Change value of update annotation from true to enabled
jeromefroe Sep 28, 2020
e11bfdb
Fix tests
jeromefroe Sep 28, 2020
03df9e9
Update signature of updatedStatefulSet function
jeromefroe Sep 29, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ require (
golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975 // indirect
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e // indirect
golang.org/x/tools v0.0.0-20200102140908-9497f49d5709 // indirect
golang.org/x/tools v0.0.0-20200522201501-cb1345f3a375 // indirect
google.golang.org/appengine v1.6.2 // indirect
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
gopkg.in/go-ini/ini.v1 v1.51.1 // indirect
Expand Down
12 changes: 9 additions & 3 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -623,6 +623,7 @@ github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17
github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
go.mongodb.org/mongo-driver v1.0.3/go.mod h1:u7ryQJ+DOzQmeO7zB6MHyr8jkEQvC8vH7qLUO4lqsUM=
Expand Down Expand Up @@ -667,7 +668,8 @@ golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTk
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0=
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
golang.org/x/mod v0.2.0 h1:KU7oHjnv3XNWfa5COkzUifxZmxp1TyI7ImMXqFxLwvQ=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20170915142106-8351a756f30f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
Expand All @@ -692,6 +694,7 @@ golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k
golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI=
golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k=
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
Expand All @@ -705,6 +708,8 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20171026204733-164713f0dfce/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
Expand Down Expand Up @@ -766,8 +771,9 @@ golang.org/x/tools v0.0.0-20190719005602-e377ae9d6386/go.mod h1:jcCCGcm9btYwXyDq
golang.org/x/tools v0.0.0-20190729092621-ff9f1409240a/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
golang.org/x/tools v0.0.0-20190909030654-5b82db07426d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200102140908-9497f49d5709 h1:AfG1EmoRkFK24HWWLxSrRKNg2G+oA3JVOG8GJsHWypQ=
golang.org/x/tools v0.0.0-20200102140908-9497f49d5709/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200522201501-cb1345f3a375 h1:SjQ2+AKWgZLc1xej6WSzL+Dfs5Uyd5xcZH1mGC411IA=
golang.org/x/tools v0.0.0-20200522201501-cb1345f3a375/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
Expand Down
95 changes: 95 additions & 0 deletions pkg/controller/controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
clientset "github.com/m3db/m3db-operator/pkg/client/clientset/versioned"
samplescheme "github.com/m3db/m3db-operator/pkg/client/clientset/versioned/scheme"
clusterlisters "github.com/m3db/m3db-operator/pkg/client/listers/m3dboperator/v1alpha1"
"github.com/m3db/m3db-operator/pkg/k8sops/annotations"
"github.com/m3db/m3db-operator/pkg/k8sops/labels"
"github.com/m3db/m3db-operator/pkg/k8sops/m3db"
"github.com/m3db/m3db-operator/pkg/k8sops/podidentity"
Expand Down Expand Up @@ -477,6 +478,34 @@ func (c *M3DBController) handleClusterUpdate(cluster *myspec.M3DBCluster) error
}
}

// Now that we know the cluster is healthy, iterate over each isolation group and check
// if it should be updated.
for i := 0; i < len(isoGroups); i++ {
name := m3db.StatefulSetName(cluster.Name, i)

// This StatefulSet is guaranteed to exist since if it didn't originally it would be
// created above when we first iterate over the isolation groups.
sts := childrenSetsByName[name]
update, err := shouldUpdateStatefulSet(cluster.Spec, sts)
if err != nil {
c.logger.Error(err.Error())
return err
}

if !update {
continue
}

_, err = c.kubeClient.AppsV1().StatefulSets(cluster.Namespace).Update(sts)
if err != nil {
c.logger.Error(err.Error())
return err
}

c.logger.Info("updated statefulset", zap.String("name", name))
return nil
}

if err := c.reconcileNamespaces(cluster); err != nil {
c.recorder.WarningEvent(cluster, eventer.ReasonFailedCreate, "failed to create namespace: %s", err)
c.logger.Error("error reconciling namespaces", zap.Error(err))
Expand Down Expand Up @@ -919,3 +948,69 @@ func validateIsolationGroups(cluster *myspec.M3DBCluster) error {
}
return nil
}

// shouldUpdateStatefulSet checks if a StatefulSet should be updated. If the Image or
// ConfigMapName fields in the ClusterSpec don't match what's in the StatefulSet then
// this function will update the provided StatefulSet to match the desired state and
// return true.
func shouldUpdateStatefulSet(spec myspec.ClusterSpec, sts *appsv1.StatefulSet) (bool, error) {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Image and config updates are definitely the most common case we want to cover, so from that perspective this looks good. There are definitely other cases where the cluster CRD changes and we want to update the statefulsets, and I'm wondering how to address that here. Two thoughts:

  1. Any idea what happens if we try to DeepEqual the generated spec and the current spec on the cluster? I'm guessing the Kubernetes API makes some sort of transformations that make that not totally possible, but I'm curious.

  2. Assuming DeepEqual doesn't work, what do you think about a second annotation that will trigger an Update regardless of whether image and config has changed?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've always had difficulties using DeepEqual in the past because the API server sets default (non-zero) values for so many fields. Based on this kubebuilder issue though, it seems that we could use DeepDerivative instead since it "is similar to DeepEqual except that unset fields in a1 are ignored ". That issue also mentions github.com/banzaicloud/k8s-objectmatcher as another alternative if DeepDerivative doesn't work for us. If neither package is sufficient though I'm not opposed to having a force-update annotation or something similar which will always trigger an update.

// The operator will only perform an update if the current StatefulSet has been
// annotated to indicate that it is okay to update it.
if val, ok := sts.Annotations[annotations.Update]; !ok || val != "true" {
return false, nil
}

var anyUpdates bool
update, err := shouldUpdateImage(spec.Image, sts)
if err != nil {
return false, err
}
anyUpdates = anyUpdates || update

update, err = shouldUpdateConfigMap(*spec.ConfigMapName, sts)
if err != nil {
return false, err
}
anyUpdates = anyUpdates || update

// If the StatefulSet will be updated, remove the update annotation.
if anyUpdates {
delete(sts.Annotations, annotations.Update)
}

return anyUpdates, nil
}

func shouldUpdateImage(image string, sts *appsv1.StatefulSet) (bool, error) {
// In StatefulSet's created by the operator the first container will run the M3DB node.
containers := sts.Spec.Template.Spec.Containers
if len(containers) == 0 {
return false, newInvalidStatefulSetError(sts.Name, "container")
}

var update bool
if containers[0].Image != image {
containers[0].Image = image
update = true
}
return update, nil
}

func shouldUpdateConfigMap(configMapName string, sts *appsv1.StatefulSet) (bool, error) {
// In StatefulSet's created by the operator this ConfigMap will be the M3DB config file.
volumes := sts.Spec.Template.Spec.Volumes
if len(volumes) == 0 {
return false, newInvalidStatefulSetError(sts.Name, "config map")
}

var update bool
if volumes[0].ConfigMap.Name != configMapName {
volumes[0].ConfigMap.Name = configMapName
update = true
}
return update, nil
}

func newInvalidStatefulSetError(name string, missing string) error {
return fmt.Errorf("StatefulSet %q is invalid: no %s in pod template", name, missing)
}
Loading