Merge pull request #11919 from YoyinZyc/downgrade-policy

[Etcd downgrade] Apply downgrade policy to server
etcd-io · Jun 24, 2020 · ae59425 · ae59425
2 parents beb5614 + 2541b0b
commit ae59425
Show file tree

Hide file tree

Showing 9 changed files with 434 additions and 36 deletions.
diff --git a/etcdserver/api/capability.go b/etcdserver/api/capability.go
@@ -17,6 +17,7 @@ package api
 import (
 	"sync"
 
+	"go.etcd.io/etcd/v3/etcdserver/api/membership"
 	"go.etcd.io/etcd/v3/version"
 	"go.uber.org/zap"
 
@@ -62,7 +63,7 @@ func UpdateCapability(lg *zap.Logger, v *semver.Version) {
 		return
 	}
 	enableMapMu.Lock()
-	if curVersion != nil && !curVersion.LessThan(*v) {
+	if curVersion != nil && !membership.IsValidVersionChange(v, curVersion) {
 		enableMapMu.Unlock()
 		return
 	}

diff --git a/etcdserver/api/membership/cluster.go b/etcdserver/api/membership/cluster.go
@@ -63,14 +63,6 @@ type RaftCluster struct {
 	downgradeInfo *DowngradeInfo
 }
 
-type DowngradeInfo struct {
-	// TargetVersion is the target downgrade version, if the cluster is not under downgrading,
-	// the targetVersion will be an empty string
-	TargetVersion string `json:"target-version"`
-	// Enabled indicates whether the cluster is enabled to downgrade
-	Enabled bool `json:"enabled"`
-}
-
 // ConfigChangeContext represents a context for confChange.
 type ConfigChangeContext struct {
 	Member
@@ -263,7 +255,14 @@ func (c *RaftCluster) Recover(onSet func(*zap.Logger, *semver.Version)) {
 		c.version = clusterVersionFromStore(c.lg, c.v2store)
 	}
 
-	mustDetectDowngrade(c.lg, c.version)
+	if c.be != nil {
+		c.downgradeInfo = downgradeInfoFromBackend(c.lg, c.be)
+	}
+	d := &DowngradeInfo{Enabled: false}
+	if c.downgradeInfo != nil {
+		d = &DowngradeInfo{Enabled: c.downgradeInfo.Enabled, TargetVersion: c.downgradeInfo.TargetVersion}
+	}
+	mustDetectDowngrade(c.lg, c.version, d)
 	onSet(c.lg, c.version)
 
 	for _, m := range c.members {
@@ -530,7 +529,7 @@ func (c *RaftCluster) SetVersion(ver *semver.Version, onSet func(*zap.Logger, *s
 	}
 	oldVer := c.version
 	c.version = ver
-	mustDetectDowngrade(c.lg, c.version)
+	mustDetectDowngrade(c.lg, c.version, c.downgradeInfo)
 	if c.v2store != nil {
 		mustSaveClusterVersionToStore(c.lg, c.v2store, ver)
 	}
@@ -768,17 +767,20 @@ func ValidateClusterAndAssignIDs(lg *zap.Logger, local *RaftCluster, existing *R
 	return nil
 }
 
-func mustDetectDowngrade(lg *zap.Logger, cv *semver.Version) {
-	lv := semver.Must(semver.NewVersion(version.Version))
-	// only keep major.minor version for comparison against cluster version
+// IsValidVersionChange checks the two scenario when version is valid to change:
+// 1. Downgrade: cluster version is 1 minor version higher than local version,
+// cluster version should change.
+// 2. Cluster start: when not all members version are available, cluster version
+// is set to MinVersion(3.0), when all members are at higher version, cluster version
+// is lower than local version, cluster version should change
+func IsValidVersionChange(cv *semver.Version, lv *semver.Version) bool {
+	cv = &semver.Version{Major: cv.Major, Minor: cv.Minor}
 	lv = &semver.Version{Major: lv.Major, Minor: lv.Minor}
-	if cv != nil && lv.LessThan(*cv) {
-		lg.Fatal(
-			"invalid downgrade; server version is lower than determined cluster version",
-			zap.String("current-server-version", version.Version),
-			zap.String("determined-cluster-version", version.Cluster(cv.String())),
-		)
+
+	if isValidDowngrade(cv, lv) || (cv.Major == lv.Major && cv.LessThan(*lv)) {
+		return true
 	}
+	return false
 }
 
 // IsLocalMemberLearner returns if the local member is raft learner

diff --git a/etcdserver/api/membership/cluster_test.go b/etcdserver/api/membership/cluster_test.go
@@ -17,6 +17,7 @@ package membership
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/coreos/go-semver/semver"
 	"path"
 	"reflect"
 	"testing"
@@ -944,3 +945,75 @@ func TestIsReadyToPromoteMember(t *testing.T) {
 		}
 	}
 }
+
+func TestIsVersionChangable(t *testing.T) {
+	v0 := semver.Must(semver.NewVersion("2.4.0"))
+	v1 := semver.Must(semver.NewVersion("3.4.0"))
+	v2 := semver.Must(semver.NewVersion("3.5.0"))
+	v3 := semver.Must(semver.NewVersion("3.5.1"))
+	v4 := semver.Must(semver.NewVersion("3.6.0"))
+
+	tests := []struct {
+		name           string
+		currentVersion *semver.Version
+		localVersion   *semver.Version
+		expectedResult bool
+	}{
+		{
+			name:           "When local version is one minor lower than cluster version",
+			currentVersion: v2,
+			localVersion:   v1,
+			expectedResult: true,
+		},
+		{
+			name:           "When local version is one minor and one patch lower than cluster version",
+			currentVersion: v3,
+			localVersion:   v1,
+			expectedResult: true,
+		},
+		{
+			name:           "When local version is one minor higher than cluster version",
+			currentVersion: v1,
+			localVersion:   v2,
+			expectedResult: true,
+		},
+		{
+			name:           "When local version is two minor higher than cluster version",
+			currentVersion: v1,
+			localVersion:   v4,
+			expectedResult: true,
+		},
+		{
+			name:           "When local version is one major higher than cluster version",
+			currentVersion: v0,
+			localVersion:   v1,
+			expectedResult: false,
+		},
+		{
+			name:           "When local version is equal to cluster version",
+			currentVersion: v1,
+			localVersion:   v1,
+			expectedResult: false,
+		},
+		{
+			name:           "When local version is one patch higher than cluster version",
+			currentVersion: v2,
+			localVersion:   v3,
+			expectedResult: false,
+		},
+		{
+			name:           "When local version is two minor lower than cluster version",
+			currentVersion: v4,
+			localVersion:   v1,
+			expectedResult: false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if ret := IsValidVersionChange(tt.currentVersion, tt.localVersion); ret != tt.expectedResult {
+				t.Errorf("Expected %v; Got %v", tt.expectedResult, ret)
+			}
+		})
+	}
+}
diff --git a/etcdserver/api/membership/downgrade.go b/etcdserver/api/membership/downgrade.go
@@ -0,0 +1,80 @@
+// Copyright 2020 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package membership
+
+import (
+	"github.com/coreos/go-semver/semver"
+	"go.etcd.io/etcd/v3/version"
+	"go.uber.org/zap"
+)
+
+type DowngradeInfo struct {
+	// TargetVersion is the target downgrade version, if the cluster is not under downgrading,
+	// the targetVersion will be an empty string
+	TargetVersion string `json:"target-version"`
+	// Enabled indicates whether the cluster is enabled to downgrade
+	Enabled bool `json:"enabled"`
+}
+
+func (d *DowngradeInfo) GetTargetVersion() *semver.Version {
+	return semver.Must(semver.NewVersion(d.TargetVersion))
+}
+
+// isValidDowngrade verifies whether the cluster can be downgraded from verFrom to verTo
+func isValidDowngrade(verFrom *semver.Version, verTo *semver.Version) bool {
+	return verTo.Equal(*AllowedDowngradeVersion(verFrom))
+}
+
+// mustDetectDowngrade will detect unexpected downgrade when the local server is recovered.
+func mustDetectDowngrade(lg *zap.Logger, cv *semver.Version, d *DowngradeInfo) {
+	lv := semver.Must(semver.NewVersion(version.Version))
+	// only keep major.minor version for comparison against cluster version
+	lv = &semver.Version{Major: lv.Major, Minor: lv.Minor}
+
+	// if the cluster enables downgrade, check local version against downgrade target version.
+	if d != nil && d.Enabled && d.TargetVersion != "" {
+		if lv.Equal(*d.GetTargetVersion()) {
+			if cv != nil {
+				lg.Info(
+					"cluster is downgrading to target version",
+					zap.String("target-cluster-version", d.TargetVersion),
+					zap.String("determined-cluster-version", version.Cluster(cv.String())),
+					zap.String("current-server-version", version.Version),
+				)
+			}
+			return
+		}
+		lg.Fatal(
+			"invalid downgrade; server version is not allowed to join when downgrade is enabled",
+			zap.String("current-server-version", version.Version),
+			zap.String("target-cluster-version", d.TargetVersion),
+		)
+	}
+
+	// if the cluster disables downgrade, check local version against determined cluster version.
+	// the validation passes when local version is not less than cluster version
+	if cv != nil && lv.LessThan(*cv) {
+		lg.Fatal(
+			"invalid downgrade; server version is lower than determined cluster version",
+			zap.String("current-server-version", version.Version),
+			zap.String("determined-cluster-version", version.Cluster(cv.String())),
+		)
+	}
+}
+
+func AllowedDowngradeVersion(ver *semver.Version) *semver.Version {
+	// Todo: handle the case that downgrading from higher major version(e.g. downgrade from v4.0 to v3.x)
+	return &semver.Version{Major: ver.Major, Minor: ver.Minor - 1}
+}