-
Notifications
You must be signed in to change notification settings - Fork 825
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
GKE Autopilot: Add support for Extended Duration pods under a feature…
… gate (#3387) Introduces Extended Duration Pod support for Autopilot under feature gate "GKEAutopilotExtendedDurationPods". This feature has some rough edges with Agones on Autopilot 1.27, so I'm introducing this as a dev feature gate until we sort everything out. ED pods have the same UI as disabling Autopilot on GKE Standard [1]: annotate with `cluster-autoscaler.kubernetes.io/safe-to-evict=false` and your pod is protected from eviction. However, Autopilot handles node upgrades specially and just autoscales new nodes at new versions, so the concept of "safe to evict on upgrade" is unnecessary. Since we can support the concept and do nothing really special, do so: We pull out the "generic" implementation of SetEviction into cloudproduct/eviction and call that. [1] https://cloud.google.com/kubernetes-engine/docs/how-to/extended-duration-pods Towards #3366
- Loading branch information
Showing
8 changed files
with
245 additions
and
132 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,57 @@ | ||
| // Copyright 2023 Google LLC All Rights Reserved. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| // Package eviction implements a generic SetEviction interface for cloud products | ||
| package eviction | ||
|
|
||
| import ( | ||
| agonesv1 "agones.dev/agones/pkg/apis/agones/v1" | ||
| "github.com/pkg/errors" | ||
| corev1 "k8s.io/api/core/v1" | ||
| ) | ||
|
|
||
| // SetEviction sets disruptions controls on a Pod based on GameServer.Status.Eviction. | ||
| func SetEviction(eviction *agonesv1.Eviction, pod *corev1.Pod) error { | ||
| if eviction == nil { | ||
| return errors.New("No eviction value set. Should be the default value") | ||
| } | ||
| if _, exists := pod.ObjectMeta.Annotations[agonesv1.PodSafeToEvictAnnotation]; !exists { | ||
| switch eviction.Safe { | ||
| case agonesv1.EvictionSafeAlways: | ||
| pod.ObjectMeta.Annotations[agonesv1.PodSafeToEvictAnnotation] = agonesv1.True | ||
| case agonesv1.EvictionSafeOnUpgrade, agonesv1.EvictionSafeNever: | ||
| // For EvictionSafeOnUpgrade and EvictionSafeNever, we block Cluster Autoscaler | ||
| // (on Autopilot, this enables Extended Duration pods, which is equivalent). | ||
| pod.ObjectMeta.Annotations[agonesv1.PodSafeToEvictAnnotation] = agonesv1.False | ||
| default: | ||
| return errors.Errorf("unknown eviction.safe value %q", string(eviction.Safe)) | ||
| } | ||
| } | ||
| if _, exists := pod.ObjectMeta.Labels[agonesv1.SafeToEvictLabel]; !exists { | ||
| switch eviction.Safe { | ||
| case agonesv1.EvictionSafeAlways, agonesv1.EvictionSafeOnUpgrade: | ||
| // For EvictionSafeAlways and EvictionSafeOnUpgrade, we use a label value | ||
| // that does not match the agones-gameserver-safe-to-evict-false PDB. But | ||
| // we go ahead and label it, in case someone wants to adopt custom logic | ||
| // for this group of game servers. | ||
| pod.ObjectMeta.Labels[agonesv1.SafeToEvictLabel] = agonesv1.True | ||
| case agonesv1.EvictionSafeNever: | ||
| // For EvictionSafeNever, match gones-gameserver-safe-to-evict-false PDB. | ||
| pod.ObjectMeta.Labels[agonesv1.SafeToEvictLabel] = agonesv1.False | ||
| default: | ||
| return errors.Errorf("unknown eviction.safe value %q", string(eviction.Safe)) | ||
| } | ||
| } | ||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,105 @@ | ||
| // Copyright 2023 Google LLC All Rights Reserved. | ||
| // | ||
| // Licensed under the Apache License, Version 2.0 (the "License"); | ||
| // you may not use this file except in compliance with the License. | ||
| // You may obtain a copy of the License at | ||
| // | ||
| // http://www.apache.org/licenses/LICENSE-2.0 | ||
| // | ||
| // Unless required by applicable law or agreed to in writing, software | ||
| // distributed under the License is distributed on an "AS IS" BASIS, | ||
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
| // See the License for the specific language governing permissions and | ||
| // limitations under the License. | ||
|
|
||
| package eviction | ||
|
|
||
| import ( | ||
| "testing" | ||
|
|
||
| agonesv1 "agones.dev/agones/pkg/apis/agones/v1" | ||
| "github.com/stretchr/testify/assert" | ||
| corev1 "k8s.io/api/core/v1" | ||
| metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
| ) | ||
|
|
||
| func TestSetEviction(t *testing.T) { | ||
| emptyPodAnd := func(f func(*corev1.Pod)) *corev1.Pod { | ||
| pod := &corev1.Pod{ | ||
| ObjectMeta: metav1.ObjectMeta{ | ||
| Annotations: map[string]string{}, | ||
| Labels: map[string]string{}, | ||
| }, | ||
| } | ||
| f(pod) | ||
| return pod | ||
| } | ||
| for desc, tc := range map[string]struct { | ||
| eviction *agonesv1.Eviction | ||
| pod *corev1.Pod | ||
| wantPod *corev1.Pod | ||
| }{ | ||
| "eviction: safe: Always, no incoming labels/annotations": { | ||
| eviction: &agonesv1.Eviction{Safe: agonesv1.EvictionSafeAlways}, | ||
| pod: emptyPodAnd(func(*corev1.Pod) {}), | ||
| wantPod: emptyPodAnd(func(pod *corev1.Pod) { | ||
| pod.ObjectMeta.Annotations[agonesv1.PodSafeToEvictAnnotation] = agonesv1.True | ||
| pod.ObjectMeta.Labels[agonesv1.SafeToEvictLabel] = agonesv1.True | ||
| }), | ||
| }, | ||
| "eviction: safe: OnUpgrade, no incoming labels/annotations": { | ||
| eviction: &agonesv1.Eviction{Safe: agonesv1.EvictionSafeOnUpgrade}, | ||
| pod: emptyPodAnd(func(*corev1.Pod) {}), | ||
| wantPod: emptyPodAnd(func(pod *corev1.Pod) { | ||
| pod.ObjectMeta.Annotations[agonesv1.PodSafeToEvictAnnotation] = agonesv1.False | ||
| pod.ObjectMeta.Labels[agonesv1.SafeToEvictLabel] = agonesv1.True | ||
| }), | ||
| }, | ||
| "eviction: safe: Never, no incoming labels/annotations": { | ||
| eviction: &agonesv1.Eviction{Safe: agonesv1.EvictionSafeNever}, | ||
| pod: emptyPodAnd(func(*corev1.Pod) {}), | ||
| wantPod: emptyPodAnd(func(pod *corev1.Pod) { | ||
| pod.ObjectMeta.Annotations[agonesv1.PodSafeToEvictAnnotation] = agonesv1.False | ||
| pod.ObjectMeta.Labels[agonesv1.SafeToEvictLabel] = agonesv1.False | ||
| }), | ||
| }, | ||
| "eviction: safe: Always, incoming labels/annotations": { | ||
| eviction: &agonesv1.Eviction{Safe: agonesv1.EvictionSafeAlways}, | ||
| pod: emptyPodAnd(func(pod *corev1.Pod) { | ||
| pod.ObjectMeta.Annotations[agonesv1.PodSafeToEvictAnnotation] = "just don't touch, ok?" | ||
| pod.ObjectMeta.Labels[agonesv1.SafeToEvictLabel] = "seriously, leave it" | ||
| }), | ||
| wantPod: emptyPodAnd(func(pod *corev1.Pod) { | ||
| pod.ObjectMeta.Annotations[agonesv1.PodSafeToEvictAnnotation] = "just don't touch, ok?" | ||
| pod.ObjectMeta.Labels[agonesv1.SafeToEvictLabel] = "seriously, leave it" | ||
| }), | ||
| }, | ||
| "eviction: safe: OnUpgrade, incoming labels/annotations": { | ||
| eviction: &agonesv1.Eviction{Safe: agonesv1.EvictionSafeOnUpgrade}, | ||
| pod: emptyPodAnd(func(pod *corev1.Pod) { | ||
| pod.ObjectMeta.Annotations[agonesv1.PodSafeToEvictAnnotation] = "better not touch" | ||
| pod.ObjectMeta.Labels[agonesv1.SafeToEvictLabel] = "not another one" | ||
| }), | ||
| wantPod: emptyPodAnd(func(pod *corev1.Pod) { | ||
| pod.ObjectMeta.Annotations[agonesv1.PodSafeToEvictAnnotation] = "better not touch" | ||
| pod.ObjectMeta.Labels[agonesv1.SafeToEvictLabel] = "not another one" | ||
| }), | ||
| }, | ||
| "eviction: safe: Never, incoming labels/annotations": { | ||
| eviction: &agonesv1.Eviction{Safe: agonesv1.EvictionSafeNever}, | ||
| pod: emptyPodAnd(func(pod *corev1.Pod) { | ||
| pod.ObjectMeta.Annotations[agonesv1.PodSafeToEvictAnnotation] = "a passthrough" | ||
| pod.ObjectMeta.Labels[agonesv1.SafeToEvictLabel] = "or is it passthru?" | ||
| }), | ||
| wantPod: emptyPodAnd(func(pod *corev1.Pod) { | ||
| pod.ObjectMeta.Annotations[agonesv1.PodSafeToEvictAnnotation] = "a passthrough" | ||
| pod.ObjectMeta.Labels[agonesv1.SafeToEvictLabel] = "or is it passthru?" | ||
| }), | ||
| }, | ||
| } { | ||
| t.Run(desc, func(t *testing.T) { | ||
| assert.NoError(t, SetEviction(tc.eviction, tc.pod)) | ||
| assert.Equal(t, tc.wantPod, tc.pod) | ||
| }) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.