Graduate FleetAllocationOverflow to Stable (#3733)

* Graduate FleetAllocationOverflow to Stable
* cleanup controller.go

cloudbuild.yaml

@@ -276,7 +276,7 @@ steps:
         declare -A versionsAndRegions=( [1.27]=us-east1 [1.28]=us-west1 [1.29]=europe-west1 )
 
         # Keep in sync with (the inverse of) pkg/util/runtime/features.go:featureDefaults
-        featureWithGate="PlayerAllocationFilter=true&PlayerTracking=true&FleetAllocationOverflow=false&CountsAndLists=true&DisableResyncOnSDKServer=false&Example=true"
+        featureWithGate="PlayerAllocationFilter=true&PlayerTracking=true&CountsAndLists=true&DisableResyncOnSDKServer=false&Example=true"
         featureWithoutGate=""
 
         # Use this if specific feature gates can only be supported on specific Kubernetes versions.

examples/fleet.yaml

@@ -50,11 +50,8 @@ spec:
       maxSurge: 25%
       # the amount to decrements GameServers by. Defaults to 25%
       maxUnavailable: 25%
-  # [Stage:Beta]
-  # [FeatureFlag:FleetAllocationOverflow]
   # Labels and/or Annotations to apply to overflowing GameServers when the number of Allocated GameServers exceeds
   # the desired replicas in the underlying `GameServerSet`
-  # Now in Beta, and enabled by default
   allocationOverflow:
     labels:
       mykey: myvalue

install/helm/agones/defaultfeaturegates.yaml

@@ -16,7 +16,6 @@
 
 # Beta features
 DisableResyncOnSDKServer: true
-FleetAllocationOverflow: true
 
 # Alpha features
 CountsAndLists: false

pkg/apis/agones/v1/fleet.go

@@ -15,14 +15,15 @@
 package v1
 
 import (
-	"agones.dev/agones/pkg"
-	"agones.dev/agones/pkg/apis"
-	"agones.dev/agones/pkg/apis/agones"
-	"agones.dev/agones/pkg/util/runtime"
 	appsv1 "k8s.io/api/apps/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/apimachinery/pkg/util/validation/field"
+
+	"agones.dev/agones/pkg"
+	"agones.dev/agones/pkg/apis"
+	"agones.dev/agones/pkg/apis/agones"
+	"agones.dev/agones/pkg/util/runtime"
 )
 
 const (
@@ -59,8 +60,6 @@ type FleetList struct {
 type FleetSpec struct {
 	// Replicas are the number of GameServers that should be in this set. Defaults to 0.
 	Replicas int32 `json:"replicas"`
-	// [Stage: Beta]
-	// [FeatureFlag:FleetAllocationOverflow]
 	// Labels and/or Annotations to apply to overflowing GameServers when the number of Allocated GameServers is more
 	// than the desired replicas on the underlying `GameServerSet`
 	// +optional
@@ -139,7 +138,7 @@ func (f *Fleet) GameServerSet() *GameServerSet {
 
 	gsSet.ObjectMeta.Labels[FleetNameLabel] = f.ObjectMeta.Name
 
-	if runtime.FeatureEnabled(runtime.FeatureFleetAllocateOverflow) && f.Spec.AllocationOverflow != nil {
+	if f.Spec.AllocationOverflow != nil {
 		gsSet.Spec.AllocationOverflow = f.Spec.AllocationOverflow.DeepCopy()
 	}
 
@@ -221,11 +220,7 @@ func (f *Fleet) Validate(apiHooks APIHooks) field.ErrorList {
 	allErrs = append(allErrs, validateObjectMeta(&f.Spec.Template.ObjectMeta, field.NewPath("spec", "template", "metadata"))...)
 
 	if f.Spec.AllocationOverflow != nil {
-		if runtime.FeatureEnabled(runtime.FeatureFleetAllocateOverflow) {
-			allErrs = append(allErrs, f.Spec.AllocationOverflow.Validate(field.NewPath("spec", "allocationOverflow"))...)
-		} else {
-			allErrs = append(allErrs, field.Forbidden(field.NewPath("spec", "allocationOverflow"), "Allocation Overflow is not enabled"))
-		}
+		allErrs = append(allErrs, f.Spec.AllocationOverflow.Validate(field.NewPath("spec", "allocationOverflow"))...)
 	}
 
 	if f.Spec.Priorities != nil && !runtime.FeatureEnabled(runtime.FeatureCountsAndLists) {

pkg/apis/agones/v1/fleet_test.go

@@ -19,8 +19,6 @@ import (
 	"strings"
 	"testing"
 
-	"agones.dev/agones/pkg/apis"
-	"agones.dev/agones/pkg/util/runtime"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	appsv1 "k8s.io/api/apps/v1"
@@ -29,6 +27,9 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/apimachinery/pkg/util/validation"
 	"k8s.io/apimachinery/pkg/util/validation/field"
+
+	"agones.dev/agones/pkg/apis"
+	"agones.dev/agones/pkg/util/runtime"
 )
 
 func TestFleetGameServerSetGameServer(t *testing.T) {
@@ -69,7 +70,7 @@ func TestFleetGameServerSetGameServer(t *testing.T) {
 	runtime.FeatureTestMutex.Lock()
 	defer runtime.FeatureTestMutex.Unlock()
 
-	runtime.Must(runtime.ParseFeatures(fmt.Sprintf("%s=true&%s=true", runtime.FeatureFleetAllocateOverflow, runtime.FeatureCountsAndLists)))
+	runtime.Must(runtime.ParseFeatures(fmt.Sprintf("%s=true", runtime.FeatureCountsAndLists)))
 	gsSet = f.GameServerSet()
 	assert.Nil(t, gsSet.Spec.AllocationOverflow)
 
@@ -214,10 +215,6 @@ func TestFleetGameserverSpec(t *testing.T) {
 
 func TestFleetAllocationOverflow(t *testing.T) {
 	t.Parallel()
-	runtime.FeatureTestMutex.Lock()
-	defer runtime.FeatureTestMutex.Unlock()
-
-	runtime.Must(runtime.ParseFeatures(fmt.Sprintf("%s=true", runtime.FeatureFleetAllocateOverflow)))
 
 	f := defaultFleet()
 	f.ApplyDefaults()
@@ -234,10 +231,6 @@ func TestFleetAllocationOverflow(t *testing.T) {
 	require.Len(t, errs, 1)
 	require.Equal(t, field.ErrorTypeInvalid, errs[0].Type)
 
-	runtime.Must(runtime.ParseFeatures(fmt.Sprintf("%s=false", runtime.FeatureFleetAllocateOverflow)))
-	errs = f.Validate(fakeAPIHooks{})
-	require.Len(t, errs, 1)
-	require.Equal(t, field.ErrorTypeForbidden, errs[0].Type)
 }
 
 func TestFleetName(t *testing.T) {

pkg/apis/agones/v1/gameserverset.go

@@ -59,8 +59,6 @@ type GameServerSetList struct {
 type GameServerSetSpec struct {
 	// Replicas are the number of GameServers that should be in this set
 	Replicas int32 `json:"replicas"`
-	// [Stage: Beta]
-	// [FeatureFlag:FleetAllocationOverflow]
 	// Labels and Annotations to apply to GameServers when the number of Allocated GameServers drops below
 	// the desired replicas on the underlying `GameServerSet`
 	// +optional

pkg/gameserversets/controller.go

@@ -20,19 +20,6 @@ import (
 	"sync"
 	"time"
 
-	"agones.dev/agones/pkg/apis"
-	"agones.dev/agones/pkg/apis/agones"
-	agonesv1 "agones.dev/agones/pkg/apis/agones/v1"
-	"agones.dev/agones/pkg/client/clientset/versioned"
-	getterv1 "agones.dev/agones/pkg/client/clientset/versioned/typed/agones/v1"
-	"agones.dev/agones/pkg/client/informers/externalversions"
-	listerv1 "agones.dev/agones/pkg/client/listers/agones/v1"
-	"agones.dev/agones/pkg/gameservers"
-	"agones.dev/agones/pkg/util/crd"
-	"agones.dev/agones/pkg/util/logfields"
-	"agones.dev/agones/pkg/util/runtime"
-	"agones.dev/agones/pkg/util/webhooks"
-	"agones.dev/agones/pkg/util/workerqueue"
 	"github.com/google/go-cmp/cmp"
 	"github.com/heptiolabs/healthcheck"
 	"github.com/pkg/errors"
@@ -49,6 +36,20 @@ import (
 	typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/tools/record"
+
+	"agones.dev/agones/pkg/apis"
+	"agones.dev/agones/pkg/apis/agones"
+	agonesv1 "agones.dev/agones/pkg/apis/agones/v1"
+	"agones.dev/agones/pkg/client/clientset/versioned"
+	getterv1 "agones.dev/agones/pkg/client/clientset/versioned/typed/agones/v1"
+	"agones.dev/agones/pkg/client/informers/externalversions"
+	listerv1 "agones.dev/agones/pkg/client/listers/agones/v1"
+	"agones.dev/agones/pkg/gameservers"
+	"agones.dev/agones/pkg/util/crd"
+	"agones.dev/agones/pkg/util/logfields"
+	"agones.dev/agones/pkg/util/runtime"
+	"agones.dev/agones/pkg/util/webhooks"
+	"agones.dev/agones/pkg/util/workerqueue"
 )
 
 var (
@@ -130,9 +131,7 @@ func NewController(
 	eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: kubeClient.CoreV1().Events("")})
 	c.recorder = eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: "gameserverset-controller"})
 
-	if runtime.FeatureEnabled(runtime.FeatureFleetAllocateOverflow) {
-		c.allocationController = NewAllocatorOverflowController(health, counter, agonesClient, agonesInformerFactory)
-	}
+	c.allocationController = NewAllocatorOverflowController(health, counter, agonesClient, agonesInformerFactory)
 
 	_, _ = gsSetInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc: c.workerqueue.Enqueue,
@@ -189,13 +188,11 @@ func (c *Controller) Run(ctx context.Context, workers int) error {
 		return errors.New("failed to wait for caches to sync")
 	}
 
-	if runtime.FeatureEnabled(runtime.FeatureFleetAllocateOverflow) {
-		go func() {
-			if err := c.allocationController.Run(ctx); err != nil {
-				c.baseLogger.WithError(err).Error("error running allocation overflow controller")
-			}
-		}()
-	}
+	go func() {
+		if err := c.allocationController.Run(ctx); err != nil {
+			c.baseLogger.WithError(err).Error("error running allocation overflow controller")
+		}
+	}()
 
 	c.workerqueue.Run(ctx, workers)
 	return nil

pkg/util/runtime/features.go

@@ -31,10 +31,6 @@ const (
 	////////////////
 	// Beta features
 
-	// FeatureFleetAllocateOverflow enables setting labels and/or annotations on Allocated GameServers
-	// if the desired number of the underlying GameServerSet drops below the number of Allocated GameServers.
-	FeatureFleetAllocateOverflow Feature = "FleetAllocationOverflow"
-
 	////////////////
 	// Alpha features
 
@@ -100,7 +96,6 @@ var (
 	featureDefaults = map[Feature]bool{
 		// Beta features
 		FeatureDisableResyncOnSDKServer: true,
-		FeatureFleetAllocateOverflow:    true,
 
 		// Alpha features
 		FeatureCountsAndLists:                   false,

site/content/en/docs/Guides/feature-stages.md

@@ -40,7 +40,6 @@ The current set of `alpha` and `beta` feature gates:
 | Feature Name                                                                                                                | Gate                               | Default  | Stage   | Since  |
 |-----------------------------------------------------------------------------------------------------------------------------|------------------------------------|----------|---------|--------|
 | [DisableResyncOnSDKServer](https://github.com/googleforgames/agones/issues/3377)                                            | `DisableResyncOnSDKServer`         | Enabled  | `Beta`  | 1.40.0 |
-| [Allocated GameServers are notified on relevant Fleet Updates][fleet-updates]                                               | `FleetAllocationOverflow`          | Enabled  | `Beta`  | 1.37.0 |
 | [CountsAndLists](https://github.com/googleforgames/agones/issues/2716)                                                      | `CountsAndLists`                   | Disabled | `Alpha` | 1.37.0 |
 | [Support for Extended Duration Pods on GKE Autopilot (*1.28+ only*)](https://github.com/googleforgames/agones/issues/3386)  | `GKEAutopilotExtendedDurationPods` | Disabled | `Alpha` | 1.37.0 |
 | [GameServer player capacity filtering on GameServerAllocations](https://github.com/googleforgames/agones/issues/1239)       | `PlayerAllocationFilter`           | Disabled | `Alpha` | 1.14.0 |

site/content/en/docs/Guides/fleet-updates.md

@@ -127,8 +127,9 @@ the rate that you deem fit for your specific rollout.
 
 ## Notifying GameServers on Fleet Update/Downscale
 
-
+{{% feature expiryVersion="1.40.0" %}}
 {{< beta title="Allocated GameSever Overflow Notification" gate="FleetAllocationOverflow" >}}
+{{% /feature %}}
 
 When `Allocated` `GameServers` are utilised for a long time, such as a Lobby `GameServer`,
 or a `GameServer` that is being reused multiple times in a row, it can be useful

site/content/en/docs/Reference/agones_crd_api_reference.html

@@ -4482,9 +4482,7 @@ <h3 id="agones.dev/v1.Fleet">Fleet
 </td>
 <td>
 <em>(Optional)</em>
-<p>[Stage: Beta]
-[FeatureFlag:FleetAllocationOverflow]
-Labels and/or Annotations to apply to overflowing GameServers when the number of Allocated GameServers is more
+<p>Labels and/or Annotations to apply to overflowing GameServers when the number of Allocated GameServers is more
 than the desired replicas on the underlying <code>GameServerSet</code></p>
 </td>
 </tr>
@@ -4855,9 +4853,7 @@ <h3 id="agones.dev/v1.GameServerSet">GameServerSet
 </td>
 <td>
 <em>(Optional)</em>
-<p>[Stage: Beta]
-[FeatureFlag:FleetAllocationOverflow]
-Labels and Annotations to apply to GameServers when the number of Allocated GameServers drops below
+<p>Labels and Annotations to apply to GameServers when the number of Allocated GameServers drops below
 the desired replicas on the underlying <code>GameServerSet</code></p>
 </td>
 </tr>
@@ -5254,9 +5250,7 @@ <h3 id="agones.dev/v1.FleetSpec">FleetSpec
 </td>
 <td>
 <em>(Optional)</em>
-<p>[Stage: Beta]
-[FeatureFlag:FleetAllocationOverflow]
-Labels and/or Annotations to apply to overflowing GameServers when the number of Allocated GameServers is more
+<p>Labels and/or Annotations to apply to overflowing GameServers when the number of Allocated GameServers is more
 than the desired replicas on the underlying <code>GameServerSet</code></p>
 </td>
 </tr>
@@ -5565,9 +5559,7 @@ <h3 id="agones.dev/v1.GameServerSetSpec">GameServerSetSpec
 </td>
 <td>
 <em>(Optional)</em>
-<p>[Stage: Beta]
-[FeatureFlag:FleetAllocationOverflow]
-Labels and Annotations to apply to GameServers when the number of Allocated GameServers drops below
+<p>Labels and Annotations to apply to GameServers when the number of Allocated GameServers drops below
 the desired replicas on the underlying <code>GameServerSet</code></p>
 </td>
 </tr>

site/content/en/docs/Reference/fleet.md

@@ -12,7 +12,7 @@ Like any other Kubernetes resource you describe a `Fleet`'s desired state via a
 
 A full `Fleet` specification is available below and in the {{< ghlink href="examples/fleet.yaml" >}}example folder{{< /ghlink >}} for reference :
 
-
+{{% feature expiryVersion="1.40.0" %}}
 ```yaml
 apiVersion: "agones.dev/v1"
 kind: Fleet
@@ -92,6 +92,87 @@ spec:
           - name: simple-game-server
             image: {{< example-image >}}
 ```
+{{% /feature %}}
+
+{{% feature publishVersion="1.40.0" %}}
+```yaml
+apiVersion: "agones.dev/v1"
+kind: Fleet
+# Fleet Metadata
+# {{< k8s-api-version href="#objectmeta-v1-meta" >}}
+metadata:
+  name: fleet-example
+spec:
+  # the number of GameServers to keep Ready or Allocated in this Fleet
+  replicas: 2
+  # defines how GameServers are organised across the cluster.
+  # Options include:
+  # "Packed" (default) is aimed at dynamic Kubernetes clusters, such as cloud providers, wherein we want to bin pack
+  # resources
+  # "Distributed" is aimed at static Kubernetes clusters, wherein we want to distribute resources across the entire
+  # cluster
+  scheduling: Packed
+  # a GameServer template - see:
+  # https://agones.dev/site/docs/reference/gameserver/ for all the options
+  strategy:
+    # The replacement strategy for when the GameServer template is changed. Default option is "RollingUpdate",
+    # "RollingUpdate" will increment by maxSurge value on each iteration, while decrementing by maxUnavailable on each
+    # iteration, until all GameServers have been switched from one version to another.
+    # "Recreate" terminates all non-allocated GameServers, and starts up a new set with the new details to replace them.
+    type: RollingUpdate
+    # Only relevant when `type: RollingUpdate`
+    rollingUpdate:
+      # the amount to increment the new GameServers by. Defaults to 25%
+      maxSurge: 25%
+      # the amount to decrements GameServers by. Defaults to 25%
+      maxUnavailable: 25%
+  # Labels and/or Annotations to apply to overflowing GameServers when the number of Allocated GameServers is more
+  # than the desired replicas on the underlying `GameServerSet`
+  allocationOverflow:
+    labels:
+      mykey: myvalue
+      version: "" # empty an existing label value
+    annotations:
+      otherkey: setthisvalue
+  #
+  # [Stage:Alpha]
+  # [FeatureFlag:CountsAndLists]
+  # Which gameservers in the Fleet are most important to keep around - impacts scale down logic.
+  # priorities:
+  # - type: Counter # Sort by a “Counter”
+  #   key: player # The name of the Counter. No impact if no GameServer found.
+  #   order: Descending # Default is "Ascending" so smaller capacity will be removed first on down scaling.
+  # - type: List # Sort by a “List”
+  #   key: room # The name of the List. No impact if no GameServer found.
+  #   order: Ascending # Default is "Ascending" so smaller capacity will be removed first on down scaling.
+  #      
+  template:
+    # GameServer metadata
+    metadata:
+      labels:
+        foo: bar
+    # GameServer specification
+    spec:
+      ports:
+      - name: default
+        portPolicy: Dynamic
+        containerPort: 26000
+      health:
+        initialDelaySeconds: 30
+        periodSeconds: 60
+      # Parameters for game server sidecar
+      sdkServer:
+        logLevel: Info
+        grpcPort: 9357
+        httpPort: 9358
+      # The GameServer's Pod template
+      template:
+        spec:
+          containers:
+          - name: simple-game-server
+            image: {{< example-image >}}
+```
+{{% /feature %}}
 
 Since Agones defines a new 
 [Custom Resources Definition (CRD)](https://kubernetes.io/docs/concepts/api-extension/custom-resources/) 

test/e2e/fleet_test.go

@@ -1774,9 +1774,6 @@ func TestFleetAggregatedListStatus(t *testing.T) {
 }
 
 func TestFleetAllocationOverflow(t *testing.T) {
-	if !runtime.FeatureEnabled(runtime.FeatureFleetAllocateOverflow) {
-		t.SkipNow()
-	}
 	t.Parallel()
 	ctx := context.Background()
 	client := framework.AgonesClient.AgonesV1()