Skip to content

Commit

Permalink
Replace Timeout with RolloutConfig
Browse files Browse the repository at this point in the history 
Uses `MinSuccessTime`, `ProgressDeadline`, and `MaxFailures` for a more
fine-tuned configuration.

Signed-off-by: Dale Haiducek <19750917+dhaiducek@users.noreply.github.com>
  • Loading branch information
@dhaiducek
dhaiducek committed Sep 15, 2023
1 parent a03dfd9 commit 5477613
Show file tree
Hide file tree
Showing 6 changed files with 228 additions and 124 deletions.
14 changes: 7 additions & 7 deletions cluster/v1alpha1/helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ func (r *RolloutHandler) getRolloutAllClusters(rolloutStrategy RolloutStrategy,
}

// Parse timeout for the rollout
failureTimeout, err := parseTimeout(strategy.All.Timeout.Timeout)
failureTimeout, err := parseTimeout(strategy.All.ProgressDeadline)
if err != nil {
return &strategy, RolloutResult{}, err
}
Expand Down Expand Up @@ -137,7 +137,7 @@ func (r *RolloutHandler) getProgressiveClusters(rolloutStrategy RolloutStrategy,
}

// Parse timeout for non-mandatory decision groups
failureTimeout, err := parseTimeout(strategy.Progressive.Timeout.Timeout)
failureTimeout, err := parseTimeout(strategy.Progressive.ProgressDeadline)
if err != nil {
return &strategy, RolloutResult{}, err
}
Expand Down Expand Up @@ -176,7 +176,7 @@ func (r *RolloutHandler) getProgressivePerGroupClusters(rolloutStrategy RolloutS
}

// Parse timeout for non-mandatory decision groups
failureTimeout, err := parseTimeout(strategy.ProgressivePerGroup.Timeout.Timeout)
failureTimeout, err := parseTimeout(strategy.ProgressivePerGroup.ProgressDeadline)
if err != nil {
return &strategy, RolloutResult{}, err
}
Expand All @@ -189,7 +189,9 @@ func (r *RolloutHandler) getProgressivePerGroupClusters(rolloutStrategy RolloutS
return &strategy, rolloutResult, nil
}

func progressivePerCluster(clusterGroupsMap clusterv1beta1.ClusterGroupsMap, length int, timeout time.Duration, statusFunc ClusterRolloutStatusFunc) RolloutResult {
func progressivePerCluster(
clusterGroupsMap clusterv1beta1.ClusterGroupsMap, length int, timeout time.Duration, statusFunc ClusterRolloutStatusFunc,
) RolloutResult {
rolloutClusters := map[string]ClusterRolloutStatus{}
timeoutClusters := map[string]ClusterRolloutStatus{}

Expand Down Expand Up @@ -289,9 +291,7 @@ func progressivePerGroup(clusterGroupsMap clusterv1beta1.ClusterGroupsMap, timeo
func determineRolloutStatusAndContinue(status ClusterRolloutStatus, timeout time.Duration) (*ClusterRolloutStatus, bool) {
newStatus := status.DeepCopy()
switch status.Status {
case ToApply:
return newStatus, true
case TimeOut, Succeeded, Skip:
case Succeeded, TimeOut, Skip:
return newStatus, false
case Progressing, Failed:
timeOutTime := getTimeOutTime(status.LastTransitionTime, timeout)
Expand Down
44 changes: 22 additions & 22 deletions cluster/v1alpha1/helpers_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ func TestGetRolloutCluster_All(t *testing.T) {
}{
{
name: "test rollout all with timeout 90s",
rolloutStrategy: RolloutStrategy{Type: All, All: &RolloutAll{Timeout: Timeout{"90s"}}},
rolloutStrategy: RolloutStrategy{Type: All, All: &RolloutAll{RolloutConfig: RolloutConfig{ProgressDeadline: "90s"}}},
existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{
{GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2"),
{GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4", "cluster5", "cluster6"),
Expand All @@ -60,7 +60,7 @@ func TestGetRolloutCluster_All(t *testing.T) {
}
return clustersRolloutStatus[clusterName]
},
expectRolloutStrategy: &RolloutStrategy{Type: All, All: &RolloutAll{Timeout: Timeout{"90s"}}},
expectRolloutStrategy: &RolloutStrategy{Type: All, All: &RolloutAll{RolloutConfig: RolloutConfig{ProgressDeadline: "90s"}}},
expectRolloutClusters: map[string]ClusterRolloutStatus{
"cluster1": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply, LastTransitionTime: &fakeTime_60s},
"cluster2": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, LastTransitionTime: &fakeTime_60s, TimeOutTime: &fakeTime30s},
Expand Down Expand Up @@ -88,7 +88,7 @@ func TestGetRolloutCluster_All(t *testing.T) {
}
return clustersRolloutStatus[clusterName]
},
expectRolloutStrategy: &RolloutStrategy{Type: All, All: &RolloutAll{Timeout: Timeout{""}}},
expectRolloutStrategy: &RolloutStrategy{Type: All, All: &RolloutAll{RolloutConfig: RolloutConfig{ProgressDeadline: ""}}},
expectRolloutClusters: map[string]ClusterRolloutStatus{
"cluster1": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply},
"cluster2": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: Progressing, TimeOutTime: &fakeTimeMax},
Expand All @@ -99,7 +99,7 @@ func TestGetRolloutCluster_All(t *testing.T) {
},
{
name: "test rollout all with timeout 0s",
rolloutStrategy: RolloutStrategy{Type: All, All: &RolloutAll{Timeout: Timeout{"0s"}}},
rolloutStrategy: RolloutStrategy{Type: All, All: &RolloutAll{RolloutConfig: RolloutConfig{ProgressDeadline: "0s"}}},
existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{
{GroupName: "group1", GroupIndex: 0}: sets.New[string]("cluster1", "cluster2"),
{GroupName: "", GroupIndex: 1}: sets.New[string]("cluster3", "cluster4", "cluster5"),
Expand All @@ -114,7 +114,7 @@ func TestGetRolloutCluster_All(t *testing.T) {
}
return clustersRolloutStatus[clusterName]
},
expectRolloutStrategy: &RolloutStrategy{Type: All, All: &RolloutAll{Timeout: Timeout{"0s"}}},
expectRolloutStrategy: &RolloutStrategy{Type: All, All: &RolloutAll{RolloutConfig: RolloutConfig{ProgressDeadline: "0s"}}},
expectRolloutClusters: map[string]ClusterRolloutStatus{
"cluster1": {GroupKey: clusterv1beta1.GroupKey{GroupName: "group1", GroupIndex: 0}, Status: ToApply},
"cluster5": {GroupKey: clusterv1beta1.GroupKey{GroupName: "", GroupIndex: 1}},
Expand Down Expand Up @@ -165,7 +165,7 @@ func TestGetRolloutCluster_Progressive(t *testing.T) {
rolloutStrategy: RolloutStrategy{
Type: Progressive,
Progressive: &RolloutProgressive{
Timeout: Timeout{"90s"},
RolloutConfig: RolloutConfig{ProgressDeadline: "90s"},
},
},
existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{
Expand All @@ -186,7 +186,7 @@ func TestGetRolloutCluster_Progressive(t *testing.T) {
expectRolloutStrategy: &RolloutStrategy{
Type: Progressive,
Progressive: &RolloutProgressive{
Timeout: Timeout{"90s"},
RolloutConfig: RolloutConfig{ProgressDeadline: "90s"},
},
},
expectRolloutClusters: map[string]ClusterRolloutStatus{
Expand All @@ -204,7 +204,7 @@ func TestGetRolloutCluster_Progressive(t *testing.T) {
rolloutStrategy: RolloutStrategy{
Type: Progressive,
Progressive: &RolloutProgressive{
Timeout: Timeout{""},
RolloutConfig: RolloutConfig{ProgressDeadline: ""},
MaxConcurrency: intstr.FromString("50%"), // 50% of total clusters
},
},
Expand All @@ -225,7 +225,7 @@ func TestGetRolloutCluster_Progressive(t *testing.T) {
expectRolloutStrategy: &RolloutStrategy{
Type: Progressive,
Progressive: &RolloutProgressive{
Timeout: Timeout{""},
RolloutConfig: RolloutConfig{ProgressDeadline: ""},
MaxConcurrency: intstr.FromString("50%"), // 50% of total clusters
},
},
Expand All @@ -241,7 +241,7 @@ func TestGetRolloutCluster_Progressive(t *testing.T) {
rolloutStrategy: RolloutStrategy{
Type: Progressive,
Progressive: &RolloutProgressive{
Timeout: Timeout{"0s"},
RolloutConfig: RolloutConfig{ProgressDeadline: "0s"},
MaxConcurrency: intstr.FromInt(3), // Maximum 3 clusters concurrently
},
},
Expand All @@ -262,7 +262,7 @@ func TestGetRolloutCluster_Progressive(t *testing.T) {
expectRolloutStrategy: &RolloutStrategy{
Type: Progressive,
Progressive: &RolloutProgressive{
Timeout: Timeout{"0s"},
RolloutConfig: RolloutConfig{ProgressDeadline: "0s"},
MaxConcurrency: intstr.FromInt(3), // Maximum 3 clusters concurrently
},
},
Expand Down Expand Up @@ -311,7 +311,7 @@ func TestGetRolloutCluster_Progressive(t *testing.T) {
},
},
MaxConcurrency: intstr.FromString("50%"),
Timeout: Timeout{""},
RolloutConfig: RolloutConfig{ProgressDeadline: ""},
},
},
expectRolloutClusters: map[string]ClusterRolloutStatus{
Expand Down Expand Up @@ -357,7 +357,7 @@ func TestGetRolloutCluster_Progressive(t *testing.T) {
},
},
MaxConcurrency: intstr.FromInt(2),
Timeout: Timeout{""},
RolloutConfig: RolloutConfig{ProgressDeadline: ""},
},
},
expectRolloutClusters: map[string]ClusterRolloutStatus{
Expand All @@ -377,7 +377,7 @@ func TestGetRolloutCluster_Progressive(t *testing.T) {
},
},
MaxConcurrency: intstr.FromString("50%"), // 50% of total clusters
Timeout: Timeout{"0s"},
RolloutConfig: RolloutConfig{ProgressDeadline: "0s"},
},
},
existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{
Expand All @@ -403,7 +403,7 @@ func TestGetRolloutCluster_Progressive(t *testing.T) {
},
},
MaxConcurrency: intstr.FromString("50%"), // 50% of total clusters
Timeout: Timeout{"0s"},
RolloutConfig: RolloutConfig{ProgressDeadline: "0s"},
},
},
expectRolloutClusters: map[string]ClusterRolloutStatus{
Expand Down Expand Up @@ -455,7 +455,7 @@ func TestGetRolloutCluster_ProgressivePerGroup(t *testing.T) {
rolloutStrategy: RolloutStrategy{
Type: ProgressivePerGroup,
ProgressivePerGroup: &RolloutProgressivePerGroup{
Timeout: Timeout{"90s"},
RolloutConfig: RolloutConfig{ProgressDeadline: "90s"},
},
},
existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{
Expand All @@ -476,7 +476,7 @@ func TestGetRolloutCluster_ProgressivePerGroup(t *testing.T) {
expectRolloutStrategy: &RolloutStrategy{
Type: ProgressivePerGroup,
ProgressivePerGroup: &RolloutProgressivePerGroup{
Timeout: Timeout{"90s"},
RolloutConfig: RolloutConfig{ProgressDeadline: "90s"},
},
},
expectRolloutClusters: map[string]ClusterRolloutStatus{
Expand Down Expand Up @@ -509,7 +509,7 @@ func TestGetRolloutCluster_ProgressivePerGroup(t *testing.T) {
expectRolloutStrategy: &RolloutStrategy{
Type: ProgressivePerGroup,
ProgressivePerGroup: &RolloutProgressivePerGroup{
Timeout: Timeout{""},
RolloutConfig: RolloutConfig{ProgressDeadline: ""},
},
},
expectRolloutClusters: map[string]ClusterRolloutStatus{
Expand All @@ -523,7 +523,7 @@ func TestGetRolloutCluster_ProgressivePerGroup(t *testing.T) {
rolloutStrategy: RolloutStrategy{
Type: ProgressivePerGroup,
ProgressivePerGroup: &RolloutProgressivePerGroup{
Timeout: Timeout{"0s"},
RolloutConfig: RolloutConfig{ProgressDeadline: "0s"},
},
},
existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{
Expand All @@ -544,7 +544,7 @@ func TestGetRolloutCluster_ProgressivePerGroup(t *testing.T) {
expectRolloutStrategy: &RolloutStrategy{
Type: ProgressivePerGroup,
ProgressivePerGroup: &RolloutProgressivePerGroup{
Timeout: Timeout{"0s"},
RolloutConfig: RolloutConfig{ProgressDeadline: "0s"},
},
},
expectRolloutClusters: map[string]ClusterRolloutStatus{
Expand Down Expand Up @@ -652,7 +652,7 @@ func TestGetRolloutCluster_ProgressivePerGroup(t *testing.T) {
{GroupName: "group1"},
},
},
Timeout: Timeout{"0s"},
RolloutConfig: RolloutConfig{ProgressDeadline: "0s"},
},
},
existingScheduledClusterGroups: map[clusterv1beta1.GroupKey]sets.Set[string]{
Expand All @@ -677,7 +677,7 @@ func TestGetRolloutCluster_ProgressivePerGroup(t *testing.T) {
{GroupName: "group1"},
},
},
Timeout: Timeout{"0s"},
RolloutConfig: RolloutConfig{ProgressDeadline: "0s"},
},
},
expectRolloutClusters: map[string]ClusterRolloutStatus{
Expand Down
67 changes: 46 additions & 21 deletions cluster/v1alpha1/types_rolloutstrategy.go
View file
Original file line number Diff line number Diff line change
@@ -1,55 +1,80 @@
package v1alpha1

import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
)

// +k8s:deepcopy-gen=true

// RolloutStrategy API used by workload applier APIs to define how the workload will be applied to the selected clusters by the Placement and DecisionStrategy.

type RolloutType string

const (
//All means apply the workload to all clusters in the decision groups at once.
All string = "All"
All RolloutType = "All"
//Progressive means apply the workload to the selected clusters progressively per cluster.
Progressive string = "Progressive"
Progressive RolloutType = "Progressive"
//ProgressivePerGroup means apply the workload to the selected clusters progressively per group.
ProgressivePerGroup string = "ProgressivePerGroup"
ProgressivePerGroup RolloutType = "ProgressivePerGroup"
)

// Rollout strategy to apply workload to the selected clusters by Placement and DecisionStrategy.
type RolloutStrategy struct {
// Rollout strategy Types are All, Progressive and ProgressivePerGroup
// 1) All means apply the workload to all clusters in the decision groups at once.
// 2) Progressive means apply the workload to the selected clusters progressively per cluster. The workload will not be applied to the next cluster unless one of the current applied clusters reach the successful state or timeout.
// 3) ProgressivePerGroup means apply the workload to decisionGroup clusters progressively per group. The workload will not be applied to the next decisionGroup unless all clusters in the current group reach the successful state or timeout.
// 2) Progressive means apply the workload to the selected clusters progressively per cluster. The workload will not
// be applied to the next cluster unless one of the current applied clusters reach the successful state and haven't
// breached the MaxFailures configuration.
// 3) ProgressivePerGroup means apply the workload to decisionGroup clusters progressively per group. The workload
// will not be applied to the next decisionGroup unless all clusters in the current group reach the successful
// state and haven't breached the MaxFailures configuration.
// +kubebuilder:validation:Enum=All;Progressive;ProgressivePerGroup
// +kubebuilder:default:=All
// +optional
Type string `json:"type,omitempty"`
Type RolloutType `json:"type,omitempty"`

// All define required fields for RolloutStrategy type All
// All defines required fields for RolloutStrategy type All
// +optional
All *RolloutAll `json:"all,omitempty"`

// Progressive define required fields for RolloutStrategy type Progressive
// Progressive defines required fields for RolloutStrategy type Progressive
// +optional
Progressive *RolloutProgressive `json:"progressive,omitempty"`

// ProgressivePerGroup define required fields for RolloutStrategy type ProgressivePerGroup
// ProgressivePerGroup defines required fields for RolloutStrategy type ProgressivePerGroup
// +optional
ProgressivePerGroup *RolloutProgressivePerGroup `json:"progressivePerGroup,omitempty"`
}

// Timeout to consider while applying the workload.
type Timeout struct {
// Timeout define how long workload applier controller will wait till workload reach successful state in the cluster.
// Timeout default value is None meaning the workload applier will not proceed apply workload to other clusters if did not reach the successful state.
// Timeout must be defined in [0-9h]|[0-9m]|[0-9s] format examples; 2h , 90m , 360s
// RolloutConfig contains common configurations for the rollout strategies.
type RolloutConfig struct {
// MinSuccessTime is a "soak" time. In other words, the minimum amount of time the workload applier controller will
// wait from the start of each rollout before proceeding (assuming a successful state has been reached and MaxFailures
// wasn't breached).
// MinSuccessTime is only considered for rollout types Progressive and ProgressivePerGroup.
// The default value is 0 meaning the workload applier proceed immediately after a successful state is reached.
// MinSuccessTime must be defined in [0-9h]|[0-9m]|[0-9s] format examples; 2h , 90m , 360s
// +kubebuilder:default:=0
// +optional
MinSuccessTime metav1.Time `json:"minSuccessTime,omitempty"`
// ProgressDeadline defines how long the workload applier controller will wait until workload reaches a successful
// state in the cluster, marking it as failed after the deadline.
// The default value is None meaning the workload applier will continue to wait for a status if did not reach the
// successful state.
// ProgressDeadline must be defined in [0-9h]|[0-9m]|[0-9s] format examples; 2h , 90m , 360s
// +kubebuilder:validation:Pattern="^(([0-9])+[h|m|s])|None$"
// +kubebuilder:default:=None
// +optional
Timeout string `json:"timeout,omitempty"`
ProgressDeadline string `json:"progressDeadline,omitempty"`
// MaxFailures is a percentage of or number of clusters in the current rollout that can fail before proceeding to the
// next rollout.
// Default is that no failures are tolerated.
// +kubebuilder:validation:Pattern="^((100|[0-9]{1,2})%|[0-9]+)$"
// +kubebuilder:default=0
// +optional
MaxFailures string `json:"maxFailures,omitempty"`
}

// MandatoryDecisionGroup set the decision group name or group index.
Expand All @@ -75,20 +100,23 @@ type MandatoryDecisionGroups struct {
// RolloutAll is a RolloutStrategy Type
type RolloutAll struct {
// +optional
Timeout `json:",inline"`
RolloutConfig `json:",inline"`
}

// RolloutProgressivePerGroup is a RolloutStrategy Type
type RolloutProgressivePerGroup struct {
// +optional
MandatoryDecisionGroups `json:",inline"`
RolloutConfig `json:",inline"`

// +optional
Timeout `json:",inline"`
MandatoryDecisionGroups `json:",inline"`
}

// RolloutProgressive is a RolloutStrategy Type
type RolloutProgressive struct {
// +optional
RolloutConfig `json:",inline"`

// +optional
MandatoryDecisionGroups `json:",inline"`

Expand All @@ -97,7 +125,4 @@ type RolloutProgressive struct {
// +kubebuilder:validation:XIntOrString
// +optional
MaxConcurrency intstr.IntOrString `json:"maxConcurrency,omitempty"`

// +optional
Timeout `json:",inline"`
}
Loading

0 comments on commit 5477613

Please sign in to comment.