From 5d8917eedd77f60b416cc0c113d47a6a85959a21 Mon Sep 17 00:00:00 2001 From: michaelawyu Date: Sat, 21 Sep 2024 03:08:40 +0800 Subject: [PATCH] interface: add additional apply strategy (#915) --- .../v1beta1/clusterresourceplacement_types.go | 221 ++++++++++++++++-- ...etes-fleet.io_clusterresourcebindings.yaml | 203 ++++++++++++++-- ...es-fleet.io_clusterresourceplacements.yaml | 203 ++++++++++++++-- .../placement.kubernetes-fleet.io_works.yaml | 203 ++++++++++++++-- 4 files changed, 766 insertions(+), 64 deletions(-) diff --git a/apis/placement/v1beta1/clusterresourceplacement_types.go b/apis/placement/v1beta1/clusterresourceplacement_types.go index 34e6fbf0b..01c8b4552 100644 --- a/apis/placement/v1beta1/clusterresourceplacement_types.go +++ b/apis/placement/v1beta1/clusterresourceplacement_types.go @@ -430,14 +430,116 @@ type RolloutStrategy struct { // Note: If multiple CRPs try to place the same resource with different apply strategy, the later ones will fail with the // reason ApplyConflictBetweenPlacements. type ApplyStrategy struct { - // Type defines the type of strategy to use. Default to ClientSideApply. - // Server-side apply is more powerful and flexible than client-side apply. - // You SHOULD use server-side apply to safely resolve any potential drift between the - // original applied resource version and the current resource on the member cluster. - // Read more about the differences between server-side apply and client-side apply: - // https://kubernetes.io/docs/reference/using-api/server-side-apply/#comparison-with-client-side-apply. - // You can also use ReportDiff to only report the difference between the resource on the member cluster - // and the resource to be applied from the hub on all the selected clusters. + // ComparisonOption controls how Fleet compares the desired state of a resource, as kept in + // a hub cluster manifest, with the current state of the resource (if applicable) in the + // member cluster. + // + // Available options are: + // + // * PartialComparison: with this option, Fleet will compare only fields that are managed by + // Fleet, i.e., the fields that are specified explicitly in the hub cluster manifest. + // Unmanaged fields are ignored. This is the default option. + // + // * FullComparison: with this option, Fleet will compare all fields of the resource, + // even if the fields are absent from the hub cluster manifest. + // + // Consider using the PartialComparison option if you would like to: + // + // * use the default values for certain fields; or + // * let another agent, e.g., HPAs, VPAs, etc., on the member cluster side manage some fields; or + // * allow ad-hoc or cluster-specific settings on the member cluster side. + // + // To use the FullComparison option, it is recommended that you: + // + // * specify all fields as appropriate in the hub cluster, even if you are OK with using default + // values; + // * make sure that no fields are managed by agents other than Fleet on the member cluster + // side, such as HPAs, VPAs, or other controllers. + // + // See the Fleet documentation for further explanations and usage examples. + // + // +kubebuilder:default=PartialComparison + // +kubebuilder:validation:Enum=PartialComparison;FullComparison + // +kubebuilder:validation:Optional + ComparisonOption ComparisonOptionType `json:"compareOption,omitempty"` + + // WhenToApply controls when Fleet would apply the manifests on the hub cluster to the member + // clusters. + // + // Available options are: + // + // * Always: with this option, Fleet will periodically apply hub cluster manifests + // on the member cluster side; this will effectively overwrite any change in the fields + // managed by Fleet (i.e., specified in the hub cluster manifest). This is the default + // option. + // + // Note that this option would revert any ad-hoc changes made on the member cluster side in + // the managed fields; if you would like to make temporary edits on the member cluster side + // in the managed fields, switch to IfNotDrifted option. Note that changes in unmanaged + // fields will be left alone; if you use the FullDiff compare option, such changes will + // be reported as drifts. + // + // * IfNotDrifted: with this option, Fleet will stop applying hub cluster manifests on + // clusters that have drifted from the desired state; apply ops would still continue on + // the rest of the clusters. Drifts are calculated using the ComparisonOption, + // as explained in the corresponding field. + // + // Use this option if you would like Fleet to detect drifts in your multi-cluster setup. + // A drift occurs when an agent makes an ad-hoc change on the member cluster side that + // makes affected resources deviate from its desired state as kept in the hub cluster; + // and this option grants you an opportunity to view the drift details and take actions + // accordingly. The drift details will be reported in the CRP status. + // + // To fix a drift, you may: + // + // * revert the changes manually on the member cluster side + // * update the hub cluster manifest; this will trigger Fleet to apply the latest revision + // of the manifests, which will overwrite the drifted fields + // (if they are managed by Fleet) + // * switch to the Always option; this will trigger Fleet to apply the current revision + // of the manifests, which will overwrite the drifted fields (if they are managed by Fleet). + // * if applicable and necessary, delete the drifted resources on the member cluster side; Fleet + // will attempt to re-create them using the hub cluster manifests + // + // +kubebuilder:default=Always + // +kubebuilder:validation:Enum=Always;IfNotDrifted + // +kubebuilder:validation:Optional + WhenToApply WhenToApplyType `json:"whenToApply,omitempty"` + + // Type is the apply strategy to use; it determines how Fleet applies manifests from the + // hub cluster to a member cluster. + // + // Available options are: + // + // * ClientSideApply: Fleet uses three-way merge to apply manifests, similar to how kubectl + // performs a client-side apply. This is the default option. + // + // Note that this strategy requires that Fleet keep the last applied configuration in the + // annoation of an applied resource. If the object gets so large that apply ops can no longer + // be executed, Fleet will switch to server-side apply. + // + // Use ComparisonOption and WhenToApply settings to control when an apply op can be executed. + // + // * ServerSideApply: Fleet uses server-side apply to apply manifests; Fleet itself will + // become the field manager for specified fields in the manifests. Specify + // ServerSideApplyConfig as appropriate if you would like Fleet to take over field + // ownership upon conflicts. This is the recommended option for most scenarios; it might + // help reduce object size and safely resolve conflicts between field values. For more + // information, please refer to the Kubernetes documentation + // (https://kubernetes.io/docs/reference/using-api/server-side-apply/#comparison-with-client-side-apply). + // + // Use ComparisonOption and WhenToApply settings to control when an apply op can be executed. + // + // * ReportDiff: Fleet will compare the desired state of a resource as kept in the hub cluster + // with its current state (if appliable) on the member cluster side, and report any + // differences. No actual apply ops would be executed, and resources will be left alone as they + // are on the member clusters. + // + // Use ComparisonOption setting to control how the difference is calculated. + // + // For a comparison between the different strategies and usage examples, refer to the + // Fleet documentation. + // // +kubebuilder:default=ClientSideApply // +kubebuilder:validation:Enum=ClientSideApply;ServerSideApply;ReportDiff // +kubebuilder:validation:Optional @@ -453,13 +555,86 @@ type ApplyStrategy struct { // +kubebuilder:validation:Optional ServerSideApplyConfig *ServerSideApplyConfig `json:"serverSideApplyConfig,omitempty"` - // TakeoverAction describes the action to take when we place the selected resources on the target cluster the first time. - // +kubebuilder:default=AlwaysApply - // +kubebuilder:validation:Enum=AlwaysApply;ApplyIfNoDiff + // WhenToTakeOver determines the action to take when Fleet applies resources to a member + // cluster for the first time and finds out that the resource already exists in the cluster. + // + // This setting is most relevant in cases where you would like Fleet to manage pre-existing + // resources on a member cluster. + // + // Available options include: + // + // * Always: with this action, Fleet will apply the hub cluster manifests to the member + // clusters even if the affected resources already exist. This is the default action. + // + // Note that this might lead to fields being overwritten on the member clusters, if they + // are specified in the hub cluster manifests. + // + // * IfNoDiff: with this action, Fleet will apply the hub cluster manifests to the member + // clusters if (and only if) pre-existing resources look the same as the hub cluster manifests. + // This is a safer option as pre-existing resources that are inconsistent with the hub cluster + // manifests will not be overwritten; in fact, Fleet will ignore them until the inconsistencies + // are resolved properly: any change you make to the hub cluster manifests would not be + // applied, and if you delete the manifests or even the ClusterResourcePlacement itself + // from the hub cluster, these pre-existing resources would not be taken away. + // + // Fleet will check for inconsistencies in accordance with the ComparisonOption setting. See also + // the comments on the ComparisonOption field for more information. + // + // If a diff has been found in a field that is **managed** by Fleet (i.e., the field + // **is specified ** in the hub cluster manifest), consider one of the following actions: + // * set the field in the member cluster to be of the same value as that in the hub cluster + // manifest. + // * update the hub cluster manifest so that its field value matches with that in the member + // cluster. + // * switch to the Always action, which will allow Fleet to overwrite the field with the + // value in the hub cluster manifest. + // + // If a diff has been found in a field that is **not managed** by Fleet (i.e., the field + // **is not specified** in the hub cluster manifest), consider one of the following actions: + // * remove the field from the member cluster. + // * update the hub cluster manifest so that the field is included in the hub cluster manifest. + // + // If appropriate, you may also delete the object from the member cluster; Fleet will recreate + // it using the hub cluster manifest. + // + // +kubebuilder:default=Always + // +kubebuilder:validation:Enum=Always;IfNoDiff // +kubebuilder:validation:Optional - TakeoverAction TakeOverActionType `json:"actionType,omitempty"` + WhenToTakeOver WhenToTakeOverType `json:"actionType,omitempty"` } +// ComparisonOptionType describes the compare option that Fleet uses to detect drifts and/or +// calculate differences. +// +enum +type ComparisonOptionType string + +const ( + // ComparisonOptionTypePartialComparison will compare only fields that are managed by Fleet, i.e., + // fields that are specified explicitly in the hub cluster manifest. Unmanaged fields + // are ignored. + ComparisonOptionTypePartialComparison ComparisonOptionType = "PartialDiff" + + // ComparisonOptionTypeFullDiff will compare all fields of the resource, even if the fields + // are absent from the hub cluster manifest. + ComparisonOptionTypeFullComparison ComparisonOptionType = "FullDiff" +) + +// WhenToApplyType describes when Fleet would apply the manifests on the hub cluster to +// the member clusters. +type WhenToApplyType string + +const ( + // WhenToApplyTypeAlways instructs Fleet to periodically apply hub cluster manifests + // on the member cluster side; this will effectively overwrite any change in the fields + // managed by Fleet (i.e., specified in the hub cluster manifest). + WhenToApplyTypeAlways WhenToApplyType = "Always" + + // WhenToApplyTypeIfNotDrifted instructs Fleet to stop applying hub cluster manifests on + // clusters that have drifted from the desired state; apply ops would still continue on + // the rest of the clusters. + WhenToApplyTypeIfNotDrifted WhenToApplyType = "IfNotDrifted" +) + // ApplyStrategyType describes the type of the strategy used to apply the resource to the target cluster. // +enum type ApplyStrategyType string @@ -475,8 +650,9 @@ const ( // Details: https://kubernetes.io/docs/reference/using-api/server-side-apply ApplyStrategyTypeServerSideApply ApplyStrategyType = "ServerSideApply" - // ApplyStrategyTypeReportDiff will generate a report of the difference between - // the resource on the member cluster and to be placed resource snapshot from the hub periodically. + // ApplyStrategyTypeReportDiff will report differences between the desired state of a + // resource as kept in the hub cluster and its current state (if applicable) on the member + // cluster side. No actual apply ops would be executed. ApplyStrategyTypeReportDiff ApplyStrategyType = "ReportDiff" ) @@ -494,19 +670,20 @@ type ServerSideApplyConfig struct { ForceConflicts bool `json:"force"` } -// TakeOverActionType describes the type of the action to take when we first apply the resources to the member cluster. +// WhenToTakeOverType describes the type of the action to take when we first apply the +// resources to the member cluster. // +enum -type TakeOverActionType string +type WhenToTakeOverType string const ( - // TakeOverActionTypeApplyIfNoDiff will apply the yaml from hub cluster only if there is no difference - // between the current resource snapshot version on the hub cluster and the existing resources on the member cluster. - // Otherwise, we will report the difference. - TakeOverActionTypeApplyIfNoDiff TakeOverActionType = "ApplyIfNoDiff" + // WhenToTakeOverTypeIfNoDiff will apply manifests from the hub cluster only if there is no difference + // between the current resource snapshot version on the hub cluster and the existing + // resources on the member cluster. Otherwise, we will report the difference. + WhenToTakeOverTypeIfNoDiff WhenToTakeOverType = "IfNoDiff" - // TakeOverActionTypeAlwaysApply will always apply the resource to the member cluster regardless + // WhenToTakeOverTypeAlways will always apply the resource to the member cluster regardless // if there are differences between the resource on the hub cluster and the existing resources on the member cluster. - TakeOverActionTypeAlwaysApply TakeOverActionType = "AlwaysApply" + WhenToTakeOverTypeAlways WhenToTakeOverType = "Always" ) // +enum diff --git a/config/crd/bases/placement.kubernetes-fleet.io_clusterresourcebindings.yaml b/config/crd/bases/placement.kubernetes-fleet.io_clusterresourcebindings.yaml index 7223f96f2..2a09a6ebf 100644 --- a/config/crd/bases/placement.kubernetes-fleet.io_clusterresourcebindings.yaml +++ b/config/crd/bases/placement.kubernetes-fleet.io_clusterresourcebindings.yaml @@ -429,13 +429,61 @@ spec: and is owned by other appliers. properties: actionType: - default: AlwaysApply - description: TakeoverAction describes the action to take when - we place the selected resources on the target cluster the first - time. + default: Always + description: |- + WhenToTakeOver determines the action to take when Fleet applies resources to a member + cluster for the first time and finds out that the resource already exists in the cluster. + + + This setting is most relevant in cases where you would like Fleet to manage pre-existing + resources on a member cluster. + + + Available options include: + + + * Always: with this action, Fleet will apply the hub cluster manifests to the member + clusters even if the affected resources already exist. This is the default action. + + + Note that this might lead to fields being overwritten on the member clusters, if they + are specified in the hub cluster manifests. + + + * IfNoDiff: with this action, Fleet will apply the hub cluster manifests to the member + clusters if (and only if) pre-existing resources look the same as the hub cluster manifests. + This is a safer option as pre-existing resources that are inconsistent with the hub cluster + manifests will not be overwritten; in fact, Fleet will ignore them until the inconsistencies + are resolved properly: any change you make to the hub cluster manifests would not be + applied, and if you delete the manifests or even the ClusterResourcePlacement itself + from the hub cluster, these pre-existing resources would not be taken away. + + + Fleet will check for inconsistencies in accordance with the ComparisonOption setting. See also + the comments on the ComparisonOption field for more information. + + + If a diff has been found in a field that is **managed** by Fleet (i.e., the field + **is specified ** in the hub cluster manifest), consider one of the following actions: + * set the field in the member cluster to be of the same value as that in the hub cluster + manifest. + * update the hub cluster manifest so that its field value matches with that in the member + cluster. + * switch to the Always action, which will allow Fleet to overwrite the field with the + value in the hub cluster manifest. + + + If a diff has been found in a field that is **not managed** by Fleet (i.e., the field + **is not specified** in the hub cluster manifest), consider one of the following actions: + * remove the field from the member cluster. + * update the hub cluster manifest so that the field is included in the hub cluster manifest. + + + If appropriate, you may also delete the object from the member cluster; Fleet will recreate + it using the hub cluster manifest. enum: - - AlwaysApply - - ApplyIfNoDiff + - Always + - IfNoDiff type: string allowCoOwnership: description: |- @@ -444,6 +492,48 @@ spec: If true, apply the resource and add fleet as a co-owner. If false, leave the resource unchanged and fail the apply. type: boolean + compareOption: + default: PartialComparison + description: |- + ComparisonOption controls how Fleet compares the desired state of a resource, as kept in + a hub cluster manifest, with the current state of the resource (if applicable) in the + member cluster. + + + Available options are: + + + * PartialComparison: with this option, Fleet will compare only fields that are managed by + Fleet, i.e., the fields that are specified explicitly in the hub cluster manifest. + Unmanaged fields are ignored. This is the default option. + + + * FullComparison: with this option, Fleet will compare all fields of the resource, + even if the fields are absent from the hub cluster manifest. + + + Consider using the PartialComparison option if you would like to: + + + * use the default values for certain fields; or + * let another agent, e.g., HPAs, VPAs, etc., on the member cluster side manage some fields; or + * allow ad-hoc or cluster-specific settings on the member cluster side. + + + To use the FullComparison option, it is recommended that you: + + + * specify all fields as appropriate in the hub cluster, even if you are OK with using default + values; + * make sure that no fields are managed by agents other than Fleet on the member cluster + side, such as HPAs, VPAs, or other controllers. + + + See the Fleet documentation for further explanations and usage examples. + enum: + - PartialComparison + - FullComparison + type: string serverSideApplyConfig: description: ServerSideApplyConfig defines the configuration for server side apply. It is honored only when type is ServerSideApply. @@ -463,19 +553,104 @@ spec: type: default: ClientSideApply description: |- - Type defines the type of strategy to use. Default to ClientSideApply. - Server-side apply is more powerful and flexible than client-side apply. - You SHOULD use server-side apply to safely resolve any potential drift between the - original applied resource version and the current resource on the member cluster. - Read more about the differences between server-side apply and client-side apply: - https://kubernetes.io/docs/reference/using-api/server-side-apply/#comparison-with-client-side-apply. - You can also use ReportDiff to only report the difference between the resource on the member cluster - and the resource to be applied from the hub on all the selected clusters. + Type is the apply strategy to use; it determines how Fleet applies manifests from the + hub cluster to a member cluster. + + + Available options are: + + + * ClientSideApply: Fleet uses three-way merge to apply manifests, similar to how kubectl + performs a client-side apply. This is the default option. + + + Note that this strategy requires that Fleet keep the last applied configuration in the + annoation of an applied resource. If the object gets so large that apply ops can no longer + be executed, Fleet will switch to server-side apply. + + + Use ComparisonOption and WhenToApply settings to control when an apply op can be executed. + + + * ServerSideApply: Fleet uses server-side apply to apply manifests; Fleet itself will + become the field manager for specified fields in the manifests. Specify + ServerSideApplyConfig as appropriate if you would like Fleet to take over field + ownership upon conflicts. This is the recommended option for most scenarios; it might + help reduce object size and safely resolve conflicts between field values. For more + information, please refer to the Kubernetes documentation + (https://kubernetes.io/docs/reference/using-api/server-side-apply/#comparison-with-client-side-apply). + + + Use ComparisonOption and WhenToApply settings to control when an apply op can be executed. + + + * ReportDiff: Fleet will compare the desired state of a resource as kept in the hub cluster + with its current state (if appliable) on the member cluster side, and report any + differences. No actual apply ops would be executed, and resources will be left alone as they + are on the member clusters. + + + Use ComparisonOption setting to control how the difference is calculated. + + + For a comparison between the different strategies and usage examples, refer to the + Fleet documentation. enum: - ClientSideApply - ServerSideApply - ReportDiff type: string + whenToApply: + default: Always + description: |- + WhenToApply controls when Fleet would apply the manifests on the hub cluster to the member + clusters. + + + Available options are: + + + * Always: with this option, Fleet will periodically apply hub cluster manifests + on the member cluster side; this will effectively overwrite any change in the fields + managed by Fleet (i.e., specified in the hub cluster manifest). This is the default + option. + + + Note that this option would revert any ad-hoc changes made on the member cluster side in + the managed fields; if you would like to make temporary edits on the member cluster side + in the managed fields, switch to IfNotDrifted option. Note that changes in unmanaged + fields will be left alone; if you use the FullDiff compare option, such changes will + be reported as drifts. + + + * IfNotDrifted: with this option, Fleet will stop applying hub cluster manifests on + clusters that have drifted from the desired state; apply ops would still continue on + the rest of the clusters. Drifts are calculated using the ComparisonOption, + as explained in the corresponding field. + + + Use this option if you would like Fleet to detect drifts in your multi-cluster setup. + A drift occurs when an agent makes an ad-hoc change on the member cluster side that + makes affected resources deviate from its desired state as kept in the hub cluster; + and this option grants you an opportunity to view the drift details and take actions + accordingly. The drift details will be reported in the CRP status. + + + To fix a drift, you may: + + + * revert the changes manually on the member cluster side + * update the hub cluster manifest; this will trigger Fleet to apply the latest revision + of the manifests, which will overwrite the drifted fields + (if they are managed by Fleet) + * switch to the Always option; this will trigger Fleet to apply the current revision + of the manifests, which will overwrite the drifted fields (if they are managed by Fleet). + * if applicable and necessary, delete the drifted resources on the member cluster side; Fleet + will attempt to re-create them using the hub cluster manifests + enum: + - Always + - IfNotDrifted + type: string type: object clusterDecision: description: ClusterDecision explains why the scheduler selected this diff --git a/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacements.yaml b/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacements.yaml index 39beb40cd..3ef0d05dd 100644 --- a/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacements.yaml +++ b/config/crd/bases/placement.kubernetes-fleet.io_clusterresourceplacements.yaml @@ -1757,13 +1757,61 @@ spec: selected resources to the target cluster. properties: actionType: - default: AlwaysApply - description: TakeoverAction describes the action to take when - we place the selected resources on the target cluster the - first time. + default: Always + description: |- + WhenToTakeOver determines the action to take when Fleet applies resources to a member + cluster for the first time and finds out that the resource already exists in the cluster. + + + This setting is most relevant in cases where you would like Fleet to manage pre-existing + resources on a member cluster. + + + Available options include: + + + * Always: with this action, Fleet will apply the hub cluster manifests to the member + clusters even if the affected resources already exist. This is the default action. + + + Note that this might lead to fields being overwritten on the member clusters, if they + are specified in the hub cluster manifests. + + + * IfNoDiff: with this action, Fleet will apply the hub cluster manifests to the member + clusters if (and only if) pre-existing resources look the same as the hub cluster manifests. + This is a safer option as pre-existing resources that are inconsistent with the hub cluster + manifests will not be overwritten; in fact, Fleet will ignore them until the inconsistencies + are resolved properly: any change you make to the hub cluster manifests would not be + applied, and if you delete the manifests or even the ClusterResourcePlacement itself + from the hub cluster, these pre-existing resources would not be taken away. + + + Fleet will check for inconsistencies in accordance with the ComparisonOption setting. See also + the comments on the ComparisonOption field for more information. + + + If a diff has been found in a field that is **managed** by Fleet (i.e., the field + **is specified ** in the hub cluster manifest), consider one of the following actions: + * set the field in the member cluster to be of the same value as that in the hub cluster + manifest. + * update the hub cluster manifest so that its field value matches with that in the member + cluster. + * switch to the Always action, which will allow Fleet to overwrite the field with the + value in the hub cluster manifest. + + + If a diff has been found in a field that is **not managed** by Fleet (i.e., the field + **is not specified** in the hub cluster manifest), consider one of the following actions: + * remove the field from the member cluster. + * update the hub cluster manifest so that the field is included in the hub cluster manifest. + + + If appropriate, you may also delete the object from the member cluster; Fleet will recreate + it using the hub cluster manifest. enum: - - AlwaysApply - - ApplyIfNoDiff + - Always + - IfNoDiff type: string allowCoOwnership: description: |- @@ -1772,6 +1820,48 @@ spec: If true, apply the resource and add fleet as a co-owner. If false, leave the resource unchanged and fail the apply. type: boolean + compareOption: + default: PartialComparison + description: |- + ComparisonOption controls how Fleet compares the desired state of a resource, as kept in + a hub cluster manifest, with the current state of the resource (if applicable) in the + member cluster. + + + Available options are: + + + * PartialComparison: with this option, Fleet will compare only fields that are managed by + Fleet, i.e., the fields that are specified explicitly in the hub cluster manifest. + Unmanaged fields are ignored. This is the default option. + + + * FullComparison: with this option, Fleet will compare all fields of the resource, + even if the fields are absent from the hub cluster manifest. + + + Consider using the PartialComparison option if you would like to: + + + * use the default values for certain fields; or + * let another agent, e.g., HPAs, VPAs, etc., on the member cluster side manage some fields; or + * allow ad-hoc or cluster-specific settings on the member cluster side. + + + To use the FullComparison option, it is recommended that you: + + + * specify all fields as appropriate in the hub cluster, even if you are OK with using default + values; + * make sure that no fields are managed by agents other than Fleet on the member cluster + side, such as HPAs, VPAs, or other controllers. + + + See the Fleet documentation for further explanations and usage examples. + enum: + - PartialComparison + - FullComparison + type: string serverSideApplyConfig: description: ServerSideApplyConfig defines the configuration for server side apply. It is honored only when type is ServerSideApply. @@ -1791,19 +1881,104 @@ spec: type: default: ClientSideApply description: |- - Type defines the type of strategy to use. Default to ClientSideApply. - Server-side apply is more powerful and flexible than client-side apply. - You SHOULD use server-side apply to safely resolve any potential drift between the - original applied resource version and the current resource on the member cluster. - Read more about the differences between server-side apply and client-side apply: - https://kubernetes.io/docs/reference/using-api/server-side-apply/#comparison-with-client-side-apply. - You can also use ReportDiff to only report the difference between the resource on the member cluster - and the resource to be applied from the hub on all the selected clusters. + Type is the apply strategy to use; it determines how Fleet applies manifests from the + hub cluster to a member cluster. + + + Available options are: + + + * ClientSideApply: Fleet uses three-way merge to apply manifests, similar to how kubectl + performs a client-side apply. This is the default option. + + + Note that this strategy requires that Fleet keep the last applied configuration in the + annoation of an applied resource. If the object gets so large that apply ops can no longer + be executed, Fleet will switch to server-side apply. + + + Use ComparisonOption and WhenToApply settings to control when an apply op can be executed. + + + * ServerSideApply: Fleet uses server-side apply to apply manifests; Fleet itself will + become the field manager for specified fields in the manifests. Specify + ServerSideApplyConfig as appropriate if you would like Fleet to take over field + ownership upon conflicts. This is the recommended option for most scenarios; it might + help reduce object size and safely resolve conflicts between field values. For more + information, please refer to the Kubernetes documentation + (https://kubernetes.io/docs/reference/using-api/server-side-apply/#comparison-with-client-side-apply). + + + Use ComparisonOption and WhenToApply settings to control when an apply op can be executed. + + + * ReportDiff: Fleet will compare the desired state of a resource as kept in the hub cluster + with its current state (if appliable) on the member cluster side, and report any + differences. No actual apply ops would be executed, and resources will be left alone as they + are on the member clusters. + + + Use ComparisonOption setting to control how the difference is calculated. + + + For a comparison between the different strategies and usage examples, refer to the + Fleet documentation. enum: - ClientSideApply - ServerSideApply - ReportDiff type: string + whenToApply: + default: Always + description: |- + WhenToApply controls when Fleet would apply the manifests on the hub cluster to the member + clusters. + + + Available options are: + + + * Always: with this option, Fleet will periodically apply hub cluster manifests + on the member cluster side; this will effectively overwrite any change in the fields + managed by Fleet (i.e., specified in the hub cluster manifest). This is the default + option. + + + Note that this option would revert any ad-hoc changes made on the member cluster side in + the managed fields; if you would like to make temporary edits on the member cluster side + in the managed fields, switch to IfNotDrifted option. Note that changes in unmanaged + fields will be left alone; if you use the FullDiff compare option, such changes will + be reported as drifts. + + + * IfNotDrifted: with this option, Fleet will stop applying hub cluster manifests on + clusters that have drifted from the desired state; apply ops would still continue on + the rest of the clusters. Drifts are calculated using the ComparisonOption, + as explained in the corresponding field. + + + Use this option if you would like Fleet to detect drifts in your multi-cluster setup. + A drift occurs when an agent makes an ad-hoc change on the member cluster side that + makes affected resources deviate from its desired state as kept in the hub cluster; + and this option grants you an opportunity to view the drift details and take actions + accordingly. The drift details will be reported in the CRP status. + + + To fix a drift, you may: + + + * revert the changes manually on the member cluster side + * update the hub cluster manifest; this will trigger Fleet to apply the latest revision + of the manifests, which will overwrite the drifted fields + (if they are managed by Fleet) + * switch to the Always option; this will trigger Fleet to apply the current revision + of the manifests, which will overwrite the drifted fields (if they are managed by Fleet). + * if applicable and necessary, delete the drifted resources on the member cluster side; Fleet + will attempt to re-create them using the hub cluster manifests + enum: + - Always + - IfNotDrifted + type: string type: object rollingUpdate: description: Rolling update config params. Present only if RolloutStrategyType diff --git a/config/crd/bases/placement.kubernetes-fleet.io_works.yaml b/config/crd/bases/placement.kubernetes-fleet.io_works.yaml index ea95cdfee..fd318bbed 100644 --- a/config/crd/bases/placement.kubernetes-fleet.io_works.yaml +++ b/config/crd/bases/placement.kubernetes-fleet.io_works.yaml @@ -337,13 +337,61 @@ spec: and is owned by other appliers. properties: actionType: - default: AlwaysApply - description: TakeoverAction describes the action to take when - we place the selected resources on the target cluster the first - time. + default: Always + description: |- + WhenToTakeOver determines the action to take when Fleet applies resources to a member + cluster for the first time and finds out that the resource already exists in the cluster. + + + This setting is most relevant in cases where you would like Fleet to manage pre-existing + resources on a member cluster. + + + Available options include: + + + * Always: with this action, Fleet will apply the hub cluster manifests to the member + clusters even if the affected resources already exist. This is the default action. + + + Note that this might lead to fields being overwritten on the member clusters, if they + are specified in the hub cluster manifests. + + + * IfNoDiff: with this action, Fleet will apply the hub cluster manifests to the member + clusters if (and only if) pre-existing resources look the same as the hub cluster manifests. + This is a safer option as pre-existing resources that are inconsistent with the hub cluster + manifests will not be overwritten; in fact, Fleet will ignore them until the inconsistencies + are resolved properly: any change you make to the hub cluster manifests would not be + applied, and if you delete the manifests or even the ClusterResourcePlacement itself + from the hub cluster, these pre-existing resources would not be taken away. + + + Fleet will check for inconsistencies in accordance with the ComparisonOption setting. See also + the comments on the ComparisonOption field for more information. + + + If a diff has been found in a field that is **managed** by Fleet (i.e., the field + **is specified ** in the hub cluster manifest), consider one of the following actions: + * set the field in the member cluster to be of the same value as that in the hub cluster + manifest. + * update the hub cluster manifest so that its field value matches with that in the member + cluster. + * switch to the Always action, which will allow Fleet to overwrite the field with the + value in the hub cluster manifest. + + + If a diff has been found in a field that is **not managed** by Fleet (i.e., the field + **is not specified** in the hub cluster manifest), consider one of the following actions: + * remove the field from the member cluster. + * update the hub cluster manifest so that the field is included in the hub cluster manifest. + + + If appropriate, you may also delete the object from the member cluster; Fleet will recreate + it using the hub cluster manifest. enum: - - AlwaysApply - - ApplyIfNoDiff + - Always + - IfNoDiff type: string allowCoOwnership: description: |- @@ -352,6 +400,48 @@ spec: If true, apply the resource and add fleet as a co-owner. If false, leave the resource unchanged and fail the apply. type: boolean + compareOption: + default: PartialComparison + description: |- + ComparisonOption controls how Fleet compares the desired state of a resource, as kept in + a hub cluster manifest, with the current state of the resource (if applicable) in the + member cluster. + + + Available options are: + + + * PartialComparison: with this option, Fleet will compare only fields that are managed by + Fleet, i.e., the fields that are specified explicitly in the hub cluster manifest. + Unmanaged fields are ignored. This is the default option. + + + * FullComparison: with this option, Fleet will compare all fields of the resource, + even if the fields are absent from the hub cluster manifest. + + + Consider using the PartialComparison option if you would like to: + + + * use the default values for certain fields; or + * let another agent, e.g., HPAs, VPAs, etc., on the member cluster side manage some fields; or + * allow ad-hoc or cluster-specific settings on the member cluster side. + + + To use the FullComparison option, it is recommended that you: + + + * specify all fields as appropriate in the hub cluster, even if you are OK with using default + values; + * make sure that no fields are managed by agents other than Fleet on the member cluster + side, such as HPAs, VPAs, or other controllers. + + + See the Fleet documentation for further explanations and usage examples. + enum: + - PartialComparison + - FullComparison + type: string serverSideApplyConfig: description: ServerSideApplyConfig defines the configuration for server side apply. It is honored only when type is ServerSideApply. @@ -371,19 +461,104 @@ spec: type: default: ClientSideApply description: |- - Type defines the type of strategy to use. Default to ClientSideApply. - Server-side apply is more powerful and flexible than client-side apply. - You SHOULD use server-side apply to safely resolve any potential drift between the - original applied resource version and the current resource on the member cluster. - Read more about the differences between server-side apply and client-side apply: - https://kubernetes.io/docs/reference/using-api/server-side-apply/#comparison-with-client-side-apply. - You can also use ReportDiff to only report the difference between the resource on the member cluster - and the resource to be applied from the hub on all the selected clusters. + Type is the apply strategy to use; it determines how Fleet applies manifests from the + hub cluster to a member cluster. + + + Available options are: + + + * ClientSideApply: Fleet uses three-way merge to apply manifests, similar to how kubectl + performs a client-side apply. This is the default option. + + + Note that this strategy requires that Fleet keep the last applied configuration in the + annoation of an applied resource. If the object gets so large that apply ops can no longer + be executed, Fleet will switch to server-side apply. + + + Use ComparisonOption and WhenToApply settings to control when an apply op can be executed. + + + * ServerSideApply: Fleet uses server-side apply to apply manifests; Fleet itself will + become the field manager for specified fields in the manifests. Specify + ServerSideApplyConfig as appropriate if you would like Fleet to take over field + ownership upon conflicts. This is the recommended option for most scenarios; it might + help reduce object size and safely resolve conflicts between field values. For more + information, please refer to the Kubernetes documentation + (https://kubernetes.io/docs/reference/using-api/server-side-apply/#comparison-with-client-side-apply). + + + Use ComparisonOption and WhenToApply settings to control when an apply op can be executed. + + + * ReportDiff: Fleet will compare the desired state of a resource as kept in the hub cluster + with its current state (if appliable) on the member cluster side, and report any + differences. No actual apply ops would be executed, and resources will be left alone as they + are on the member clusters. + + + Use ComparisonOption setting to control how the difference is calculated. + + + For a comparison between the different strategies and usage examples, refer to the + Fleet documentation. enum: - ClientSideApply - ServerSideApply - ReportDiff type: string + whenToApply: + default: Always + description: |- + WhenToApply controls when Fleet would apply the manifests on the hub cluster to the member + clusters. + + + Available options are: + + + * Always: with this option, Fleet will periodically apply hub cluster manifests + on the member cluster side; this will effectively overwrite any change in the fields + managed by Fleet (i.e., specified in the hub cluster manifest). This is the default + option. + + + Note that this option would revert any ad-hoc changes made on the member cluster side in + the managed fields; if you would like to make temporary edits on the member cluster side + in the managed fields, switch to IfNotDrifted option. Note that changes in unmanaged + fields will be left alone; if you use the FullDiff compare option, such changes will + be reported as drifts. + + + * IfNotDrifted: with this option, Fleet will stop applying hub cluster manifests on + clusters that have drifted from the desired state; apply ops would still continue on + the rest of the clusters. Drifts are calculated using the ComparisonOption, + as explained in the corresponding field. + + + Use this option if you would like Fleet to detect drifts in your multi-cluster setup. + A drift occurs when an agent makes an ad-hoc change on the member cluster side that + makes affected resources deviate from its desired state as kept in the hub cluster; + and this option grants you an opportunity to view the drift details and take actions + accordingly. The drift details will be reported in the CRP status. + + + To fix a drift, you may: + + + * revert the changes manually on the member cluster side + * update the hub cluster manifest; this will trigger Fleet to apply the latest revision + of the manifests, which will overwrite the drifted fields + (if they are managed by Fleet) + * switch to the Always option; this will trigger Fleet to apply the current revision + of the manifests, which will overwrite the drifted fields (if they are managed by Fleet). + * if applicable and necessary, delete the drifted resources on the member cluster side; Fleet + will attempt to re-create them using the hub cluster manifests + enum: + - Always + - IfNotDrifted + type: string type: object workload: description: Workload represents the manifest workload to be deployed