From add88107f31b4690c9a58fe7d824a443a22154f8 Mon Sep 17 00:00:00 2001 From: Danil Grigorev Date: Wed, 25 Sep 2024 17:19:26 +0200 Subject: [PATCH] Use local clusterctl overrides for determining latest version for provider (#751) * Refactor auto-upgrade functionality Signed-off-by: Danil-Grigorev * Notify users about version update being available Signed-off-by: Danil-Grigorev * Add integration tests for custom providers Signed-off-by: Danil-Grigorev --------- Signed-off-by: Danil-Grigorev --- api/v1alpha1/clusterctl_config_types.go | 11 +- api/v1alpha1/conditions_consts.go | 8 + .../templates/rancher-turtles-components.yaml | 13 +- ...tles-capi.cattle.io_clusterctlconfigs.yaml | 13 +- go.mod | 2 +- internal/controllers/clusterctl/config.go | 124 ++++++++++++++- .../clusterctlconfig_controller.go | 37 +---- internal/sync/provider_sync.go | 67 +++++--- internal/sync/provider_sync_test.go | 150 +++++++++++++++++- internal/test/helpers/envtest.go | 10 ++ main.go | 3 +- test/e2e/helpers.go | 2 + .../chart-upgrade/chart_upgrade_test.go | 23 +++ test/framework/turtles.go | 52 ++++++ 14 files changed, 427 insertions(+), 88 deletions(-) create mode 100644 test/framework/turtles.go diff --git a/api/v1alpha1/clusterctl_config_types.go b/api/v1alpha1/clusterctl_config_types.go index eca18ff7..957abe52 100644 --- a/api/v1alpha1/clusterctl_config_types.go +++ b/api/v1alpha1/clusterctl_config_types.go @@ -30,10 +30,12 @@ const ( //nolint:lll type ClusterctlConfigSpec struct { // Images is a list of image overrided for specified providers - Images []Image `json:"images"` + // +optional + Images []Image `json:"images,omitempty"` // Provider overrides - Providers ProviderList `json:"providers"` + // +optional + Providers ProviderList `json:"providers,omitempty"` } // Provider allows to define providers with known URLs to pull the components. @@ -48,9 +50,8 @@ type Provider struct { // Type is the type of the provider // +required - // +kubebuilder:validation:Enum=infrastructure;core;controlPlane;bootstrap;addon;runtimeextension;ipam - // +kubebuilder:example=infrastructure - ProviderType Type `json:"type"` + // +kubebuilder:example=InfrastructureProvider + Type string `json:"type"` } // ProviderList is a list of providers. diff --git a/api/v1alpha1/conditions_consts.go b/api/v1alpha1/conditions_consts.go index 6e3ed1ef..60ae8798 100644 --- a/api/v1alpha1/conditions_consts.go +++ b/api/v1alpha1/conditions_consts.go @@ -34,3 +34,11 @@ const ( // CheckLatestVersionTime is set as a timestamp info of the last timestamp of the latest version being up-to-date for the CAPIProvider. CheckLatestVersionTime = "CheckLatestVersionTime" ) + +const ( + // CheckLatestUpdateAvailableReason is a reason for a False condition, due to update being available. + CheckLatestUpdateAvailableReason = "UpdateAvailable" + + // CheckLatestProviderUnknownReason is a reason for an Unknown condition, due to provider not being available. + CheckLatestProviderUnknownReason = "ProviderUnknown" +) diff --git a/charts/rancher-turtles/templates/rancher-turtles-components.yaml b/charts/rancher-turtles/templates/rancher-turtles-components.yaml index b7b29350..8bd16164 100644 --- a/charts/rancher-turtles/templates/rancher-turtles-components.yaml +++ b/charts/rancher-turtles/templates/rancher-turtles-components.yaml @@ -3170,15 +3170,7 @@ spec: type: string type: description: Type is the type of the provider - enum: - - infrastructure - - core - - controlPlane - - bootstrap - - addon - - runtimeextension - - ipam - example: infrastructure + example: InfrastructureProvider type: string url: description: URL of the provider components. Will be used unless @@ -3190,9 +3182,6 @@ spec: - url type: object type: array - required: - - images - - providers type: object type: object x-kubernetes-validations: diff --git a/config/crd/bases/turtles-capi.cattle.io_clusterctlconfigs.yaml b/config/crd/bases/turtles-capi.cattle.io_clusterctlconfigs.yaml index 62f5c373..98e26348 100644 --- a/config/crd/bases/turtles-capi.cattle.io_clusterctlconfigs.yaml +++ b/config/crd/bases/turtles-capi.cattle.io_clusterctlconfigs.yaml @@ -74,15 +74,7 @@ spec: type: string type: description: Type is the type of the provider - enum: - - infrastructure - - core - - controlPlane - - bootstrap - - addon - - runtimeextension - - ipam - example: infrastructure + example: InfrastructureProvider type: string url: description: URL of the provider components. Will be used unless @@ -94,9 +86,6 @@ spec: - url type: object type: array - required: - - images - - providers type: object type: object x-kubernetes-validations: diff --git a/go.mod b/go.mod index 553b6e42..cf6b08c2 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/rancher/turtles go 1.22.0 require ( + github.com/blang/semver/v4 v4.0.0 github.com/go-logr/logr v1.4.2 github.com/onsi/ginkgo/v2 v2.20.0 github.com/onsi/gomega v1.34.1 @@ -25,7 +26,6 @@ require ( require ( github.com/MakeNowJust/heredoc v1.0.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/blang/semver/v4 v4.0.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.12.0 // indirect diff --git a/internal/controllers/clusterctl/config.go b/internal/controllers/clusterctl/config.go index 79e7b7b2..edb7c167 100644 --- a/internal/controllers/clusterctl/config.go +++ b/internal/controllers/clusterctl/config.go @@ -18,13 +18,22 @@ package clusterctl import ( "cmp" + "context" + "fmt" "os" + "slices" + "strings" _ "embed" + "github.com/blang/semver/v4" corev1 "k8s.io/api/core/v1" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/yaml" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" + + turtlesv1 "github.com/rancher/turtles/api/v1alpha1" ) var ( @@ -34,14 +43,127 @@ var ( config *corev1.ConfigMap ) +const ( + latestVersionKey = "latest" +) + func init() { utilruntime.Must(yaml.UnmarshalStrict(configDefault, &config)) } -// Config returns current set of turtles clusterctl overrides. +// ConfigRepository is a direct clusterctl config repository representation. +type ConfigRepository struct { + Providers turtlesv1.ProviderList `json:"providers"` + Images map[string]ConfigImage `json:"images"` +} + +// ConfigImage is a direct clusterctl representation of image config value. +type ConfigImage struct { + // Repository sets the container registry override to pull images from. + Repository string `json:"repository,omitempty"` + + // Tag allows to specify a tag for the images. + Tag string `json:"tag,omitempty"` +} + +// Config returns current set of embedded turtles clusterctl overrides. func Config() *corev1.ConfigMap { configMap := config.DeepCopy() configMap.Namespace = cmp.Or(os.Getenv("POD_NAMESPACE"), "rancher-turtles-system") return configMap } + +// ClusterConfig collects overrides config from the local in-memory state +// and the user-specified ClusterctlConfig overrides layer. +func ClusterConfig(ctx context.Context, c client.Client) (*ConfigRepository, error) { + log := log.FromContext(ctx) + + configMap := Config() + + config := &turtlesv1.ClusterctlConfig{} + if err := c.Get(ctx, client.ObjectKeyFromObject(configMap), config); client.IgnoreNotFound(err) != nil { + log.Error(err, "Unable to collect ClusterctlConfig resource") + + return nil, err + } + + clusterctlConfig := &ConfigRepository{} + if err := yaml.UnmarshalStrict([]byte(configMap.Data["clusterctl.yaml"]), &clusterctlConfig); err != nil { + log.Error(err, "Unable to deserialize initial clusterctl config") + + return nil, err + } + + if clusterctlConfig.Images == nil { + clusterctlConfig.Images = map[string]ConfigImage{} + } + + clusterctlConfig.Providers = append(clusterctlConfig.Providers, config.Spec.Providers...) + + for _, image := range config.Spec.Images { + clusterctlConfig.Images[image.Name] = ConfigImage{ + Tag: image.Tag, + Repository: image.Repository, + } + } + + return clusterctlConfig, nil +} + +// GetProviderVersion collects version of the collected provider overrides state. +// Returns latest if the version is not found. +func (r *ConfigRepository) GetProviderVersion(ctx context.Context, name, providerType string) (version string, providerKnown bool) { + for _, provider := range r.Providers { + if provider.Name == name && strings.EqualFold(provider.Type, providerType) { + return collectVersion(ctx, provider.URL), true + } + } + + return latestVersionKey, false +} + +func collectVersion(ctx context.Context, url string) string { + version := strings.Split(url, "/") + slices.Reverse(version) + + if len(version) < 2 { + log.FromContext(ctx).Info("Provider url is invalid for version resolve, defaulting to latest", "url", url) + + return latestVersionKey + } + + return version[1] +} + +// IsLatestVersion checks version against the expected max version, and returns false +// if the version given is newer then the latest in the clusterctlconfig override. +func (r *ConfigRepository) IsLatestVersion(providerVersion, expected string) (bool, error) { + // Return true for providers without version boundary or unknown providers + if providerVersion == latestVersionKey { + return true, nil + } + + version, _ := strings.CutPrefix(providerVersion, "v") + + maxVersion, err := semver.Parse(version) + if err != nil { + return false, fmt.Errorf("unable to parse default provider version %s: %w", providerVersion, err) + } + + expected = cmp.Or(expected, latestVersionKey) + if expected == latestVersionKey { + // Latest should be reduced to the actual version set on the clusterctlprovider resource + return false, nil + } + + version, _ = strings.CutPrefix(expected, "v") + + desiredVersion, err := semver.Parse(version) + if err != nil { + return false, fmt.Errorf("unable to parse desired version %s: %w", expected, err) + } + + // Disallow versions beyond current clusterctl.yaml override default + return maxVersion.LTE(desiredVersion), nil +} diff --git a/internal/controllers/clusterctlconfig_controller.go b/internal/controllers/clusterctlconfig_controller.go index 2f6627a6..36a13302 100644 --- a/internal/controllers/clusterctlconfig_controller.go +++ b/internal/controllers/clusterctlconfig_controller.go @@ -87,44 +87,22 @@ func (r *ClusterctlConfigReconciler) SetupWithManager(ctx context.Context, mgr c return nil } -//+kubebuilder:rbac:groups=turtles-capi.cattle.io,resources=clusterctlcofigs,verbs=get;list;watch;patch -//+kubebuilder:rbac:groups=turtles-capi.cattle.io,resources=clusterctlcofigs/status,verbs=get;list;watch;patch -//+kubebuilder:rbac:groups=turtles-capi.cattle.io,resources=clusterctlcofigs/finalizers,verbs=get;list;watch;patch;update +//+kubebuilder:rbac:groups=turtles-capi.cattle.io,resources=clusterctlconfigs,verbs=get;list;watch;patch +//+kubebuilder:rbac:groups=turtles-capi.cattle.io,resources=clusterctlconfigs/status,verbs=get;list;watch;patch +//+kubebuilder:rbac:groups=turtles-capi.cattle.io,resources=clusterctlconfigs/finalizers,verbs=get;list;watch;patch;update //+kubebuilder:rbac:groups="",resources=configmaps,verbs=get;list;watch;create;patch // Reconcile reconciles the EtcdMachineSnapshot object. -func (r *ClusterctlConfigReconciler) Reconcile(ctx context.Context, req reconcile.Request) (ctrl.Result, error) { +func (r *ClusterctlConfigReconciler) Reconcile(ctx context.Context, _ reconcile.Request) (ctrl.Result, error) { log := log.FromContext(ctx) - configMap := clusterctl.Config() - - config := &turtlesv1.ClusterctlConfig{} - if err := r.Client.Get(ctx, req.NamespacedName, config); client.IgnoreNotFound(err) != nil { - log.Error(err, "Unable to collect ClusterctlConfig resource") - - return ctrl.Result{}, err - } - - clusterctlConfig := &Config{} - if err := yaml.UnmarshalStrict([]byte(configMap.Data["clusterctl.yaml"]), &clusterctlConfig); err != nil { - log.Error(err, "Unable to deserialize initial clusterctl config") + clusterctlConfig, err := clusterctl.ClusterConfig(ctx, r.Client) + if err != nil { + log.Error(err, "Unable to serialize updated clusterctl config") return ctrl.Result{}, err } - if clusterctlConfig.Images == nil { - clusterctlConfig.Images = map[string]ConfigImage{} - } - - clusterctlConfig.Providers = append(clusterctlConfig.Providers, config.Spec.Providers...) - - for _, image := range config.Spec.Images { - clusterctlConfig.Images[image.Name] = ConfigImage{ - Tag: image.Tag, - Repository: image.Repository, - } - } - clusterctlYaml, err := yaml.Marshal(clusterctlConfig) if err != nil { log.Error(err, "Unable to serialize updated clusterctl config") @@ -132,6 +110,7 @@ func (r *ClusterctlConfigReconciler) Reconcile(ctx context.Context, req reconcil return ctrl.Result{}, err } + configMap := clusterctl.Config() configMap.Data["clusterctl.yaml"] = string(clusterctlYaml) if err := r.Client.Patch(ctx, configMap, client.Apply, []client.PatchOption{ diff --git a/internal/sync/provider_sync.go b/internal/sync/provider_sync.go index dad56cd1..616a952b 100644 --- a/internal/sync/provider_sync.go +++ b/internal/sync/provider_sync.go @@ -17,19 +17,23 @@ limitations under the License. package sync import ( + "cmp" "context" + "fmt" "strings" "time" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/log" operatorv1 "sigs.k8s.io/cluster-api-operator/api/v1alpha2" + clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" "sigs.k8s.io/cluster-api/util/conditions" turtlesv1 "github.com/rancher/turtles/api/v1alpha1" "github.com/rancher/turtles/internal/api" + "github.com/rancher/turtles/internal/controllers/clusterctl" ) // AppliedSpecHashAnnotation is a spec hash annotation set by CAPI Operator, @@ -91,9 +95,13 @@ func (ProviderSync) Template(capiProvider *turtlesv1.CAPIProvider) client.Object // Spec -> down // up <- Status. func (s *ProviderSync) Sync(ctx context.Context) error { + if err := s.updateLatestVersion(ctx); err != nil { + return err + } + s.SyncObjects() - return s.updateLatestVersion(ctx) + return nil } // SyncObjects updates the Source CAPIProvider object and the destination provider object states. @@ -152,31 +160,48 @@ func (s *ProviderSync) rolloutInfrastructure() { } func (s *ProviderSync) updateLatestVersion(ctx context.Context) error { - // Skip for user specified versions - if s.Source.Spec.Version != "" { - return nil - } + log := log.FromContext(ctx) - now := time.Now().UTC() - lastCheck := conditions.Get(s.Source, turtlesv1.CheckLatestVersionTime) - - if lastCheck != nil && lastCheck.Status == corev1.ConditionTrue && lastCheck.LastTransitionTime.Add(24*time.Hour).After(now) { - return nil + config, err := clusterctl.ClusterConfig(ctx, s.client) + if err != nil { + return err } - patchBase := client.MergeFrom(s.Destination) + providerVersion, knownProvider := config.GetProviderVersion(ctx, cmp.Or(s.Source.Spec.Name, s.Source.Name), s.Source.Spec.Type.ToKind()) - // Unsetting .spec.version to force latest version rollout - spec := s.Destination.GetSpec() - spec.Version = "" - s.Destination.SetSpec(spec) + latest, err := config.IsLatestVersion(providerVersion, s.Source.Spec.Version) + if err != nil { + return err + } - conditions.MarkTrue(s.Source, turtlesv1.CheckLatestVersionTime) + switch { + case !knownProvider: + conditions.MarkUnknown(s.Source, turtlesv1.CheckLatestVersionTime, turtlesv1.CheckLatestProviderUnknownReason, "Provider is unknown") + case s.Source.Spec.Version != "" && latest: + conditions.MarkTrue(s.Source, turtlesv1.CheckLatestVersionTime) + case s.Source.Spec.Version != "" && !latest: + conditions.MarkFalse( + s.Source, + turtlesv1.CheckLatestVersionTime, + turtlesv1.CheckLatestUpdateAvailableReason, + clusterv1.ConditionSeverityInfo, + "Provider version update available. Current latest is %s", providerVersion, + ) + case !latest: + lastCheck := conditions.Get(s.Source, turtlesv1.CheckLatestVersionTime) + updatedMessage := fmt.Sprintf("Updated to latest %s version", providerVersion) + + if lastCheck == nil || lastCheck.Message != updatedMessage { + log.Info(fmt.Sprintf("Version %s is beyond current latest, updated to %s", cmp.Or(s.Source.Spec.Version, "latest"), providerVersion)) + + lastCheck = conditions.TrueCondition(turtlesv1.CheckLatestVersionTime) + lastCheck.Message = updatedMessage + + conditions.Set(s.Source, lastCheck) + } - err := s.client.Patch(ctx, s.Destination, patchBase) - if err != nil { - conditions.MarkUnknown(s.Source, turtlesv1.CheckLatestVersionTime, "Requesting latest version rollout", "") + s.Source.Spec.Version = providerVersion } - return client.IgnoreNotFound(err) + return nil } diff --git a/internal/sync/provider_sync_test.go b/internal/sync/provider_sync_test.go index 7b7ca647..eb2c0343 100644 --- a/internal/sync/provider_sync_test.go +++ b/internal/sync/provider_sync_test.go @@ -17,6 +17,7 @@ limitations under the License. package sync_test import ( + "os" "time" . "github.com/onsi/ginkgo/v2" @@ -40,12 +41,17 @@ var _ = Describe("Provider sync", func() { ns *corev1.Namespace otherNs *corev1.Namespace capiProvider *turtlesv1.CAPIProvider + customCAPIProvider *turtlesv1.CAPIProvider + unknownCAPIProvider *turtlesv1.CAPIProvider capiProviderAzure *turtlesv1.CAPIProvider capiProviderDuplicate *turtlesv1.CAPIProvider infrastructure *operatorv1.InfrastructureProvider + customInfrastructure *operatorv1.InfrastructureProvider + unknownInfrastructure *operatorv1.InfrastructureProvider infrastructureStatusOutdated operatorv1.ProviderStatus infrastructureDuplicate *operatorv1.InfrastructureProvider infrastructureAzure *operatorv1.InfrastructureProvider + clusterctlconfig *turtlesv1.ClusterctlConfig ) BeforeEach(func() { @@ -70,6 +76,14 @@ var _ = Describe("Provider sync", func() { capiProviderAzure.Spec.Name = "azure" capiProviderAzure.Name = "azure" + customCAPIProvider = capiProvider.DeepCopy() + customCAPIProvider.Name = "custom-provider" + customCAPIProvider.Spec.Name = "custom-provider" + + unknownCAPIProvider = capiProvider.DeepCopy() + unknownCAPIProvider.Name = "unknown-provider" + unknownCAPIProvider.Spec.Name = "unknown-provider" + capiProviderDuplicate = capiProvider.DeepCopy() capiProviderDuplicate.Namespace = otherNs.Name @@ -78,6 +92,16 @@ var _ = Describe("Provider sync", func() { Namespace: ns.Name, }} + customInfrastructure = &operatorv1.InfrastructureProvider{ObjectMeta: metav1.ObjectMeta{ + Name: string(customCAPIProvider.Spec.Name), + Namespace: ns.Name, + }} + + unknownInfrastructure = &operatorv1.InfrastructureProvider{ObjectMeta: metav1.ObjectMeta{ + Name: string(unknownCAPIProvider.Spec.Name), + Namespace: ns.Name, + }} + infrastructureAzure = &operatorv1.InfrastructureProvider{ObjectMeta: metav1.ObjectMeta{ Name: string(capiProviderAzure.Spec.Name), Namespace: ns.Name, @@ -92,6 +116,7 @@ var _ = Describe("Provider sync", func() { Conditions: clusterv1.Conditions{ { Type: turtlesv1.CheckLatestVersionTime, + Message: "Updated to latest v1.4.6 version", Status: corev1.ConditionTrue, LastTransitionTime: metav1.NewTime(time.Now().UTC().Truncate(time.Second).Add(-23 * time.Hour)), }, @@ -103,18 +128,134 @@ var _ = Describe("Provider sync", func() { }, } + clusterctlconfig = &turtlesv1.ClusterctlConfig{ + ObjectMeta: metav1.ObjectMeta{ + Name: turtlesv1.ClusterctlConfigName, + Namespace: ns.Name, + }, + Spec: turtlesv1.ClusterctlConfigSpec{ + Providers: turtlesv1.ProviderList{{ + Name: "custom-provider", + URL: "https://github.com/org/repo/releases/v1.2.3/components.yaml", + Type: "InfrastructureProvider", + }}, + }, + } + + os.Setenv("POD_NAMESPACE", ns.Name) + Expect(testEnv.Client.Create(ctx, capiProvider)).To(Succeed()) Expect(testEnv.Client.Create(ctx, capiProviderDuplicate)).To(Succeed()) Expect(testEnv.Client.Create(ctx, capiProviderAzure)).To(Succeed()) + Expect(testEnv.Client.Create(ctx, customCAPIProvider)).To(Succeed()) + Expect(testEnv.Client.Create(ctx, unknownCAPIProvider)).To(Succeed()) + Expect(testEnv.Client.Create(ctx, clusterctlconfig)).To(Succeed()) }) AfterEach(func() { testEnv.Cleanup(ctx, ns, otherNs) }) - It("Should sync spec down", func() { + It("Should sync spec down and set version to latest", func() { + s := sync.NewProviderSync(testEnv, capiProvider.DeepCopy()) + + expected := capiProvider.DeepCopy() + expected.Spec.Version = "v1.7.3" + + Eventually(func(g Gomega) { + g.Expect(s.Get(ctx)).To(Succeed()) + g.Expect(s.Sync(ctx)).To(Succeed()) + var err error = nil + s.Apply(ctx, &err) + g.Expect(err).To(Succeed()) + }).Should(Succeed()) + + Eventually(Object(infrastructure)).Should( + HaveField("Spec.ProviderSpec", Equal(expected.Spec.ProviderSpec))) + }) + + It("Should create unknown provider to clusterctl override with unchanged 'latest' version", func() { + s := sync.NewProviderSync(testEnv, unknownCAPIProvider.DeepCopy()) + + expected := unknownCAPIProvider.DeepCopy() + + Eventually(func(g Gomega) { + g.Expect(s.Get(ctx)).To(Succeed()) + g.Expect(s.Sync(ctx)).To(Succeed()) + var err error = nil + s.Apply(ctx, &err) + g.Expect(err).To(Succeed()) + g.Expect(conditions.IsUnknown(expected, turtlesv1.CheckLatestVersionTime)).To(BeTrue()) + }).Should(Succeed()) + + Eventually(Object(unknownInfrastructure)).Should( + HaveField("Spec.ProviderSpec", Equal(expected.Spec.ProviderSpec))) + }) + + It("Should create unknown provider to clusterctl override with unchanged specific version", func() { + expected := unknownCAPIProvider.DeepCopy() + expected.Spec.Version = "v1.0.0" + s := sync.NewProviderSync(testEnv, expected) + + Eventually(func(g Gomega) { + g.Expect(s.Get(ctx)).To(Succeed()) + g.Expect(s.Sync(ctx)).To(Succeed()) + var err error = nil + s.Apply(ctx, &err) + g.Expect(err).To(Succeed()) + g.Expect(conditions.IsTrue(expected, turtlesv1.LastAppliedConfigurationTime)).To(BeTrue()) + g.Expect(conditions.IsUnknown(expected, turtlesv1.CheckLatestVersionTime)).To(BeTrue()) + }).Should(Succeed()) + + Eventually(Object(unknownInfrastructure)).Should( + HaveField("Spec.ProviderSpec", Equal(expected.Spec.ProviderSpec))) + }) + + It("Should set custom provider version to latest according to clusterctlconfig override", func() { + s := sync.NewProviderSync(testEnv, customCAPIProvider.DeepCopy()) + + expected := customCAPIProvider.DeepCopy() + expected.Spec.Version = "v1.2.3" + + Eventually(func(g Gomega) { + g.Expect(s.Get(ctx)).To(Succeed()) + g.Expect(s.Sync(ctx)).To(Succeed()) + var err error = nil + s.Apply(ctx, &err) + g.Expect(err).To(Succeed()) + }).Should(Succeed()) + + Eventually(Object(customInfrastructure)).Should( + HaveField("Spec.ProviderSpec", Equal(expected.Spec.ProviderSpec))) + }) + + It("Should not change custom provider version even if it is in the clusterctlconfig override", func() { + expected := customCAPIProvider.DeepCopy() + expected.Spec.Version = "v1.0.0" + s := sync.NewProviderSync(testEnv, expected) + + Eventually(func(g Gomega) { + g.Expect(s.Get(ctx)).To(Succeed()) + g.Expect(s.Sync(ctx)).To(Succeed()) + var err error = nil + s.Apply(ctx, &err) + g.Expect(err).To(Succeed()) + g.Expect(conditions.IsTrue(expected, turtlesv1.LastAppliedConfigurationTime)).To(BeTrue()) + g.Expect(conditions.IsFalse(expected, turtlesv1.CheckLatestVersionTime)).To(BeTrue()) + }).Should(Succeed()) + + Eventually(Object(customInfrastructure)).Should( + HaveField("Spec.ProviderSpec", Equal(expected.Spec.ProviderSpec))) + Consistently(Object(customInfrastructure)).Should( + HaveField("Spec.ProviderSpec", Equal(expected.Spec.ProviderSpec))) + }) + + It("Should sync spec down and set version to latest", func() { s := sync.NewProviderSync(testEnv, capiProvider.DeepCopy()) + expected := capiProvider.DeepCopy() + expected.Spec.Version = "v1.7.3" + Eventually(func(g Gomega) { g.Expect(s.Get(ctx)).To(Succeed()) g.Expect(s.Sync(ctx)).To(Succeed()) @@ -124,7 +265,7 @@ var _ = Describe("Provider sync", func() { }).Should(Succeed()) Eventually(Object(infrastructure)).Should( - HaveField("Spec.ProviderSpec", Equal(capiProvider.Spec.ProviderSpec))) + HaveField("Spec.ProviderSpec", Equal(expected.Spec.ProviderSpec))) }) It("Should sync azure spec", func() { @@ -198,13 +339,10 @@ var _ = Describe("Provider sync", func() { s.Apply(ctx, &err) g.Expect(err).ToNot(HaveOccurred()) g.Expect(testEnv.Get(ctx, client.ObjectKeyFromObject(infrastructure), dest)).To(Succeed()) - g.Expect(dest.GetAnnotations()).To(HaveKeyWithValue(sync.AppliedSpecHashAnnotation, "")) - g.Expect(testEnv.Get(ctx, client.ObjectKeyFromObject(infrastructure), dest)).To(Succeed()) g.Expect(capiProvider.Status.Conditions).To(HaveLen(2)) g.Expect(conditions.IsTrue(capiProvider, turtlesv1.LastAppliedConfigurationTime)).To(BeTrue()) g.Expect(conditions.IsTrue(capiProvider, turtlesv1.CheckLatestVersionTime)).To(BeTrue()) - g.Expect(conditions.Get(capiProvider, turtlesv1.CheckLatestVersionTime).LastTransitionTime.Equal( - &lastVersionCheckCondition.LastTransitionTime)).To(BeTrue()) + g.Expect(conditions.Get(capiProvider, turtlesv1.CheckLatestVersionTime).Message).To(Equal("Updated to latest v1.7.3 version")) g.Expect(conditions.Get(capiProvider, turtlesv1.LastAppliedConfigurationTime).LastTransitionTime.After( appliedCondition.LastTransitionTime.Time)).To(BeTrue()) }).Should(Succeed()) diff --git a/internal/test/helpers/envtest.go b/internal/test/helpers/envtest.go index 96250a25..c1a628fe 100644 --- a/internal/test/helpers/envtest.go +++ b/internal/test/helpers/envtest.go @@ -47,6 +47,8 @@ import ( clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1" logf "sigs.k8s.io/cluster-api/cmd/clusterctl/log" + + turtlesv1 "github.com/rancher/turtles/api/v1alpha1" ) var root string @@ -63,6 +65,7 @@ func init() { utilruntime.Must(apiextensionsv1.AddToScheme(clientgoscheme.Scheme)) utilruntime.Must(admissionv1.AddToScheme(clientgoscheme.Scheme)) utilruntime.Must(clusterv1.AddToScheme(clientgoscheme.Scheme)) + utilruntime.Must(turtlesv1.AddToScheme(clientgoscheme.Scheme)) // Get the root of the current file to use in CRD paths. _, filename, _, _ := goruntime.Caller(0) //nolint @@ -168,6 +171,13 @@ func (t *TestEnvironmentConfiguration) Build() (*TestEnvironment, error) { Metrics: server.Options{ BindAddress: "0", }, + Client: client.Options{ + Cache: &client.CacheOptions{ + DisableFor: []client.Object{ + &turtlesv1.ClusterctlConfig{}, + }, + }, + }, } mgr, err := ctrl.NewManager(t.env.Config, options) diff --git a/main.go b/main.go index b5af5806..c91f8115 100644 --- a/main.go +++ b/main.go @@ -150,6 +150,7 @@ func main() { DisableFor: []client.Object{ &corev1.ConfigMap{}, &corev1.Secret{}, + &turtlesv1.ClusterctlConfig{}, }, }, }, @@ -285,7 +286,7 @@ func setupReconcilers(ctx context.Context, mgr ctrl.Manager) { setupLog.Info("enabling Clusterctl Config synchronization controller") if err := (&controllers.ClusterctlConfigReconciler{ - Client: uncachedClient, + Client: mgr.GetClient(), }).SetupWithManager(ctx, mgr, controller.Options{ MaxConcurrentReconciles: concurrencyNumber, CacheSyncTimeout: maxDuration, diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go index 3773a1d0..e169bdec 100644 --- a/test/e2e/helpers.go +++ b/test/e2e/helpers.go @@ -29,6 +29,7 @@ import ( . "github.com/onsi/gomega" + turtlesv1 "github.com/rancher/turtles/api/v1alpha1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -84,6 +85,7 @@ func DumpSpecResourcesAndCleanup(ctx context.Context, specName string, clusterPr func InitScheme() *runtime.Scheme { scheme := runtime.NewScheme() framework.TryAddDefaultSchemes(scheme) + Expect(turtlesv1.AddToScheme(scheme)).To(Succeed()) Expect(operatorv1.AddToScheme(scheme)).To(Succeed()) Expect(clusterv1.AddToScheme(scheme)).To(Succeed()) Expect(provisioningv1.AddToScheme(scheme)).To(Succeed()) diff --git a/test/e2e/suites/chart-upgrade/chart_upgrade_test.go b/test/e2e/suites/chart-upgrade/chart_upgrade_test.go index 17a7b1fb..645d41bd 100644 --- a/test/e2e/suites/chart-upgrade/chart_upgrade_test.go +++ b/test/e2e/suites/chart-upgrade/chart_upgrade_test.go @@ -101,6 +101,29 @@ var _ = Describe("Chart upgrade functionality should work", Label(e2e.ShortTestL Expect(setupClusterResult.BootstrapClusterProxy.Apply(ctx, e2e.AddonProviderFleetHostNetworkPatch)).To(Succeed()) }) + upgradeInput.PostUpgradeSteps = append(upgradeInput.PostUpgradeSteps, func() { + framework.WaitForCAPIProviderRollout(ctx, framework.WaitForCAPIProviderRolloutInput{ + Getter: setupClusterResult.BootstrapClusterProxy.GetClient(), + Version: "v1.7.3", + Name: "cluster-api", + Namespace: "capi-system", + }, e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers")...) + }, func() { + framework.WaitForCAPIProviderRollout(ctx, framework.WaitForCAPIProviderRolloutInput{ + Getter: setupClusterResult.BootstrapClusterProxy.GetClient(), + Version: "v1.7.3", + Name: "kubeadm-control-plane", + Namespace: "capi-kubeadm-control-plane-system", + }, e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers")...) + }, func() { + framework.WaitForCAPIProviderRollout(ctx, framework.WaitForCAPIProviderRolloutInput{ + Getter: setupClusterResult.BootstrapClusterProxy.GetClient(), + Version: "v1.7.3", + Name: "docker", + Namespace: "capd-system", + }, e2eConfig.GetIntervals(setupClusterResult.BootstrapClusterProxy.GetName(), "wait-controllers")...) + }) + testenv.UpgradeRancherTurtles(ctx, upgradeInput) }) }) diff --git a/test/framework/turtles.go b/test/framework/turtles.go new file mode 100644 index 00000000..3ac6e4c9 --- /dev/null +++ b/test/framework/turtles.go @@ -0,0 +1,52 @@ +/* +Copyright © 2023 - 2024 SUSE LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package framework + +import ( + "context" + + "k8s.io/apimachinery/pkg/types" + "k8s.io/klog/v2" + + turtlesv1 "github.com/rancher/turtles/api/v1alpha1" + capiframework "sigs.k8s.io/cluster-api/test/framework" + + . "github.com/onsi/gomega" +) + +type WaitForCAPIProviderRolloutInput struct { + capiframework.Getter + Name, Namespace, Version string +} + +func WaitForCAPIProviderRollout(ctx context.Context, input WaitForCAPIProviderRolloutInput, intervals ...interface{}) { + capiProvider := &turtlesv1.CAPIProvider{} + key := types.NamespacedName{ + Name: input.Name, + Namespace: input.Namespace, + } + + Byf("Waiting for CAPIProvider %s to be at version %s", key.String(), input.Version) + + Eventually(func(g Gomega) { + g.Expect(input.Getter.Get(ctx, key, capiProvider)).To(Succeed()) + g.Expect(capiProvider.Status.InstalledVersion).ToNot(BeNil()) + g.Expect(*capiProvider.Status.InstalledVersion).To(Equal(input.Version)) + }, intervals...).Should(Succeed(), + "Failed to get CAPIProvider %s with version %s. Last observed: %s", + key.String(), input.Version, klog.KObj(capiProvider)) +}