From 7bd8932b8bc9c83bc897b8c0df6a2513beabc175 Mon Sep 17 00:00:00 2001 From: Alexander Demicev Date: Fri, 28 Jun 2024 12:28:02 +0300 Subject: [PATCH] Add v1->v3 cluster migration (#575) * Add v1v3 cluster migration Signed-off-by: Alexandr Demicev * chore: minor formatting changes on v3 controller Signed-off-by: Carlos Salas --------- Signed-off-by: Alexandr Demicev Signed-off-by: Carlos Salas Co-authored-by: Carlos Salas --- Tiltfile | 1 + internal/controllers/helpers.go | 1 + internal/controllers/import_controller_v3.go | 159 ++++++++++++++---- .../controllers/import_controller_v3_test.go | 84 +++++++++ 4 files changed, 210 insertions(+), 35 deletions(-) diff --git a/Tiltfile b/Tiltfile index c4139a69..d08109cd 100644 --- a/Tiltfile +++ b/Tiltfile @@ -46,6 +46,7 @@ projects = { "go.mod", "go.sum", "internal", + "features", ], "kustomize_dir": "config/default", "label": "turtles" diff --git a/internal/controllers/helpers.go b/internal/controllers/helpers.go index 93828689..e81ad35e 100644 --- a/internal/controllers/helpers.go +++ b/internal/controllers/helpers.go @@ -49,6 +49,7 @@ const ( ownedLabelName = "cluster-api.cattle.io/owned" capiClusterOwner = "cluster-api.cattle.io/capi-cluster-owner" capiClusterOwnerNamespace = "cluster-api.cattle.io/capi-cluster-owner-ns" + v1ClusterMigrated = "cluster-api.cattle.io/migrated" defaultRequeueDuration = 1 * time.Minute ) diff --git a/internal/controllers/import_controller_v3.go b/internal/controllers/import_controller_v3.go index 529bcf02..3fa92000 100644 --- a/internal/controllers/import_controller_v3.go +++ b/internal/controllers/import_controller_v3.go @@ -46,11 +46,17 @@ import ( "github.com/rancher/turtles/feature" managementv3 "github.com/rancher/turtles/internal/rancher/management/v3" + provisioningv1 "github.com/rancher/turtles/internal/rancher/provisioning/v1" "github.com/rancher/turtles/util" turtlesannotations "github.com/rancher/turtles/util/annotations" + turtlesnaming "github.com/rancher/turtles/util/naming" turtlespredicates "github.com/rancher/turtles/util/predicates" ) +const ( + missingLabelMsg = "missing label" +) + // CAPIImportManagementV3Reconciler represents a reconciler for importing CAPI clusters in Rancher. type CAPIImportManagementV3Reconciler struct { Client client.Client @@ -92,7 +98,15 @@ func (r *CAPIImportManagementV3Reconciler) SetupWithManager(ctx context.Context, // Watch Rancher managementv3 clusters if err := c.Watch( source.Kind(mgr.GetCache(), &managementv3.Cluster{}), - handler.EnqueueRequestsFromMapFunc(r.rancherClusterToCapiCluster(ctx, capiPredicates)), + handler.EnqueueRequestsFromMapFunc(r.rancherV3ClusterToCapiCluster(ctx, capiPredicates)), + ); err != nil { + return fmt.Errorf("adding watch for Rancher cluster: %w", err) + } + + // Watch Rancher provisioningv1 clusters that don't have the migrated annotation and are related to a CAPI cluster + if err := c.Watch( + source.Kind(mgr.GetCache(), &provisioningv1.Cluster{}), + handler.EnqueueRequestsFromMapFunc(r.rancherV1ClusterToCapiCluster(ctx, capiPredicates)), ); err != nil { return fmt.Errorf("adding watch for Rancher cluster: %w", err) } @@ -123,7 +137,7 @@ func (r *CAPIImportManagementV3Reconciler) SetupWithManager(ctx context.Context, // +kubebuilder:rbac:groups=management.cattle.io,resources=clusters;clusterregistrationtokens;clusterregistrationtokens/status,verbs=get;list;watch // Reconcile reconciles a CAPI cluster, creating a Rancher cluster if needed and applying the import manifests. -func (r *CAPIImportManagementV3Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl.Result, reterr error) { +func (r *CAPIImportManagementV3Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { log := log.FromContext(ctx) log.Info("Reconciling CAPI cluster") @@ -138,7 +152,7 @@ func (r *CAPIImportManagementV3Reconciler) Reconcile(ctx context.Context, req ct if capiCluster.ObjectMeta.DeletionTimestamp.IsZero() && !turtlesannotations.HasClusterImportAnnotation(capiCluster) && !controllerutil.ContainsFinalizer(capiCluster, managementv3.CapiClusterFinalizer) { - log.Info("capi cluster is imported, adding finalizer") + log.Info("CAPI cluster is imported, adding finalizer") controllerutil.AddFinalizer(capiCluster, managementv3.CapiClusterFinalizer) if err := r.Client.Update(ctx, capiCluster); err != nil { @@ -186,26 +200,25 @@ func (r *CAPIImportManagementV3Reconciler) Reconcile(ctx context.Context, req ct func (r *CAPIImportManagementV3Reconciler) reconcile(ctx context.Context, capiCluster *clusterv1.Cluster) (ctrl.Result, error) { log := log.FromContext(ctx) - rancherCluster := &managementv3.Cluster{ - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - capiClusterOwner: capiCluster.Name, - capiClusterOwnerNamespace: capiCluster.Namespace, - }, - }, + migrated, err := r.verifyV1ClusterMigration(ctx, capiCluster) + if err != nil || !migrated { + return ctrl.Result{Requeue: true}, err + } + + labels := map[string]string{ + capiClusterOwner: capiCluster.Name, + capiClusterOwnerNamespace: capiCluster.Namespace, + ownedLabelName: "", } + rancherCluster := &managementv3.Cluster{} + rancherClusterList := &managementv3.ClusterList{} selectors := []client.ListOption{ - client.MatchingLabels{ - capiClusterOwner: capiCluster.Name, - capiClusterOwnerNamespace: capiCluster.Namespace, - ownedLabelName: "", - }, + client.MatchingLabels(labels), } - err := r.RancherClient.List(ctx, rancherClusterList, selectors...) - if client.IgnoreNotFound(err) != nil { + if err := r.RancherClient.List(ctx, rancherClusterList, selectors...); client.IgnoreNotFound(err) != nil { log.Error(err, fmt.Sprintf("Unable to fetch rancher cluster %s", client.ObjectKeyFromObject(rancherCluster))) return ctrl.Result{Requeue: true}, err } @@ -283,6 +296,24 @@ func (r *CAPIImportManagementV3Reconciler) reconcileNormal(ctx context.Context, return ctrl.Result{}, err } + if feature.Gates.Enabled(feature.PropagateLabels) { + patchBase := client.MergeFromWithOptions(rancherCluster.DeepCopy(), client.MergeFromWithOptimisticLock{}) + + if rancherCluster.Labels == nil { + rancherCluster.Labels = map[string]string{} + } + + for labelKey, labelVal := range capiCluster.Labels { + rancherCluster.Labels[labelKey] = labelVal + } + + if err := r.Client.Patch(ctx, rancherCluster, patchBase); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to patch Rancher cluster: %w", err) + } + + log.Info("Successfully propagated labels to Rancher cluster") + } + if conditions.IsTrue(rancherCluster, managementv3.ClusterConditionAgentDeployed) { log.Info("agent already deployed, no action needed") return ctrl.Result{}, nil @@ -312,45 +343,69 @@ func (r *CAPIImportManagementV3Reconciler) reconcileNormal(ctx context.Context, log.Info("Successfully applied import manifest") - if feature.Gates.Enabled(feature.PropagateLabels) { - patchBase := client.MergeFromWithOptions(rancherCluster.DeepCopy(), client.MergeFromWithOptimisticLock{}) + return ctrl.Result{}, nil +} - if rancherCluster.Labels == nil { - rancherCluster.Labels = map[string]string{} +func (r *CAPIImportManagementV3Reconciler) rancherV3ClusterToCapiCluster(ctx context.Context, clusterPredicate predicate.Funcs) handler.MapFunc { + log := log.FromContext(ctx) + + return func(_ context.Context, cluster client.Object) []ctrl.Request { + labels := cluster.GetLabels() + if _, ok := labels[ownedLabelName]; !ok { // Ignore clusters that are not owned by turtles + log.V(5).Info(missingLabelMsg+ownedLabelName, "cluster", cluster.GetName()) + return nil } - for labelKey, labelVal := range capiCluster.Labels { - rancherCluster.Labels[labelKey] = labelVal + if _, ok := labels[capiClusterOwner]; !ok { + log.V(5).Info(missingLabelMsg+capiClusterOwner, "cluster", cluster.GetName()) + return nil } - if err := r.Client.Patch(ctx, rancherCluster, patchBase); err != nil { - return ctrl.Result{}, fmt.Errorf("failed to patch Rancher cluster: %w", err) + if _, ok := labels[capiClusterOwnerNamespace]; !ok { + log.V(5).Info(missingLabelMsg+capiClusterOwnerNamespace, "cluster", cluster.GetName()) + return nil } - log.Info("Successfully propagated labels to Rancher cluster") - } + capiCluster := &clusterv1.Cluster{ObjectMeta: metav1.ObjectMeta{ + Name: labels[capiClusterOwner], + Namespace: labels[capiClusterOwnerNamespace], + }} - return ctrl.Result{}, nil + if err := r.Client.Get(ctx, client.ObjectKeyFromObject(capiCluster), capiCluster); err != nil { + if !apierrors.IsNotFound(err) { + log.Error(err, "getting capi cluster") + } + + return nil + } + + if !clusterPredicate.Generic(event.GenericEvent{Object: capiCluster}) { + return nil + } + + return []ctrl.Request{{NamespacedName: client.ObjectKey{Namespace: capiCluster.Namespace, Name: capiCluster.Name}}} + } } -func (r *CAPIImportManagementV3Reconciler) rancherClusterToCapiCluster(ctx context.Context, clusterPredicate predicate.Funcs) handler.MapFunc { +func (r *CAPIImportManagementV3Reconciler) rancherV1ClusterToCapiCluster(ctx context.Context, clusterPredicate predicate.Funcs) handler.MapFunc { log := log.FromContext(ctx) return func(_ context.Context, cluster client.Object) []ctrl.Request { labels := cluster.GetLabels() - if _, ok := labels[capiClusterOwner]; !ok { - log.Error(fmt.Errorf("missing label %s", capiClusterOwner), "getting rancher cluster labels") + if _, ok := labels[ownedLabelName]; !ok { // Ignore clusters that are not owned by turtles + log.V(5).Info(missingLabelMsg+ownedLabelName, "cluster", cluster.GetName()) return nil } - if _, ok := labels[capiClusterOwnerNamespace]; !ok { - log.Error(fmt.Errorf("missing label %s", capiClusterOwnerNamespace), "getting rancher cluster labels") + annotations := cluster.GetAnnotations() + if _, ok := annotations[v1ClusterMigrated]; ok { // Ignore watching clusters that are already migrated + log.V(5).Info("migrated annotation is present"+v1ClusterMigrated, "cluster", cluster.GetName()) return nil } capiCluster := &clusterv1.Cluster{ObjectMeta: metav1.ObjectMeta{ - Name: labels[capiClusterOwner], - Namespace: labels[capiClusterOwnerNamespace], + Name: turtlesnaming.Name(cluster.GetName()).ToCapiName(), + Namespace: cluster.GetNamespace(), }} if err := r.Client.Get(ctx, client.ObjectKeyFromObject(capiCluster), capiCluster); err != nil { @@ -411,3 +466,37 @@ func (r *CAPIImportManagementV3Reconciler) deleteDependentRancherCluster(ctx con return r.RancherClient.DeleteAllOf(ctx, &managementv3.Cluster{}, selectors...) } + +// verifyV1ClusterMigration verifies if a v1 cluster has been successfully migrated. +// It checks if the v1 cluster exists for a v3 cluster and if it has the "cluster-api.cattle.io/migrated" annotation. +// If the cluster is not migrated yet, it returns false and requeues the reconciliation. +func (r *CAPIImportManagementV3Reconciler) verifyV1ClusterMigration(ctx context.Context, capiCluster *clusterv1.Cluster) (bool, error) { + log := log.FromContext(ctx) + + v1rancherCluster := &provisioningv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: capiCluster.Namespace, + Name: turtlesnaming.Name(capiCluster.Name).ToRancherName(), + }, + } + + err := r.RancherClient.Get(ctx, client.ObjectKeyFromObject(v1rancherCluster), v1rancherCluster) + if client.IgnoreNotFound(err) != nil { + log.Error(err, fmt.Sprintf("Unable to fetch rancher cluster %s", client.ObjectKeyFromObject(v1rancherCluster))) + return false, err + } + + if apierrors.IsNotFound(err) { + log.V(5).Info("V1 Cluster is migrated or doesn't exist, continuing with v3 reconciliation") + + return true, nil + } + + if _, present := v1rancherCluster.Annotations[v1ClusterMigrated]; !present { + log.Info("Cluster is not migrated yet, requeue, name", "name", v1rancherCluster.Name) + + return false, nil + } + + return true, nil +} diff --git a/internal/controllers/import_controller_v3_test.go b/internal/controllers/import_controller_v3_test.go index 156d6c36..53010ac0 100644 --- a/internal/controllers/import_controller_v3_test.go +++ b/internal/controllers/import_controller_v3_test.go @@ -27,7 +27,9 @@ import ( . "github.com/onsi/gomega" "github.com/rancher/turtles/internal/controllers/testdata" managementv3 "github.com/rancher/turtles/internal/rancher/management/v3" + provisioningv1 "github.com/rancher/turtles/internal/rancher/provisioning/v1" "github.com/rancher/turtles/internal/test" + turtlesnaming "github.com/rancher/turtles/util/naming" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" @@ -49,6 +51,7 @@ var _ = Describe("reconcile CAPI Cluster", func() { rancherClusters *managementv3.ClusterList rancherCluster *managementv3.Cluster clusterRegistrationToken *managementv3.ClusterRegistrationToken + v1rancherCluster *provisioningv1.Cluster capiKubeconfigSecret *corev1.Secret selectors []client.ListOption capiClusterName = "generated-rancher-cluster" @@ -117,6 +120,13 @@ var _ = Describe("reconcile CAPI Cluster", func() { secret.KubeconfigDataName: kubeConfigBytes, }, } + + v1rancherCluster = &provisioningv1.Cluster{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: capiCluster.Namespace, + Name: turtlesnaming.Name(capiCluster.Name).ToRancherName(), + }, + } }) AfterEach(func() { @@ -124,6 +134,7 @@ var _ = Describe("reconcile CAPI Cluster", func() { clientObjs := []client.Object{ capiCluster, rancherCluster, + v1rancherCluster, clusterRegistrationToken, capiKubeconfigSecret, } @@ -450,4 +461,77 @@ var _ = Describe("reconcile CAPI Cluster", func() { g.Expect(res.Requeue).To(BeTrue()) }).Should(Succeed()) }) + + It("should reconcile a CAPI Cluster when V1 cluster exists and is migrated", func() { + ns.Labels = map[string]string{} + Expect(cl.Update(ctx, ns)).To(Succeed()) + capiCluster.Labels = map[string]string{ + importLabelName: "true", + testLabelName: testLabelVal, + } + Expect(cl.Create(ctx, capiCluster)).To(Succeed()) + capiCluster.Status.ControlPlaneReady = true + Expect(cl.Status().Update(ctx, capiCluster)).To(Succeed()) + + Expect(cl.Create(ctx, rancherCluster)).To(Succeed()) + + Eventually(ctx, func(g Gomega) { + g.Expect(cl.Get(ctx, client.ObjectKeyFromObject(rancherCluster), rancherCluster)).To(Succeed()) + conditions.Set(rancherCluster, conditions.TrueCondition(managementv3.ClusterConditionAgentDeployed)) + g.Expect(conditions.IsTrue(rancherCluster, managementv3.ClusterConditionAgentDeployed)).To(BeTrue()) + g.Expect(cl.Status().Update(ctx, rancherCluster)).To(Succeed()) + }).Should(Succeed()) + + v1rancherCluster.Annotations = map[string]string{ + v1ClusterMigrated: "true", + } + Expect(cl.Create(ctx, v1rancherCluster)).To(Succeed()) + + Eventually(func(g Gomega) { + _, err := r.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: capiCluster.Namespace, + Name: capiCluster.Name, + }, + }) + g.Expect(err).ToNot(HaveOccurred()) + }).Should(Succeed()) + + Eventually(ctx, func(g Gomega) { + g.Expect(cl.List(ctx, rancherClusters, selectors...)).ToNot(HaveOccurred()) + g.Expect(rancherClusters.Items).To(HaveLen(1)) + }).Should(Succeed()) + Expect(rancherClusters.Items[0].Name).To(ContainSubstring("c-")) + Expect(rancherClusters.Items[0].Labels).To(HaveKeyWithValue(testLabelName, testLabelVal)) + }) + + It("should reconcile a CAPI Cluster when V1 cluster exists and not migrated", func() { + ns.Labels = map[string]string{} + Expect(cl.Update(ctx, ns)).To(Succeed()) + capiCluster.Labels = map[string]string{ + importLabelName: "true", + testLabelName: testLabelVal, + } + Expect(cl.Create(ctx, capiCluster)).To(Succeed()) + capiCluster.Status.ControlPlaneReady = true + Expect(cl.Status().Update(ctx, capiCluster)).To(Succeed()) + + Expect(cl.Create(ctx, v1rancherCluster)).To(Succeed()) + + Eventually(func(g Gomega) { + res, err := r.Reconcile(ctx, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: capiCluster.Namespace, + Name: capiCluster.Name, + }, + }) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(res.Requeue).To(BeTrue()) + }).Should(Succeed()) + + Eventually(ctx, func(g Gomega) { + Expect(cl.List(ctx, rancherClusters, selectors...)).ToNot(HaveOccurred()) + Expect(rancherClusters.Items).To(HaveLen(0)) + }).Should(Succeed()) + }) })