Skip to content

Commit

Permalink
Add v1->v3 cluster migration (#575)
Browse files Browse the repository at this point in the history 
* Add v1v3 cluster migration

Signed-off-by: Alexandr Demicev <alexandr.demicev@suse.com>

* chore: minor formatting changes on v3 controller

Signed-off-by: Carlos Salas <carlos.salas@suse.com>

---------

Signed-off-by: Alexandr Demicev <alexandr.demicev@suse.com>
Signed-off-by: Carlos Salas <carlos.salas@suse.com>
Co-authored-by: Carlos Salas <carlos.salas@suse.com>
  • Loading branch information
@alexander-demicev @salasberryfin
alexander-demicev and salasberryfin authored Jun 28, 2024
1 parent b759ca2 commit 7bd8932
Show file tree
Hide file tree
Showing 4 changed files with 210 additions and 35 deletions.
1 change: 1 addition & 0 deletions Tiltfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ projects = {
"go.mod",
"go.sum",
"internal",
"features",
],
"kustomize_dir": "config/default",
"label": "turtles"
Expand Down
1 change: 1 addition & 0 deletions internal/controllers/helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,7 @@ const (
ownedLabelName = "cluster-api.cattle.io/owned"
capiClusterOwner = "cluster-api.cattle.io/capi-cluster-owner"
capiClusterOwnerNamespace = "cluster-api.cattle.io/capi-cluster-owner-ns"
v1ClusterMigrated = "cluster-api.cattle.io/migrated"

defaultRequeueDuration = 1 * time.Minute
)
Expand Down
159 changes: 124 additions & 35 deletions internal/controllers/import_controller_v3.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,11 +46,17 @@ import (

"github.com/rancher/turtles/feature"
managementv3 "github.com/rancher/turtles/internal/rancher/management/v3"
provisioningv1 "github.com/rancher/turtles/internal/rancher/provisioning/v1"
"github.com/rancher/turtles/util"
turtlesannotations "github.com/rancher/turtles/util/annotations"
turtlesnaming "github.com/rancher/turtles/util/naming"
turtlespredicates "github.com/rancher/turtles/util/predicates"
)

const (
missingLabelMsg = "missing label"
)

// CAPIImportManagementV3Reconciler represents a reconciler for importing CAPI clusters in Rancher.
type CAPIImportManagementV3Reconciler struct {
Client client.Client
Expand Down Expand Up @@ -92,7 +98,15 @@ func (r *CAPIImportManagementV3Reconciler) SetupWithManager(ctx context.Context,
// Watch Rancher managementv3 clusters
if err := c.Watch(
source.Kind(mgr.GetCache(), &managementv3.Cluster{}),
handler.EnqueueRequestsFromMapFunc(r.rancherClusterToCapiCluster(ctx, capiPredicates)),
handler.EnqueueRequestsFromMapFunc(r.rancherV3ClusterToCapiCluster(ctx, capiPredicates)),
); err != nil {
return fmt.Errorf("adding watch for Rancher cluster: %w", err)
}

// Watch Rancher provisioningv1 clusters that don't have the migrated annotation and are related to a CAPI cluster
if err := c.Watch(
source.Kind(mgr.GetCache(), &provisioningv1.Cluster{}),
handler.EnqueueRequestsFromMapFunc(r.rancherV1ClusterToCapiCluster(ctx, capiPredicates)),
); err != nil {
return fmt.Errorf("adding watch for Rancher cluster: %w", err)
}
Expand Down Expand Up @@ -123,7 +137,7 @@ func (r *CAPIImportManagementV3Reconciler) SetupWithManager(ctx context.Context,
// +kubebuilder:rbac:groups=management.cattle.io,resources=clusters;clusterregistrationtokens;clusterregistrationtokens/status,verbs=get;list;watch

// Reconcile reconciles a CAPI cluster, creating a Rancher cluster if needed and applying the import manifests.
func (r *CAPIImportManagementV3Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (res ctrl.Result, reterr error) {
func (r *CAPIImportManagementV3Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
log := log.FromContext(ctx)
log.Info("Reconciling CAPI cluster")

Expand All @@ -138,7 +152,7 @@ func (r *CAPIImportManagementV3Reconciler) Reconcile(ctx context.Context, req ct

if capiCluster.ObjectMeta.DeletionTimestamp.IsZero() && !turtlesannotations.HasClusterImportAnnotation(capiCluster) &&
!controllerutil.ContainsFinalizer(capiCluster, managementv3.CapiClusterFinalizer) {
log.Info("capi cluster is imported, adding finalizer")
log.Info("CAPI cluster is imported, adding finalizer")
controllerutil.AddFinalizer(capiCluster, managementv3.CapiClusterFinalizer)

if err := r.Client.Update(ctx, capiCluster); err != nil {
Expand Down Expand Up @@ -186,26 +200,25 @@ func (r *CAPIImportManagementV3Reconciler) Reconcile(ctx context.Context, req ct
func (r *CAPIImportManagementV3Reconciler) reconcile(ctx context.Context, capiCluster *clusterv1.Cluster) (ctrl.Result, error) {
log := log.FromContext(ctx)

rancherCluster := &managementv3.Cluster{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
capiClusterOwner: capiCluster.Name,
capiClusterOwnerNamespace: capiCluster.Namespace,
},
},
migrated, err := r.verifyV1ClusterMigration(ctx, capiCluster)
if err != nil || !migrated {
return ctrl.Result{Requeue: true}, err
}

labels := map[string]string{
capiClusterOwner: capiCluster.Name,
capiClusterOwnerNamespace: capiCluster.Namespace,
ownedLabelName: "",
}

rancherCluster := &managementv3.Cluster{}

rancherClusterList := &managementv3.ClusterList{}
selectors := []client.ListOption{
client.MatchingLabels{
capiClusterOwner: capiCluster.Name,
capiClusterOwnerNamespace: capiCluster.Namespace,
ownedLabelName: "",
},
client.MatchingLabels(labels),
}
err := r.RancherClient.List(ctx, rancherClusterList, selectors...)

if client.IgnoreNotFound(err) != nil {
if err := r.RancherClient.List(ctx, rancherClusterList, selectors...); client.IgnoreNotFound(err) != nil {
log.Error(err, fmt.Sprintf("Unable to fetch rancher cluster %s", client.ObjectKeyFromObject(rancherCluster)))
return ctrl.Result{Requeue: true}, err
}
Expand Down Expand Up @@ -283,6 +296,24 @@ func (r *CAPIImportManagementV3Reconciler) reconcileNormal(ctx context.Context,
return ctrl.Result{}, err
}

if feature.Gates.Enabled(feature.PropagateLabels) {
patchBase := client.MergeFromWithOptions(rancherCluster.DeepCopy(), client.MergeFromWithOptimisticLock{})

if rancherCluster.Labels == nil {
rancherCluster.Labels = map[string]string{}
}

for labelKey, labelVal := range capiCluster.Labels {
rancherCluster.Labels[labelKey] = labelVal
}

if err := r.Client.Patch(ctx, rancherCluster, patchBase); err != nil {
return ctrl.Result{}, fmt.Errorf("failed to patch Rancher cluster: %w", err)
}

log.Info("Successfully propagated labels to Rancher cluster")
}

if conditions.IsTrue(rancherCluster, managementv3.ClusterConditionAgentDeployed) {
log.Info("agent already deployed, no action needed")
return ctrl.Result{}, nil
Expand Down Expand Up @@ -312,45 +343,69 @@ func (r *CAPIImportManagementV3Reconciler) reconcileNormal(ctx context.Context,

log.Info("Successfully applied import manifest")

if feature.Gates.Enabled(feature.PropagateLabels) {
patchBase := client.MergeFromWithOptions(rancherCluster.DeepCopy(), client.MergeFromWithOptimisticLock{})
return ctrl.Result{}, nil
}

if rancherCluster.Labels == nil {
rancherCluster.Labels = map[string]string{}
func (r *CAPIImportManagementV3Reconciler) rancherV3ClusterToCapiCluster(ctx context.Context, clusterPredicate predicate.Funcs) handler.MapFunc {
log := log.FromContext(ctx)

return func(_ context.Context, cluster client.Object) []ctrl.Request {
labels := cluster.GetLabels()
if _, ok := labels[ownedLabelName]; !ok { // Ignore clusters that are not owned by turtles
log.V(5).Info(missingLabelMsg+ownedLabelName, "cluster", cluster.GetName())
return nil
}

for labelKey, labelVal := range capiCluster.Labels {
rancherCluster.Labels[labelKey] = labelVal
if _, ok := labels[capiClusterOwner]; !ok {
log.V(5).Info(missingLabelMsg+capiClusterOwner, "cluster", cluster.GetName())
return nil
}

if err := r.Client.Patch(ctx, rancherCluster, patchBase); err != nil {
return ctrl.Result{}, fmt.Errorf("failed to patch Rancher cluster: %w", err)
if _, ok := labels[capiClusterOwnerNamespace]; !ok {
log.V(5).Info(missingLabelMsg+capiClusterOwnerNamespace, "cluster", cluster.GetName())
return nil
}

log.Info("Successfully propagated labels to Rancher cluster")
}
capiCluster := &clusterv1.Cluster{ObjectMeta: metav1.ObjectMeta{
Name: labels[capiClusterOwner],
Namespace: labels[capiClusterOwnerNamespace],
}}

return ctrl.Result{}, nil
if err := r.Client.Get(ctx, client.ObjectKeyFromObject(capiCluster), capiCluster); err != nil {
if !apierrors.IsNotFound(err) {
log.Error(err, "getting capi cluster")
}

return nil
}

if !clusterPredicate.Generic(event.GenericEvent{Object: capiCluster}) {
return nil
}

return []ctrl.Request{{NamespacedName: client.ObjectKey{Namespace: capiCluster.Namespace, Name: capiCluster.Name}}}
}
}

func (r *CAPIImportManagementV3Reconciler) rancherClusterToCapiCluster(ctx context.Context, clusterPredicate predicate.Funcs) handler.MapFunc {
func (r *CAPIImportManagementV3Reconciler) rancherV1ClusterToCapiCluster(ctx context.Context, clusterPredicate predicate.Funcs) handler.MapFunc {
log := log.FromContext(ctx)

return func(_ context.Context, cluster client.Object) []ctrl.Request {
labels := cluster.GetLabels()
if _, ok := labels[capiClusterOwner]; !ok {
log.Error(fmt.Errorf("missing label %s", capiClusterOwner), "getting rancher cluster labels")
if _, ok := labels[ownedLabelName]; !ok { // Ignore clusters that are not owned by turtles
log.V(5).Info(missingLabelMsg+ownedLabelName, "cluster", cluster.GetName())
return nil
}

if _, ok := labels[capiClusterOwnerNamespace]; !ok {
log.Error(fmt.Errorf("missing label %s", capiClusterOwnerNamespace), "getting rancher cluster labels")
annotations := cluster.GetAnnotations()
if _, ok := annotations[v1ClusterMigrated]; ok { // Ignore watching clusters that are already migrated
log.V(5).Info("migrated annotation is present"+v1ClusterMigrated, "cluster", cluster.GetName())
return nil
}

capiCluster := &clusterv1.Cluster{ObjectMeta: metav1.ObjectMeta{
Name: labels[capiClusterOwner],
Namespace: labels[capiClusterOwnerNamespace],
Name: turtlesnaming.Name(cluster.GetName()).ToCapiName(),
Namespace: cluster.GetNamespace(),
}}

if err := r.Client.Get(ctx, client.ObjectKeyFromObject(capiCluster), capiCluster); err != nil {
Expand Down Expand Up @@ -411,3 +466,37 @@ func (r *CAPIImportManagementV3Reconciler) deleteDependentRancherCluster(ctx con

return r.RancherClient.DeleteAllOf(ctx, &managementv3.Cluster{}, selectors...)
}

// verifyV1ClusterMigration verifies if a v1 cluster has been successfully migrated.
// It checks if the v1 cluster exists for a v3 cluster and if it has the "cluster-api.cattle.io/migrated" annotation.
// If the cluster is not migrated yet, it returns false and requeues the reconciliation.
func (r *CAPIImportManagementV3Reconciler) verifyV1ClusterMigration(ctx context.Context, capiCluster *clusterv1.Cluster) (bool, error) {
log := log.FromContext(ctx)

v1rancherCluster := &provisioningv1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Namespace: capiCluster.Namespace,
Name: turtlesnaming.Name(capiCluster.Name).ToRancherName(),
},
}

err := r.RancherClient.Get(ctx, client.ObjectKeyFromObject(v1rancherCluster), v1rancherCluster)
if client.IgnoreNotFound(err) != nil {
log.Error(err, fmt.Sprintf("Unable to fetch rancher cluster %s", client.ObjectKeyFromObject(v1rancherCluster)))
return false, err
}

if apierrors.IsNotFound(err) {
log.V(5).Info("V1 Cluster is migrated or doesn't exist, continuing with v3 reconciliation")

return true, nil
}

if _, present := v1rancherCluster.Annotations[v1ClusterMigrated]; !present {
log.Info("Cluster is not migrated yet, requeue, name", "name", v1rancherCluster.Name)

return false, nil
}

return true, nil
}
84 changes: 84 additions & 0 deletions internal/controllers/import_controller_v3_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,9 @@ import (
. "github.com/onsi/gomega"
"github.com/rancher/turtles/internal/controllers/testdata"
managementv3 "github.com/rancher/turtles/internal/rancher/management/v3"
provisioningv1 "github.com/rancher/turtles/internal/rancher/provisioning/v1"
"github.com/rancher/turtles/internal/test"
turtlesnaming "github.com/rancher/turtles/util/naming"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
Expand All @@ -49,6 +51,7 @@ var _ = Describe("reconcile CAPI Cluster", func() {
rancherClusters *managementv3.ClusterList
rancherCluster *managementv3.Cluster
clusterRegistrationToken *managementv3.ClusterRegistrationToken
v1rancherCluster *provisioningv1.Cluster
capiKubeconfigSecret *corev1.Secret
selectors []client.ListOption
capiClusterName = "generated-rancher-cluster"
Expand Down Expand Up @@ -117,13 +120,21 @@ var _ = Describe("reconcile CAPI Cluster", func() {
secret.KubeconfigDataName: kubeConfigBytes,
},
}

v1rancherCluster = &provisioningv1.Cluster{
ObjectMeta: metav1.ObjectMeta{
Namespace: capiCluster.Namespace,
Name: turtlesnaming.Name(capiCluster.Name).ToRancherName(),
},
}
})

AfterEach(func() {
objs, err := manifestToObjects(strings.NewReader(sampleTemplate))
clientObjs := []client.Object{
capiCluster,
rancherCluster,
v1rancherCluster,
clusterRegistrationToken,
capiKubeconfigSecret,
}
Expand Down Expand Up @@ -450,4 +461,77 @@ var _ = Describe("reconcile CAPI Cluster", func() {
g.Expect(res.Requeue).To(BeTrue())
}).Should(Succeed())
})

It("should reconcile a CAPI Cluster when V1 cluster exists and is migrated", func() {
ns.Labels = map[string]string{}
Expect(cl.Update(ctx, ns)).To(Succeed())
capiCluster.Labels = map[string]string{
importLabelName: "true",
testLabelName: testLabelVal,
}
Expect(cl.Create(ctx, capiCluster)).To(Succeed())
capiCluster.Status.ControlPlaneReady = true
Expect(cl.Status().Update(ctx, capiCluster)).To(Succeed())

Expect(cl.Create(ctx, rancherCluster)).To(Succeed())

Eventually(ctx, func(g Gomega) {
g.Expect(cl.Get(ctx, client.ObjectKeyFromObject(rancherCluster), rancherCluster)).To(Succeed())
conditions.Set(rancherCluster, conditions.TrueCondition(managementv3.ClusterConditionAgentDeployed))
g.Expect(conditions.IsTrue(rancherCluster, managementv3.ClusterConditionAgentDeployed)).To(BeTrue())
g.Expect(cl.Status().Update(ctx, rancherCluster)).To(Succeed())
}).Should(Succeed())

v1rancherCluster.Annotations = map[string]string{
v1ClusterMigrated: "true",
}
Expect(cl.Create(ctx, v1rancherCluster)).To(Succeed())

Eventually(func(g Gomega) {
_, err := r.Reconcile(ctx, reconcile.Request{
NamespacedName: types.NamespacedName{
Namespace: capiCluster.Namespace,
Name: capiCluster.Name,
},
})
g.Expect(err).ToNot(HaveOccurred())
}).Should(Succeed())

Eventually(ctx, func(g Gomega) {
g.Expect(cl.List(ctx, rancherClusters, selectors...)).ToNot(HaveOccurred())
g.Expect(rancherClusters.Items).To(HaveLen(1))
}).Should(Succeed())
Expect(rancherClusters.Items[0].Name).To(ContainSubstring("c-"))
Expect(rancherClusters.Items[0].Labels).To(HaveKeyWithValue(testLabelName, testLabelVal))
})

It("should reconcile a CAPI Cluster when V1 cluster exists and not migrated", func() {
ns.Labels = map[string]string{}
Expect(cl.Update(ctx, ns)).To(Succeed())
capiCluster.Labels = map[string]string{
importLabelName: "true",
testLabelName: testLabelVal,
}
Expect(cl.Create(ctx, capiCluster)).To(Succeed())
capiCluster.Status.ControlPlaneReady = true
Expect(cl.Status().Update(ctx, capiCluster)).To(Succeed())

Expect(cl.Create(ctx, v1rancherCluster)).To(Succeed())

Eventually(func(g Gomega) {
res, err := r.Reconcile(ctx, reconcile.Request{
NamespacedName: types.NamespacedName{
Namespace: capiCluster.Namespace,
Name: capiCluster.Name,
},
})
g.Expect(err).ToNot(HaveOccurred())
g.Expect(res.Requeue).To(BeTrue())
}).Should(Succeed())

Eventually(ctx, func(g Gomega) {
Expect(cl.List(ctx, rancherClusters, selectors...)).ToNot(HaveOccurred())
Expect(rancherClusters.Items).To(HaveLen(0))
}).Should(Succeed())
})
})

0 comments on commit 7bd8932

Please sign in to comment.