Create a garbage collection reconciler for machine

pkg/controllers/machine/controller.go

@@ -17,7 +17,9 @@ package machine
 import (
 	"context"
 	"fmt"
+	"time"
 
+	"github.com/patrickmn/go-cache"
 	"github.com/samber/lo"
 	"go.uber.org/multierr"
 	v1 "k8s.io/api/core/v1"
@@ -57,6 +59,7 @@ type Controller struct {
 	recorder      events.Recorder
 	terminator    *terminator.Terminator
 
+	garbageCollect *GarbageCollect
 	launch         *Launch
 	registration   *Registration
 	initialization *Initialization
@@ -72,6 +75,7 @@ func NewController(clk clock.Clock, kubeClient client.Client, cloudProvider clou
 		recorder:      recorder,
 		terminator:    terminator,
 
+		garbageCollect: &GarbageCollect{kubeClient: kubeClient, cloudProvider: cloudProvider, lastChecked: cache.New(time.Minute*10, 1*time.Minute)},
 		launch:         &Launch{kubeClient: kubeClient, cloudProvider: cloudProvider},
 		registration:   &Registration{kubeClient: kubeClient},
 		initialization: &Initialization{kubeClient: kubeClient},
@@ -98,6 +102,7 @@ func (c *Controller) Reconcile(ctx context.Context, machine *v1alpha5.Machine) (
 	var results []reconcile.Result
 	var errs error
 	for _, reconciler := range []machineReconciler{
+		c.garbageCollect,
 		c.launch,
 		c.registration,
 		c.initialization,

pkg/controllers/machine/garbagecollect.go

@@ -0,0 +1,51 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package machine
+
+import (
+	"context"
+	"time"
+
+	"github.com/patrickmn/go-cache"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+	"github.com/aws/karpenter-core/pkg/apis/v1alpha5"
+	"github.com/aws/karpenter-core/pkg/cloudprovider"
+)
+
+type GarbageCollect struct {
+	kubeClient    client.Client
+	cloudProvider cloudprovider.CloudProvider
+	lastChecked   *cache.Cache
+}
+
+func (e *GarbageCollect) Reconcile(ctx context.Context, machine *v1alpha5.Machine) (reconcile.Result, error) {
+	if !machine.StatusConditions().GetCondition(v1alpha5.MachineCreated).IsTrue() {
+		return reconcile.Result{}, nil
+	}
+	// If there is no node representation for the machine, then check if there is a representation at the cloudprovider
+	if _, err := nodeForMachine(ctx, e.kubeClient, machine); err == nil || !IsNodeNotFoundError(err) {
+		return reconcile.Result{}, nil
+	}
+	if _, expireTime, ok := e.lastChecked.GetWithExpiration(client.ObjectKeyFromObject(machine).String()); ok {
+		return reconcile.Result{RequeueAfter: time.Until(expireTime)}, nil
+	}
+	if _, err := e.cloudProvider.Get(ctx, machine.Name, machine.Labels[v1alpha5.ProvisionerNameLabelKey]); cloudprovider.IsMachineNotFoundError(err) {
+		return reconcile.Result{}, client.IgnoreNotFound(e.kubeClient.Delete(ctx, machine))
+	}
+	e.lastChecked.SetDefault(client.ObjectKeyFromObject(machine).String(), nil)
+	return reconcile.Result{}, nil
+}

pkg/controllers/machine/garbagecollect_test.go

@@ -0,0 +1,80 @@
+/*
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package machine_test
+
+import (
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+
+	"github.com/aws/karpenter-core/pkg/apis/v1alpha5"
+	"github.com/aws/karpenter-core/pkg/test"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	. "github.com/aws/karpenter-core/pkg/test/expectations"
+)
+
+var _ = Describe("GarbageCollection", func() {
+	var provisioner *v1alpha5.Provisioner
+
+	BeforeEach(func() {
+		provisioner = test.Provisioner()
+	})
+	It("should delete the Machine when the Node never appears and the instance is gone", func() {
+		machine := test.Machine(v1alpha5.Machine{
+			ObjectMeta: metav1.ObjectMeta{
+				Labels: map[string]string{
+					v1alpha5.ProvisionerNameLabelKey: provisioner.Name,
+				},
+			},
+		})
+		ExpectApplied(ctx, env.Client, provisioner, machine)
+		ExpectReconcileSucceeded(ctx, machineController, client.ObjectKeyFromObject(machine))
+		machine = ExpectExists(ctx, env.Client, machine)
+
+		// Delete the machine from the cloudprovider
+		Expect(cloudProvider.Delete(ctx, machine)).To(Succeed())
+
+		// Wait for the cache expiration to complete
+		time.Sleep(time.Second)
+
+		// Expect the Machine to be removed now that the Instance is gone
+		ExpectReconcileSucceeded(ctx, machineController, client.ObjectKeyFromObject(machine))
+		ExpectReconcileSucceeded(ctx, machineController, client.ObjectKeyFromObject(machine)) // Reconcile again to handle termination flow
+		ExpectNotFound(ctx, env.Client, machine)
+	})
+	It("shouldn't delete the Machine when the Node isn't there but the instance is there", func() {
+		machine := test.Machine(v1alpha5.Machine{
+			ObjectMeta: metav1.ObjectMeta{
+				Labels: map[string]string{
+					v1alpha5.ProvisionerNameLabelKey: provisioner.Name,
+				},
+			},
+		})
+		ExpectApplied(ctx, env.Client, provisioner, machine)
+		ExpectReconcileSucceeded(ctx, machineController, client.ObjectKeyFromObject(machine))
+		machine = ExpectExists(ctx, env.Client, machine)
+
+		// Wait for the cache expiration to complete
+		time.Sleep(time.Second)
+
+		// Reconcile the Machine. It should not be deleted by this flow since it has never been registered
+		ExpectReconcileSucceeded(ctx, machineController, client.ObjectKeyFromObject(machine))
+		ExpectExists(ctx, env.Client, machine)
+	})
+})

pkg/controllers/machine/initialization.go

@@ -43,6 +43,9 @@ func (i *Initialization) Reconcile(ctx context.Context, machine *v1alpha5.Machin
 	if machine.Status.ProviderID == "" {
 		return reconcile.Result{}, nil
 	}
+	if machine.StatusConditions().GetCondition(v1alpha5.MachineInitialized).IsTrue() {
+		return reconcile.Result{}, nil
+	}
 	ctx = logging.WithLogger(ctx, logging.FromContext(ctx).With("provider-id", machine.Status.ProviderID))
 	node, err := nodeForMachine(ctx, i.kubeClient, machine)
 	if err != nil {