Merge pull request #2241 from loft-sh/conformance-debugging-2

Justfile

@@ -169,9 +169,7 @@ build-dev-image tag="":
   docker build -t vcluster:dev-{{tag}} -f Dockerfile.release --build-arg TARGETARCH=$(uname -m) --build-arg TARGETOS=linux .
   rm ./vcluster
 
-run-conformance k8s_version="1.31.1" mode="conformance-lite" tag="conf": (build-dev-image tag)
-  minikube start --kubernetes-version {{k8s_version}} --nodes=2
-  minikube addons enable metrics-server
+run-conformance k8s_version="1.31.1" mode="conformance-lite" tag="conf": (create-conformance k8s_version) (build-dev-image tag)
   minikube image load vcluster:dev-{{tag}}
 
   vcluster create vcluster -n vcluster -f ./conformance/v1.31/vcluster.yaml
@@ -183,3 +181,15 @@ conformance-status:
 
 conformance-logs:
   sonobuoy logs
+
+dev-conformance *ARGS:
+  devspace dev --profile test-conformance --namespace vcluster {{ARGS}}
+
+create-conformance k8s_version="1.31.1":
+  minikube start --kubernetes-version {{k8s_version}} --nodes=2
+  minikube addons enable metrics-server
+
+delete-conformance:
+  -minikube delete
+
+recreate-conformance: delete-conformance create-conformance

devspace.yaml

@@ -7,6 +7,7 @@ vars:
   DEVSPACE_FLAGS: "-n vcluster"
   SYNCER_IMAGE: ghcr.io/loft-sh/loft-enterprise/dev-vcluster
   COMMON_VALUES: ./test/commonValues.yaml
+  CONFORMANCE_VALUES: ./test/conformanceValues.yaml
   PRO_VALUES: ./test/proValues.yaml
   VALUES_FILE: ./test/e2e/values.yaml
   RELEASE_NAME: "vcluster"
@@ -163,6 +164,10 @@ pipelines:
       # Build the vcluster image
       build_images --all
 
+      if $(is_in "test-conformance" "${DEVSPACE_PROFILES}"); then
+        minikube image load $(get_image "vcluster") --overwrite=false || true
+      fi
+
       # Deploy the vcluster
       if is_equal $(get_flag distro) k8s; then
         create_deployments vcluster-k8s
@@ -216,6 +221,17 @@ profiles:
             - ${COMMON_VALUES}
             - ${VALUES_FILE}
             - ${PRO_VALUES}
+  - name: test-conformance
+    patches:
+      - op: replace
+        path: images.vcluster.buildKit
+        value:
+          preferMinikube: false
+      - op: add
+        path: deployments.vcluster-k8s.helm
+        value:
+          valuesFiles:
+            - ${CONFORMANCE_VALUES}
 
 commands:
   # e.g. devspace run test k3s

pkg/syncer/handler.go

@@ -10,7 +10,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/handler"
 )
 
-type enqueueFunc func(ctx context.Context, obj client.Object, q workqueue.TypedRateLimitingInterface[ctrl.Request], isDelete bool)
+type enqueueFunc func(ctx context.Context, obj client.Object, q workqueue.TypedRateLimitingInterface[ctrl.Request], isDelete, isPendingDelete bool)
 
 func newEventHandler(enqueue enqueueFunc) handler.EventHandler {
 	return &eventHandler{enqueue: enqueue}
@@ -22,21 +22,21 @@ type eventHandler struct {
 
 // Create is called in response to an create event - e.g. Pod Creation.
 func (r *eventHandler) Create(ctx context.Context, evt event.CreateEvent, q workqueue.TypedRateLimitingInterface[ctrl.Request]) {
-	r.enqueue(ctx, evt.Object, q, false)
+	r.enqueue(ctx, evt.Object, q, false, false)
 }
 
 // Update is called in response to an update event -  e.g. Pod Updated.
 func (r *eventHandler) Update(ctx context.Context, evt event.UpdateEvent, q workqueue.TypedRateLimitingInterface[ctrl.Request]) {
-	r.enqueue(ctx, evt.ObjectNew, q, false)
+	r.enqueue(ctx, evt.ObjectNew, q, false, !evt.ObjectNew.GetDeletionTimestamp().IsZero())
 }
 
 // Delete is called in response to a delete event - e.g. Pod Deleted.
 func (r *eventHandler) Delete(ctx context.Context, evt event.DeleteEvent, q workqueue.TypedRateLimitingInterface[ctrl.Request]) {
-	r.enqueue(ctx, evt.Object, q, true)
+	r.enqueue(ctx, evt.Object, q, true, false)
 }
 
 // Generic is called in response to an event of an unknown type or a synthetic event triggered as a cron or
 // external trigger request - e.g. reconcile Autoscaling, or a Webhook.
 func (r *eventHandler) Generic(ctx context.Context, evt event.GenericEvent, q workqueue.TypedRateLimitingInterface[ctrl.Request]) {
-	r.enqueue(ctx, evt.Object, q, false)
+	r.enqueue(ctx, evt.Object, q, false, !evt.Object.GetDeletionTimestamp().IsZero())
 }

pkg/syncer/synccontext/events.go

@@ -5,8 +5,9 @@ import "sigs.k8s.io/controller-runtime/pkg/client"
 type SyncEventType string
 
 const (
-	SyncEventTypeUnknown SyncEventType = ""
-	SyncEventTypeDelete  SyncEventType = "Delete"
+	SyncEventTypeUnknown       SyncEventType = ""
+	SyncEventTypeDelete        SyncEventType = "Delete"
+	SyncEventTypePendingDelete SyncEventType = "PendingDelete"
 )
 
 type SyncEventSource string

pkg/syncer/syncer.go

@@ -31,8 +31,9 @@ import (
 )
 
 const (
-	hostObjectRequestPrefix   = "host#"
-	deleteObjectRequestPrefix = "delete#"
+	hostObjectRequestPrefix          = "host#"
+	deleteObjectRequestPrefix        = "delete#"
+	pendingDeleteObjectRequestPrefix = "pendingdelete#"
 )
 
 func NewSyncController(ctx *synccontext.RegisterContext, syncer syncertypes.Syncer) (*SyncController, error) {
@@ -142,10 +143,11 @@ func (r *SyncController) Reconcile(ctx context.Context, origReq ctrl.Request) (_
 	// same time.
 	//
 	// This is FIFO, we use a special mutex for this (fifomu.Mutex)
-	r.locker.Lock(vReq.String())
-	defer func() {
-		_ = r.locker.Unlock(vReq.String())
-	}()
+	lockKey := vReq.String()
+	r.locker.Lock(lockKey)
+	defer func(lockKey string) {
+		_ = r.locker.Unlock(lockKey)
+	}(lockKey)
 
 	// check if we should skip reconcile
 	lifecycle, ok := r.syncer.(syncertypes.Starter)
@@ -214,6 +216,10 @@ func (r *SyncController) Reconcile(ctx context.Context, origReq ctrl.Request) (_
 			}
 		}
 
+		if syncEventSource == synccontext.SyncEventSourceVirtual && syncEventType == synccontext.SyncEventTypePendingDelete {
+			return ctrl.Result{}, nil
+		}
+
 		return r.genericSyncer.SyncToVirtual(syncContext, &synccontext.SyncToVirtualEvent[client.Object]{
 			Type:   syncEventType,
 			Source: syncEventSource,
@@ -398,7 +404,7 @@ func (r *SyncController) extractRequest(ctx *synccontext.SyncContext, req ctrl.R
 	return req, pReq, nil
 }
 
-func (r *SyncController) enqueueVirtual(_ context.Context, obj client.Object, q workqueue.TypedRateLimitingInterface[ctrl.Request], isDelete bool) {
+func (r *SyncController) enqueueVirtual(_ context.Context, obj client.Object, q workqueue.TypedRateLimitingInterface[ctrl.Request], isDelete, isPendingDelete bool) {
 	if obj == nil {
 		return
 	}
@@ -416,6 +422,19 @@ func (r *SyncController) enqueueVirtual(_ context.Context, obj client.Object, q
 		return
 	}
 
+	// add a new request for the host object as otherwise this information might be lost after update + delete event
+	if isPendingDelete {
+		// add a new request for the virtual object
+		q.Add(toPendingDeleteRequest(reconcile.Request{
+			NamespacedName: types.NamespacedName{
+				Namespace: obj.GetNamespace(),
+				Name:      obj.GetName(),
+			},
+		}))
+
+		return
+	}
+
 	// add a new request for the virtual object
 	q.Add(reconcile.Request{
 		NamespacedName: types.NamespacedName{
@@ -425,7 +444,7 @@ func (r *SyncController) enqueueVirtual(_ context.Context, obj client.Object, q
 	})
 }
 
-func (r *SyncController) enqueuePhysical(ctx context.Context, obj client.Object, q workqueue.TypedRateLimitingInterface[ctrl.Request], isDelete bool) {
+func (r *SyncController) enqueuePhysical(ctx context.Context, obj client.Object, q workqueue.TypedRateLimitingInterface[ctrl.Request], isDelete, _ bool) {
 	if obj == nil {
 		return
 	}
@@ -609,6 +628,15 @@ func toDeleteRequest(name reconcile.Request) reconcile.Request {
 	}
 }
 
+func toPendingDeleteRequest(name reconcile.Request) reconcile.Request {
+	return reconcile.Request{
+		NamespacedName: types.NamespacedName{
+			Namespace: pendingDeleteObjectRequestPrefix + name.Namespace,
+			Name:      name.Name,
+		},
+	}
+}
+
 func toHostRequest(name reconcile.Request) reconcile.Request {
 	return reconcile.Request{
 		NamespacedName: types.NamespacedName{
@@ -623,16 +651,25 @@ func isHostRequest(name reconcile.Request) bool {
 }
 
 func fromDeleteRequest(req reconcile.Request) (reconcile.Request, synccontext.SyncEventType) {
-	if !strings.HasPrefix(req.Namespace, deleteObjectRequestPrefix) {
-		return req, synccontext.SyncEventTypeUnknown
+	if strings.HasPrefix(req.Namespace, deleteObjectRequestPrefix) {
+		return reconcile.Request{
+			NamespacedName: types.NamespacedName{
+				Namespace: strings.TrimPrefix(req.Namespace, deleteObjectRequestPrefix),
+				Name:      req.Name,
+			},
+		}, synccontext.SyncEventTypeDelete
 	}
 
-	return reconcile.Request{
-		NamespacedName: types.NamespacedName{
-			Namespace: strings.TrimPrefix(req.Namespace, deleteObjectRequestPrefix),
-			Name:      req.Name,
-		},
-	}, synccontext.SyncEventTypeDelete
+	if strings.HasPrefix(req.Namespace, pendingDeleteObjectRequestPrefix) {
+		return reconcile.Request{
+			NamespacedName: types.NamespacedName{
+				Namespace: strings.TrimPrefix(req.Namespace, pendingDeleteObjectRequestPrefix),
+				Name:      req.Name,
+			},
+		}, synccontext.SyncEventTypePendingDelete
+	}
+
+	return req, synccontext.SyncEventTypeUnknown
 }
 
 func fromHostRequest(req reconcile.Request) reconcile.Request {

test/conformanceValues.yaml

@@ -0,0 +1,59 @@
+controlPlane:
+  advanced:
+    virtualScheduler:
+      enabled: true
+  backingStore:
+    etcd:
+      deploy:
+        enabled: true
+        statefulSet:
+          image:
+            tag: 3.5.14-0
+  distro:
+    k8s:
+      apiServer:
+        extraArgs:
+          - --service-account-jwks-uri=https://kubernetes.default.svc.cluster.local/openid/v1/jwks
+        image:
+          tag: v1.31.1
+      controllerManager:
+        image:
+          tag: v1.31.1
+      enabled: true
+      scheduler:
+        image:
+          tag: v1.31.1
+  statefulSet:
+    scheduling:
+      podManagementPolicy: OrderedReady
+    image:
+      registry: ""
+      repository: ghcr.io/loft-sh/loft-enterprise/vcluster
+      tag: ""
+    env:
+      - name: DEBUG
+        value: "true"
+
+networking:
+  advanced:
+    proxyKubelets:
+      byHostname: false
+      byIP: false
+
+sync:
+  fromHost:
+    csiDrivers:
+      enabled: false
+    csiStorageCapacities:
+      enabled: false
+    nodes:
+      enabled: true
+      selector:
+        all: true
+  toHost:
+    persistentVolumes:
+      enabled: true
+    priorityClasses:
+      enabled: true
+    storageClasses:
+      enabled: true