feat: report error when common PVC cleanup job hangs

Fix devfile#551

Signed-off-by: Andrew Obuchowicz <aobuchow@redhat.com>

pkg/provision/storage/cleanup.go

@@ -28,7 +28,9 @@ import (
 	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/types"
+	k8sclient "sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	"github.com/devfile/devworkspace-operator/internal/images"
@@ -52,6 +54,10 @@ var (
 	pvcCleanupPodCPURequest    = resource.MustParse(constants.PVCCleanupPodCPURequest)
 )
 
+var unrecoverablePodEventReasons = map[string]bool{
+	"FailedScheduling": true,
+}
+
 func runCommonPVCCleanupJob(workspace *dw.DevWorkspace, clusterAPI sync.ClusterAPI) error {
 	PVCexists, err := commonPVCExists(workspace, clusterAPI)
 	if err != nil {
@@ -78,10 +84,12 @@ func runCommonPVCCleanupJob(workspace *dw.DevWorkspace, clusterAPI sync.ClusterA
 	}
 
 	clusterJob := clusterObj.(*batchv1.Job)
+
 	for _, condition := range clusterJob.Status.Conditions {
 		if condition.Status != corev1.ConditionTrue {
 			continue
 		}
+
 		switch condition.Type {
 		case batchv1.JobComplete:
 			return nil
@@ -91,6 +99,14 @@ func runCommonPVCCleanupJob(workspace *dw.DevWorkspace, clusterAPI sync.ClusterA
 			}
 		}
 	}
+
+	if msg, err := checkCommonPVCCleanupJobEvents(clusterJob, workspace.Status.DevWorkspaceId, clusterAPI); err != nil || msg != "" {
+		return &ProvisioningError{
+			Err:     err,
+			Message: fmt.Sprintf("DevWorkspace PVC cleanup job failed: see logs for job %q for details. Additional information: %s", clusterJob.Name, msg),
+		}
+	}
+
 	// Requeue at least each 10 seconds to check if PVC is not removed by someone else
 	return &NotReadyError{
 		Message:      "Cleanup job is not in completed state",
@@ -203,3 +219,25 @@ func commonPVCExists(workspace *dw.DevWorkspace, clusterAPI sync.ClusterAPI) (bo
 	}
 	return true, nil
 }
+
+func checkCommonPVCCleanupJobEvents(job *batchv1.Job, workspaceID string, clusterAPI sync.ClusterAPI) (msg string, err error) {
+	evs := &corev1.EventList{}
+	selector, err := fields.ParseSelector(fmt.Sprintf("involvedObject.name=%s", job.Name))
+	if err != nil {
+		return "", fmt.Errorf("failed to parse field selector: %s", err)
+	}
+	if err := clusterAPI.Client.List(clusterAPI.Ctx, evs, k8sclient.InNamespace(job.Namespace), k8sclient.MatchingFieldsSelector{Selector: selector}); err != nil {
+		return "", fmt.Errorf("failed to list events in namespace %s: %w", job.Namespace, err)
+	}
+	for _, ev := range evs.Items {
+		if ev.InvolvedObject.Kind != "Job" {
+			continue
+		}
+
+		if _, ok := unrecoverablePodEventReasons[ev.Reason]; ok {
+			msg = fmt.Sprintf("Detected unrecoverable event %s: %s", ev.Reason, ev.Message)
+			return msg, nil
+		}
+	}
+	return "", nil
+}