feat: when a new build comes in, check the namespace for running builds to determine state

controllers/v1beta1/build_controller.go

@@ -259,9 +259,58 @@ func (r *LagoonBuildReconciler) createNamespaceBuild(ctx context.Context,
 
 	// if everything is all good controller will handle the new build resource that gets created as it will have
 	// the `lagoon.sh/buildStatus = Pending` now
-	// so end this reconcile process
-	pendingBuilds := &lagoonv1beta1.LagoonBuildList{}
-	return ctrl.Result{}, helpers.CancelExtraBuilds(ctx, r.Client, opLog, pendingBuilds, namespace.ObjectMeta.Name, lagoonv1beta1.BuildStatusPending.String())
+	err = helpers.CancelExtraBuilds(ctx, r.Client, opLog, namespace.ObjectMeta.Name, lagoonv1beta1.BuildStatusPending.String())
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	// as this is a new build coming through, check if there are any running builds in the namespace
+	// if there are, then check the status of that build. if the build pod is missing then the build running will block
+	// if the pod exists, attempt to get the status of it (only if its complete or failed) and ship the status
+	runningBuilds := &lagoonv1beta1.LagoonBuildList{}
+	listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{
+		client.InNamespace(namespace.ObjectMeta.Name),
+		client.MatchingLabels(map[string]string{"lagoon.sh/buildStatus": lagoonv1beta1.BuildStatusRunning.String()}),
+	})
+	// list all builds in the namespace that have the running buildstatus
+	if err := r.List(ctx, runningBuilds, listOption); err != nil {
+		return ctrl.Result{}, fmt.Errorf("Unable to list builds in the namespace, there may be none or something went wrong: %v", err)
+	}
+	// if there are running builds still, check if the pod exists or if the pod is complete/failed and attempt to get the status
+	for _, rBuild := range runningBuilds.Items {
+		runningBuild := rBuild.DeepCopy()
+		lagoonBuildPod := corev1.Pod{}
+		err := r.Get(ctx, types.NamespacedName{
+			Namespace: rBuild.ObjectMeta.Namespace,
+			Name:      rBuild.ObjectMeta.Name,
+		}, &lagoonBuildPod)
+		buildCondition := lagoonv1beta1.BuildStatusCancelled
+		if err != nil {
+			// cancel the build as there is no pod available
+			opLog.Info(fmt.Sprintf("Setting build %s as cancelled", runningBuild.ObjectMeta.Name))
+			runningBuild.Labels["lagoon.sh/buildStatus"] = buildCondition.String()
+		} else {
+			// get the status from the pod and update the build
+			if lagoonBuildPod.Status.Phase == corev1.PodFailed || lagoonBuildPod.Status.Phase == corev1.PodSucceeded {
+				buildCondition = helpers.GetBuildConditionFromPod(lagoonBuildPod.Status.Phase)
+				opLog.Info(fmt.Sprintf("Setting build %s as %s", runningBuild.ObjectMeta.Name, buildCondition.String()))
+				runningBuild.Labels["lagoon.sh/buildStatus"] = buildCondition.String()
+			} else {
+				// drop out, don't do anything else
+				continue
+			}
+		}
+		if err := r.Update(ctx, runningBuild); err != nil {
+			// log the error and drop out
+			opLog.Error(err, fmt.Sprintf("Error setting build %s as cancelled", runningBuild.ObjectMeta.Name))
+			continue
+		}
+		// send the status change to lagoon
+		r.updateDeploymentAndEnvironmentTask(ctx, opLog, runningBuild, nil, buildCondition, "cancelled")
+		continue
+	}
+	// handle processing running but no pod/failed pod builds
+	return ctrl.Result{}, nil
 }
 
 // getOrCreateBuildResource will deepcopy the lagoon build into a new resource and push it to the new namespace

controllers/v1beta1/build_deletionhandlers.go

@@ -214,7 +214,7 @@ Build cancelled
 		// send any messages to lagoon message queues
 		// update the deployment with the status of cancelled in lagoon
 		r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusCancelled, "cancelled")
-		r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusCancelled)
+		r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusCancelled, "cancelled")
 		r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, allContainerLogs, lagoonv1beta1.BuildStatusCancelled)
 	}
 	return nil
@@ -279,6 +279,7 @@ func (r *LagoonBuildReconciler) updateDeploymentAndEnvironmentTask(ctx context.C
 	lagoonBuild *lagoonv1beta1.LagoonBuild,
 	lagoonEnv *corev1.ConfigMap,
 	buildCondition lagoonv1beta1.BuildStatusType,
+	buildStep string,
 ) {
 	namespace := helpers.GenerateNamespaceName(
 		lagoonBuild.Spec.Project.NamespacePattern, // the namespace pattern or `openshiftProjectPattern` from Lagoon is never received by the controller
@@ -296,6 +297,7 @@ func (r *LagoonBuildReconciler) updateDeploymentAndEnvironmentTask(ctx context.C
 				Environment: lagoonBuild.Spec.Project.Environment,
 				Project:     lagoonBuild.Spec.Project.Name,
 				BuildPhase:  buildCondition.ToLower(),
+				BuildStep:   buildStep,
 				BuildName:   lagoonBuild.ObjectMeta.Name,
 				LogLink:     lagoonBuild.Spec.Project.UILink,
 				RemoteID:    string(lagoonBuild.ObjectMeta.UID),

controllers/v1beta1/build_helpers.go

@@ -901,11 +901,11 @@ func (r *LagoonBuildReconciler) updateQueuedBuild(
 	// update the deployment with the status, lagoon v2.12.0 supports queued status, otherwise use pending
 	if helpers.CheckLagoonVersion(&lagoonBuild, "2.12.0") {
 		r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusQueued, fmt.Sprintf("queued %v/%v", queuePosition, queueLength))
-		r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusQueued)
+		r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusQueued, fmt.Sprintf("queued %v/%v", queuePosition, queueLength))
 		r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, allContainerLogs, lagoonv1beta1.BuildStatusQueued)
 	} else {
 		r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusPending, fmt.Sprintf("queued %v/%v", queuePosition, queueLength))
-		r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusPending)
+		r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusPending, fmt.Sprintf("queued %v/%v", queuePosition, queueLength))
 		r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, allContainerLogs, lagoonv1beta1.BuildStatusPending)
 
 	}
@@ -961,7 +961,7 @@ Build cancelled
 	// send any messages to lagoon message queues
 	// update the deployment with the status of cancelled in lagoon
 	r.buildStatusLogsToLagoonLogs(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusCancelled, "cancelled")
-	r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusCancelled)
+	r.updateDeploymentAndEnvironmentTask(ctx, opLog, &lagoonBuild, &lagoonEnv, lagoonv1beta1.BuildStatusCancelled, "cancelled")
 	if cancelled {
 		r.buildLogsToLagoonLogs(ctx, opLog, &lagoonBuild, allContainerLogs, lagoonv1beta1.BuildStatusCancelled)
 	}

controllers/v1beta1/build_qoshandler.go

@@ -117,8 +117,7 @@ func (r *LagoonBuildReconciler) whichBuildNext(ctx context.Context, opLog logr.L
 					}
 					// if there are no running builds, check if there are any pending builds that can be started
 					if len(runningNSBuilds.Items) == 0 {
-						pendingNSBuilds := &lagoonv1beta1.LagoonBuildList{}
-						if err := helpers.CancelExtraBuilds(ctx, r.Client, opLog, pendingNSBuilds, pBuild.ObjectMeta.Namespace, "Running"); err != nil {
+						if err := helpers.CancelExtraBuilds(ctx, r.Client, opLog, pBuild.ObjectMeta.Namespace, "Running"); err != nil {
 							// only return if there is an error doing this operation
 							// continue on otherwise to allow the queued status updater to run
 							return err

controllers/v1beta1/build_standardhandler.go

@@ -52,8 +52,7 @@ func (r *LagoonBuildReconciler) standardBuildProcessor(ctx context.Context,
 
 	// if there are no running builds, check if there are any pending builds that can be started
 	if len(runningBuilds.Items) == 0 {
-		pendingBuilds := &lagoonv1beta1.LagoonBuildList{}
-		return ctrl.Result{}, helpers.CancelExtraBuilds(ctx, r.Client, opLog, pendingBuilds, req.Namespace, "Running")
+		return ctrl.Result{}, helpers.CancelExtraBuilds(ctx, r.Client, opLog, req.Namespace, "Running")
 	}
 	// The object is not being deleted, so if it does not have our finalizer,
 	// then lets add the finalizer and update the object. This is equivalent

controllers/v1beta1/podmonitor_buildhandlers.go

@@ -460,17 +460,7 @@ func (r *LagoonMonitorReconciler) updateDeploymentWithLogs(
 	cancel bool,
 ) error {
 	opLog := r.Log.WithValues("lagoonmonitor", req.NamespacedName)
-	var buildCondition lagoonv1beta1.BuildStatusType
-	switch jobPod.Status.Phase {
-	case corev1.PodFailed:
-		buildCondition = lagoonv1beta1.BuildStatusFailed
-	case corev1.PodSucceeded:
-		buildCondition = lagoonv1beta1.BuildStatusComplete
-	case corev1.PodPending:
-		buildCondition = lagoonv1beta1.BuildStatusPending
-	case corev1.PodRunning:
-		buildCondition = lagoonv1beta1.BuildStatusRunning
-	}
+	buildCondition := helpers.GetBuildConditionFromPod(jobPod.Status.Phase)
 	collectLogs := true
 	if cancel {
 		// only set the status to cancelled if the pod is running/pending/queued

controllers/v1beta1/podmonitor_controller.go

@@ -145,7 +145,6 @@ func (r *LagoonMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 		// if there are no running jobs, we check for any pending jobs
 		// sorted by their creation timestamp and set the first to running
 		opLog.Info(fmt.Sprintf("Checking for any pending builds."))
-		pendingBuilds := &lagoonv1beta1.LagoonBuildList{}
 		runningBuilds := &lagoonv1beta1.LagoonBuildList{}
 		listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{
 			client.InNamespace(req.Namespace),
@@ -157,7 +156,7 @@ func (r *LagoonMonitorReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 		}
 		// if we have no running builds, then check for any pending builds
 		if len(runningBuilds.Items) == 0 {
-			return ctrl.Result{}, helpers.CancelExtraBuilds(ctx, r.Client, opLog, pendingBuilds, req.Namespace, "Running")
+			return ctrl.Result{}, helpers.CancelExtraBuilds(ctx, r.Client, opLog, req.Namespace, "Running")
 		}
 	}
 	return ctrl.Result{}, nil

controllers/v1beta1/podmonitor_taskhandlers.go

@@ -267,17 +267,7 @@ func (r *LagoonMonitorReconciler) updateTaskWithLogs(
 	cancel bool,
 ) error {
 	opLog := r.Log.WithValues("lagoonmonitor", req.NamespacedName)
-	var taskCondition lagoonv1beta1.TaskStatusType
-	switch jobPod.Status.Phase {
-	case corev1.PodFailed:
-		taskCondition = lagoonv1beta1.TaskStatusFailed
-	case corev1.PodSucceeded:
-		taskCondition = lagoonv1beta1.TaskStatusComplete
-	case corev1.PodPending:
-		taskCondition = lagoonv1beta1.TaskStatusPending
-	case corev1.PodRunning:
-		taskCondition = lagoonv1beta1.TaskStatusRunning
-	}
+	taskCondition := helpers.GetTaskConditionFromPod(jobPod.Status.Phase)
 	collectLogs := true
 	if cancel {
 		taskCondition = lagoonv1beta1.TaskStatusCancelled

internal/helpers/helpers.go

@@ -18,6 +18,7 @@ import (
 	"github.com/go-logr/logr"
 	"github.com/hashicorp/go-version"
 	lagoonv1beta1 "github.com/uselagoon/remote-controller/apis/lagoon/v1beta1"
+	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -265,7 +266,8 @@ func CheckLagoonVersion(build *lagoonv1beta1.LagoonBuild, checkVersion string) b
 }
 
 // CancelExtraBuilds cancels extra builds.
-func CancelExtraBuilds(ctx context.Context, r client.Client, opLog logr.Logger, pendingBuilds *lagoonv1beta1.LagoonBuildList, ns string, status string) error {
+func CancelExtraBuilds(ctx context.Context, r client.Client, opLog logr.Logger, ns string, status string) error {
+	pendingBuilds := &lagoonv1beta1.LagoonBuildList{}
 	listOption := (&client.ListOptions{}).ApplyOptions([]client.ListOption{
 		client.InNamespace(ns),
 		client.MatchingLabels(map[string]string{"lagoon.sh/buildStatus": lagoonv1beta1.BuildStatusPending.String()}),
@@ -298,6 +300,36 @@ func CancelExtraBuilds(ctx context.Context, r client.Client, opLog logr.Logger,
 	return nil
 }
 
+func GetBuildConditionFromPod(phase corev1.PodPhase) lagoonv1beta1.BuildStatusType {
+	var buildCondition lagoonv1beta1.BuildStatusType
+	switch phase {
+	case corev1.PodFailed:
+		buildCondition = lagoonv1beta1.BuildStatusFailed
+	case corev1.PodSucceeded:
+		buildCondition = lagoonv1beta1.BuildStatusComplete
+	case corev1.PodPending:
+		buildCondition = lagoonv1beta1.BuildStatusPending
+	case corev1.PodRunning:
+		buildCondition = lagoonv1beta1.BuildStatusRunning
+	}
+	return buildCondition
+}
+
+func GetTaskConditionFromPod(phase corev1.PodPhase) lagoonv1beta1.TaskStatusType {
+	var taskCondition lagoonv1beta1.TaskStatusType
+	switch phase {
+	case corev1.PodFailed:
+		taskCondition = lagoonv1beta1.TaskStatusFailed
+	case corev1.PodSucceeded:
+		taskCondition = lagoonv1beta1.TaskStatusComplete
+	case corev1.PodPending:
+		taskCondition = lagoonv1beta1.TaskStatusPending
+	case corev1.PodRunning:
+		taskCondition = lagoonv1beta1.TaskStatusRunning
+	}
+	return taskCondition
+}
+
 // GenerateNamespaceName handles the generation of the namespace name from environment and project name with prefixes and patterns
 func GenerateNamespaceName(pattern, environmentName, projectname, prefix, controllerNamespace string, randomPrefix bool) string {
 	nsPattern := pattern