Avoid race condition during importer termination by returning 0 exitC…

…ode when scratch space is required The restart policy on failure along with manual pod deletion caused some issues after the importer exited with scratch space needed. This commit sets the exit code to 0 when exiting for scratch space required so we manually delete the pod and avoid the described race condition. Signed-off-by: Alvaro Romero <alromero@redhat.com>
kubevirt · Feb 29, 2024 · 64641fb · 64641fb
1 parent d3421bf
commit 64641fb
Show file tree

Hide file tree

Showing 4 changed files with 10 additions and 14 deletions.
diff --git a/cmd/cdi-importer/importer.go b/cmd/cdi-importer/importer.go
@@ -187,7 +187,7 @@ func handleImport(
 			if err := util.WriteTerminationMessage(common.ScratchSpaceRequired); err != nil {
 				klog.Errorf("%+v", err)
 			}
-			return common.ScratchSpaceNeededExitCode
+			return 0
 		}
 		err = util.WriteTerminationMessage(fmt.Sprintf("Unable to process data: %v", err.Error()))
 		if err != nil {

diff --git a/pkg/common/common.go b/pkg/common/common.go
@@ -205,9 +205,6 @@ const (
 	// DefaultResyncPeriod sets a 10 minute resync period, used in the controller pkg and the controller cmd executable
 	DefaultResyncPeriod = 10 * time.Minute
 
-	// ScratchSpaceNeededExitCode is the exit code that indicates the importer pod requires scratch space to function properly.
-	ScratchSpaceNeededExitCode = 42
-
 	// ScratchNameSuffix (controller pkg only)
 	ScratchNameSuffix = "scratch"
 

diff --git a/pkg/controller/import-controller.go b/pkg/controller/import-controller.go
@@ -372,15 +372,14 @@ func (r *ImportReconciler) updatePvcFromPod(pvc *corev1.PersistentVolumeClaim, p
 
 	scratchExitCode := false
 	if pod.Status.ContainerStatuses != nil &&
-		pod.Status.ContainerStatuses[0].LastTerminationState.Terminated != nil &&
-		pod.Status.ContainerStatuses[0].LastTerminationState.Terminated.ExitCode > 0 {
-		log.Info("Pod termination code", "pod.Name", pod.Name, "ExitCode", pod.Status.ContainerStatuses[0].LastTerminationState.Terminated.ExitCode)
-		if pod.Status.ContainerStatuses[0].LastTerminationState.Terminated.ExitCode == common.ScratchSpaceNeededExitCode {
+		pod.Status.ContainerStatuses[0].State.Terminated != nil {
+		if message := pod.Status.ContainerStatuses[0].State.Terminated.Message; strings.Contains(message, common.ScratchSpaceRequired) {
 			log.V(1).Info("Pod requires scratch space, terminating pod, and restarting with scratch space", "pod.Name", pod.Name)
 			scratchExitCode = true
 			anno[cc.AnnRequiresScratch] = "true"
-		} else {
-			r.recorder.Event(pvc, corev1.EventTypeWarning, ErrImportFailedPVC, pod.Status.ContainerStatuses[0].LastTerminationState.Terminated.Message)
+		} else if pod.Status.ContainerStatuses[0].State.Terminated.ExitCode > 0 {
+			log.Info("Pod termination code", "pod.Name", pod.Name, "ExitCode", pod.Status.ContainerStatuses[0].State.Terminated.ExitCode)
+			r.recorder.Event(pvc, corev1.EventTypeWarning, ErrImportFailedPVC, pod.Status.ContainerStatuses[0].State.Terminated.Message)
 		}
 	}
 

diff --git a/pkg/controller/import-controller_test.go b/pkg/controller/import-controller_test.go
@@ -542,8 +542,8 @@ var _ = Describe("Update PVC from POD", func() {
 				{
 					LastTerminationState: corev1.ContainerState{
 						Terminated: &corev1.ContainerStateTerminated{
-							ExitCode: common.ScratchSpaceNeededExitCode,
-							Message:  "scratch space needed",
+							ExitCode: 0,
+							Message:  common.ScratchSpaceRequired,
 						},
 					},
 					State: v1.ContainerState{
@@ -683,8 +683,8 @@ var _ = Describe("Update PVC from POD", func() {
 				{
 					LastTerminationState: corev1.ContainerState{
 						Terminated: &corev1.ContainerStateTerminated{
-							ExitCode: common.ScratchSpaceNeededExitCode,
-							Message:  "",
+							ExitCode: 0,
+							Message:  common.ScratchSpaceRequired,
 						},
 					},
 				},