Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Complete implementation of image pull failure handling #4952

Merged
merged 1 commit into from
Jun 13, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 15 additions & 7 deletions pkg/reconciler/taskrun/taskrun.go
Original file line number Diff line number Diff line change
Expand Up @@ -156,8 +156,8 @@ func (c *Reconciler) ReconcileKind(ctx context.Context, tr *v1beta1.TaskRun) pkg
}

// Check for Pod Failures
if failed, message := c.checkPodFailures(ctx, tr); failed {
err := c.failTaskRun(ctx, tr, v1beta1.TaskRunReasonImagePullFailed, message)
if failed, reason, message := c.checkPodFailed(ctx, tr); failed {
err := c.failTaskRun(ctx, tr, reason, message)
return c.finishReconcileUpdateEmitEvents(ctx, tr, before, err)
}

Expand Down Expand Up @@ -194,14 +194,22 @@ func (c *Reconciler) ReconcileKind(ctx context.Context, tr *v1beta1.TaskRun) pkg
return nil
}

func (c *Reconciler) checkPodFailures(ctx context.Context, tr *v1beta1.TaskRun) (bool, string) {
for _, step := range tr.Status.Steps {
func (c *Reconciler) checkPodFailed(ctx context.Context, tr *v1beta1.TaskRun) (bool, v1beta1.TaskRunReason, string) {
for index, step := range tr.Status.Steps {
if step.Waiting != nil && step.Waiting.Reason == "ImagePullBackOff" {
message := fmt.Sprintf(`A step in TaskRun %q failed to pull the image. The pod errored with the message: "%s."`, tr.Name, step.Waiting.Message)
return true, message
image := tr.Status.TaskSpec.Steps[index].Image
message := fmt.Sprintf(`The step %q in TaskRun %q failed to pull the image %q. The pod errored with the message: "%s."`, step.Name, tr.Name, image, step.Waiting.Message)
return true, v1beta1.TaskRunReasonImagePullFailed, message
}
}
return false, ""
for index, sidecar := range tr.Status.Sidecars {
if sidecar.Waiting != nil && sidecar.Waiting.Reason == "ImagePullBackOff" {
image := tr.Status.TaskSpec.Sidecars[index].Image
message := fmt.Sprintf(`The sidecar %q in TaskRun %q failed to pull the image %q. The pod errored with the message: "%s."`, sidecar.Name, tr.Name, image, sidecar.Waiting.Message)
return true, v1beta1.TaskRunReasonImagePullFailed, message
}
}
return false, "", ""
}

func (c *Reconciler) durationAndCountMetrics(ctx context.Context, tr *v1beta1.TaskRun) {
Expand Down
77 changes: 74 additions & 3 deletions pkg/reconciler/taskrun/taskrun_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1884,7 +1884,7 @@ status:
}
}

func TestReconcilePodFailures(t *testing.T) {
func TestReconcilePodFailuresStepImagePullFailed(t *testing.T) {
taskRun := parse.MustParseTaskRun(t, `
metadata:
name: test-imagepull-fail
Expand All @@ -1908,12 +1908,83 @@ status:
Type: apis.ConditionSucceeded,
Status: corev1.ConditionFalse,
Reason: "TaskRunImagePullFailed",
Message: `A step in TaskRun "test-imagepull-fail" failed to pull the image. The pod errored with the message: "Back-off pulling image "whatever"."`,
Message: `The step "unnamed-0" in TaskRun "test-imagepull-fail" failed to pull the image "whatever". The pod errored with the message: "Back-off pulling image "whatever"."`,
}

wantEvents := []string{
"Normal Started ",
`Warning Failed A step in TaskRun "test-imagepull-fail" failed to pull the image. The pod errored with the message: "Back-off pulling image "whatever".`,
`Warning Failed The step "unnamed-0" in TaskRun "test-imagepull-fail" failed to pull the image "whatever". The pod errored with the message: "Back-off pulling image "whatever".`,
}
d := test.Data{
TaskRuns: []*v1beta1.TaskRun{taskRun},
}
testAssets, cancel := getTaskRunController(t, d)
defer cancel()
c := testAssets.Controller
clients := testAssets.Clients

if err := c.Reconciler.Reconcile(testAssets.Ctx, getRunName(taskRun)); err != nil {
t.Fatalf("Unexpected error when reconciling completed TaskRun : %v", err)
}
newTr, err := clients.Pipeline.TektonV1beta1().TaskRuns(taskRun.Namespace).Get(testAssets.Ctx, taskRun.Name, metav1.GetOptions{})
if err != nil {
t.Fatalf("Expected completed TaskRun %s to exist but instead got error when getting it: %v", taskRun.Name, err)
}
condition := newTr.Status.GetCondition(apis.ConditionSucceeded)
if d := cmp.Diff(expectedStatus, condition, ignoreLastTransitionTime); d != "" {
t.Fatalf("Did not get expected condition %s", diff.PrintWantGot(d))
}
err = eventstest.CheckEventsOrdered(t, testAssets.Recorder.Events, taskRun.Name, wantEvents)
if err != nil {
t.Errorf(err.Error())
}
}

func TestReconcilePodFailuresSidecarImagePullFailed(t *testing.T) {
taskRun := parse.MustParseTaskRun(t, `
metadata:
name: test-imagepull-fail
namespace: foo
spec:
taskSpec:
sidecars:
- image: ubuntu
- image: whatever
steps:
- image: alpine
status:
sidecars:
- container: step-unnamed-0
name: unnamed-0
running:
startedAt: "2022-06-09T10:13:41Z"
- container: step-unnamed-1
name: unnamed-1
waiting:
message: Back-off pulling image "whatever"
reason: ImagePullBackOff
steps:
- container: step-unnamed-2
name: unnamed-2
running:
startedAt: "2022-06-09T10:13:41Z"
taskSpec:
sidecars:
- image: ubuntu
- image: whatever
steps:
- image: alpine
`)
expectedStatus := &apis.Condition{
Type: apis.ConditionSucceeded,
Status: corev1.ConditionFalse,
Reason: "TaskRunImagePullFailed",
Message: `The sidecar "unnamed-1" in TaskRun "test-imagepull-fail" failed to pull the image "whatever". The pod errored with the message: "Back-off pulling image "whatever"."`,
}

wantEvents := []string{
"Normal Started ",
`Warning Failed The sidecar "unnamed-1" in TaskRun "test-imagepull-fail" failed to pull the image "whatever". The pod errored with the message: "Back-off pulling image "whatever".`,
}
d := test.Data{
TaskRuns: []*v1beta1.TaskRun{taskRun},
Expand Down