diff --git a/examples/component-actions/zarf.yaml b/examples/component-actions/zarf.yaml index 357c37a184..717a146b45 100644 --- a/examples/component-actions/zarf.yaml +++ b/examples/component-actions/zarf.yaml @@ -37,7 +37,7 @@ components: - thing=stuff # maxRetries is the number of times to retry the action if it fails maxRetries: 0 - # maxTotalSeconds is the maximum amount of times the action can run before it is killed, including retries + # maxTotalSeconds is the maximum amount of times the action can run before it is killed, over all retries maxTotalSeconds: 30 # mute determine if actions output should be printed to the console mute: false @@ -145,12 +145,10 @@ components: actions: # runs during "zarf package deploy" onDeploy: - # defaults allow you to specify default values for the actions in that acitonSet + # defaults allow you to specify default values for the actions in that actionSet defaults: - # maxTotalSeconds is the maximum amount of time the action can run before it is killed, including retries + # maxTotalSeconds is the maximum amount of time the action can run before it is killed, over all retries maxTotalSeconds: 1 - # maxRetries is the maximum number of times the action will be retried on failure - maxRetries: 3 before: # this action will fail after 1 second - cmd: sleep 10 diff --git a/src/pkg/packager/actions.go b/src/pkg/packager/actions.go index 5f92085c4f..75d2cc1822 100644 --- a/src/pkg/packager/actions.go +++ b/src/pkg/packager/actions.go @@ -100,6 +100,7 @@ func (p *Packager) runAction(defaultCfg types.ZarfComponentActionDefaults, actio timeout := time.After(duration) // Keep trying until the max retries is reached. +retryCmd: for remaining := cfg.MaxRetries + 1; remaining > 0; remaining-- { // Perform the action run. @@ -135,7 +136,7 @@ func (p *Packager) runAction(defaultCfg types.ZarfComponentActionDefaults, actio if cfg.MaxTotalSeconds < 1 { spinner.Updatef("Waiting for \"%s\" (no timeout)", cmdEscaped) if err := tryCmd(context.TODO()); err != nil { - continue + continue retryCmd } return nil @@ -144,23 +145,31 @@ func (p *Packager) runAction(defaultCfg types.ZarfComponentActionDefaults, actio // Run the command on repeat until success or timeout. spinner.Updatef("Waiting for \"%s\" (timeout: %ds)", cmdEscaped, cfg.MaxTotalSeconds) select { - // On timeout abort. + // On timeout break the loop to abort. case <-timeout: - cancel() - return fmt.Errorf("command \"%s\" timed out", cmdEscaped) + break retryCmd // Otherwise, try running the command. default: ctx, cancel = context.WithTimeout(context.Background(), duration) defer cancel() - if err := tryCmd(ctx); err == nil { - return nil + if err := tryCmd(ctx); err != nil { + continue retryCmd } + + return nil } } - // If we've reached this point, the retry limit has been reached. - return fmt.Errorf("command \"%s\" failed after %d retries", cmdEscaped, cfg.MaxRetries) + select { + case <-timeout: + // If we reached this point, the timeout was reached. + return fmt.Errorf("command \"%s\" timed out after %d seconds", cmdEscaped, cfg.MaxTotalSeconds) + + default: + // If we reached this point, the retry limit was reached. + return fmt.Errorf("command \"%s\" failed after %d retries", cmdEscaped, cfg.MaxRetries) + } } // convertWaitToCmd will return the wait command if it exists, otherwise it will return the original command. diff --git a/src/test/e2e/02_component_actions_test.go b/src/test/e2e/02_component_actions_test.go index 5d9206cdb3..06e4892714 100644 --- a/src/test/e2e/02_component_actions_test.go +++ b/src/test/e2e/02_component_actions_test.go @@ -77,6 +77,7 @@ func TestComponentActions(t *testing.T) { // Deploy the simple action that should fail the timeout. stdOut, stdErr, err = e2e.Zarf("package", "deploy", path, "--components=on-deploy-with-timeout", "--confirm") require.Error(t, err, stdOut, stdErr) + require.Contains(t, stdErr, "after 1 second") require.Contains(t, stdErr, "😭😭😭 this action failed because it took too long to run 😭😭😭") })