Adding support for optional rollback/uninstall when helm test fails

chart/helm-operator/crds/helmrelease.yaml

@@ -205,6 +205,10 @@ spec:
                 enable:
                   description: Enable will mark this Helm release for tests.
                   type: boolean
+                ignoreFailures:
+                  description: IgnoreFailures will cause a Helm release to be rolled
+                    back if it fails otherwise it will be left in a released state
+                  type: boolean
                 timeout:
                   description: Timeout is the time to wait for any individual Kubernetes
                     operation (like Jobs for hooks) during test.

deploy/crds.yaml

@@ -206,6 +206,10 @@ spec:
                 enable:
                   description: Enable will mark this Helm release for tests.
                   type: boolean
+                ignoreFailures:
+                  description: IgnoreFailures will cause a Helm release to be rolled
+                    back if it fails otherwise it will be left in a released state
+                  type: boolean
                 timeout:
                   description: Timeout is the time to wait for any individual Kubernetes
                     operation (like Jobs for hooks) during test.

docs/helmrelease-guide/tests.md

@@ -23,6 +23,19 @@ spec:
 When tests are enabled, [resource waiting](release-configuration.md#wait-for-resources-to-be-ready)
 defaults to `true` since this is likely needed for test pre-conditions to be satisfied.
 
+## Uninstall/Rollback release on test failure
+
+The `spec.test.ignoreFailures` allows the helm release to be left in a released state if the tests fail.
+Setting ignoreFailures to false will automatically uninstall/rollback the Helm Release if any of the tests fail.
+If the tests are ignored, the `Released` condition will be left as true and `Tested` will be false.
+
+```yaml
+spec:
+  test:
+    enable: true
+    ignoreFailures: false
+```
+
 ## Test timeout
 
 Test timeout can be set via the `.test.timeout` option.

docs/references/helmrelease-custom-resource.md

@@ -1722,6 +1722,19 @@ bool
 </tr>
 <tr>
 <td>
+<code>ignoreFailures</code><br>
+<em>
+bool
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>IgnoreFailures will cause a Helm release to be rolled back
+if it fails otherwise it will be left in a released state</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>timeout</code><br>
 <em>
 int64

pkg/apis/helm.fluxcd.io/v1/types_helmrelease.go

@@ -326,6 +326,10 @@ type Test struct {
 	// Enable will mark this Helm release for tests.
 	// +optional
 	Enable bool `json:"enable,omitempty"`
+	// IgnoreFailures will cause a Helm release to be rolled back
+	// if it fails otherwise it will be left in a released state
+	// +optional
+	IgnoreFailures *bool `json:"ignoreFailures,omitempty"`
 	// Timeout is the time to wait for any individual Kubernetes
 	// operation (like Jobs for hooks) during test.
 	// +optional
@@ -336,6 +340,17 @@ type Test struct {
 	Cleanup *bool `json:"cleanup,omitempty"`
 }
 
+// IgnoreFailures returns the configured ignoreFailures flag,
+// or the default of false to preserve backwards compatible
+func (t Test) GetIgnoreFailures() bool {
+	switch t.IgnoreFailures {
+	case nil:
+		return false
+	default:
+		return *t.IgnoreFailures
+	}
+}
+
 // GetTimeout returns the configured timout for the Helm release,
 // or the default of 300s.
 func (t Test) GetTimeout() time.Duration {

pkg/install/templates/crds.yaml.tmpl

@@ -206,6 +206,10 @@ spec:
                 enable:
                   description: Enable will mark this Helm release for tests.
                   type: boolean
+                ignoreFailures:
+                  description: IgnoreFailures will cause a Helm release to be rolled
+                    back if it fails otherwise it will be left in a released state
+                  type: boolean
                 timeout:
                   description: Timeout is the time to wait for any individual Kubernetes
                     operation (like Jobs for hooks) during test.

pkg/release/release.go

@@ -364,19 +364,24 @@ next:
 	case TestAction:
 		if hr.Spec.Test.Enable {
 			logger.Log("info", "running test", "action", TestAction)
+
 			if err = r.test(client, hr); err != nil {
 				logger.Log("error", err, "action", TestAction)
 				errs = append(errs, err)
 
-				if curRel == nil {
-					action = UninstallAction
+				if !hr.Spec.Test.GetIgnoreFailures() {
+					if curRel == nil {
+						action = UninstallAction
+					} else {
+						action = RollbackAction
+					}
+					goto next
 				} else {
-					action = RollbackAction
+					logger.Log("info", "test failed - ignoring failures", "revision", chart.revision)
 				}
-				goto next
+			} else {
+				logger.Log("info", "test succeeded", "revision", chart.revision, "action", action)
 			}
-
-			logger.Log("info", "test succeeded", "revision", chart.revision, "action", action)
 		}
 
 		status.SetStatusPhaseWithRevision(r.hrClient.HelmReleases(hr.Namespace), hr, apiV1.HelmReleasePhaseSucceeded, chart.revision)

pkg/status/conditions.go

@@ -133,11 +133,13 @@ func ConditionsForPhase(hr *v1.HelmRelease, phase v1.HelmReleasePhase) ([]v1.Hel
 		condition.Type = v1.HelmReleaseTested
 		condition.Status = v1.ConditionFalse
 		condition.Message = message
-		conditions = append(conditions, &v1.HelmReleaseCondition{
-			Type:    v1.HelmReleaseReleased,
-			Status:  v1.ConditionFalse,
-			Message: message,
-		})
+		if !hr.Spec.Test.GetIgnoreFailures() {
+			conditions = append(conditions, &v1.HelmReleaseCondition{
+				Type:    v1.HelmReleaseReleased,
+				Status:  v1.ConditionFalse,
+				Message: message,
+			})
+		}
 	case v1.HelmReleasePhaseRollingBack:
 		condition.Type = v1.HelmReleaseRolledBack
 		condition.Status = v1.ConditionUnknown

test/e2e/40_tests.bats

@@ -36,7 +36,7 @@ function setup() {
   poll_until_equals 'release deploy' 'True' "kubectl -n $DEMO_NAMESPACE get helmrelease/podinfo-helm-repository -o jsonpath='{.status.conditions[?(@.type==\"Tested\")].status}'"
 }
 
-@test "When test.enable is set, releases with failed tests are uninstalled" {
+@test "When test.enable is set and test.ignoreFailures is false, releases with failed tests are uninstalled" {
   # Apply the HelmRelease
   kubectl apply -f "$FIXTURES_DIR/releases/test/fail.yaml" >&3
 
@@ -49,7 +49,7 @@ function setup() {
 }
 
 # TODO: Fail tests on install instead of upgrade once install retries can be disabled.
-@test "When tests fail, Tested and Released conditions are False" {
+@test "When tests fail and test.ignoreFailures is false, Tested and Released conditions are False" {
   # Apply the HelmRelease
   kubectl apply -f "$FIXTURES_DIR/releases/test/success.yaml" >&3
 
@@ -70,7 +70,36 @@ function setup() {
   [ "$output" = 'False' ]
 }
 
-@test "When test.enable and rollback.enable are set, releases with failed tests are rolled back" {
+@test "When tests fail and test.ignoreFailures is true, release has Released condition is True & Tested condition is False" {
+  # Apply the HelmRelease that has test failure
+  kubectl apply -f "$FIXTURES_DIR/releases/test/fail-ignored.yaml" >&3
+
+  # Wait for test failure
+  poll_until_true 'test failure' "kubectl -n $E2E_NAMESPACE logs deploy/helm-operator | grep -E \"test failed\""
+
+  # Assert `Tested` condition is `False`
+  poll_until_equals 'tested condition False' 'False' "kubectl -n $DEMO_NAMESPACE get helmrelease/podinfo-helm-repository -o jsonpath='{.status.conditions[?(@.type==\"Tested\")].status}'"
+
+  # Assert test failures are ignored but logged
+  poll_until_true 'test failure - ignore failures' "kubectl -n $E2E_NAMESPACE logs deploy/helm-operator | grep -E \"test failed - ignoring failures\""
+
+  # Assert `Released` condition is `True`
+  poll_until_equals 'released condition True' 'True' "kubectl -n $DEMO_NAMESPACE get helmrelease/podinfo-helm-repository -o jsonpath='{.status.conditions[?(@.type==\"Released\")].status}'"
+
+  # Change the number of replicas to trigger upgrade
+  kubectl patch -f "$FIXTURES_DIR/releases/test/fail-ignored.yaml" --type='json' -p='[{"op": "replace", "path": "/spec/values/replicaCount", "value":"1"}]' >&3
+
+  # Wait to make sure upgrade is successful
+  poll_until_true 'upgrade complete' "kubectl -n $E2E_NAMESPACE logs deploy/helm-operator | grep -E \"upgrade succeeded\""
+
+  # Assert `Tested` condition is `False`
+  poll_until_equals 'tested condition False' 'False' "kubectl -n $DEMO_NAMESPACE get helmrelease/podinfo-helm-repository -o jsonpath='{.status.conditions[?(@.type==\"Tested\")].status}'"
+
+  # Assert `Released` condition is `True`
+  poll_until_equals 'released condition True' 'True' "kubectl -n $DEMO_NAMESPACE get helmrelease/podinfo-helm-repository -o jsonpath='{.status.conditions[?(@.type==\"Released\")].status}'"
+}
+
+@test "When test.enable and rollback.enable are set and test.ignoreFailures is false, releases with failed tests are rolled back" {
   # Apply the HelmRelease
   kubectl apply -f "$FIXTURES_DIR/releases/test/success.yaml" >&3
 

test/e2e/fixtures/releases/test/fail-no-rollback.yaml → test/e2e/fixtures/releases/test/fail-ignored.yaml

@@ -9,7 +9,11 @@ spec:
   timeout: 30
   test:
     enable: true
+    ignoreFailures: true
     timeout: 30
+  rollback:
+    enable: true
+    wait: true
   chart:
     repository: https://stefanprodan.github.io/podinfo
     name: podinfo

test/e2e/fixtures/releases/test/fail.yaml

@@ -9,6 +9,7 @@ spec:
   timeout: 30
   test:
     enable: true
+    ignoreFailures: false
     timeout: 30
   rollback:
     enable: true