Adding support for optional rollback/uninstall when helm test fails

chart/helm-operator/crds/helmrelease.yaml

@@ -205,6 +205,10 @@ spec:
                 enable:
                   description: Enable will mark this Helm release for tests.
                   type: boolean
+                ignoreFailures:
+                  description: IgnoreFailures will cause a Helm release to be rolled
+                    back if it fails otherwise it will be left in a released state
+                  type: boolean
                 timeout:
                   description: Timeout is the time to wait for any individual Kubernetes
                     operation (like Jobs for hooks) during test.

deploy/crds.yaml

@@ -206,6 +206,10 @@ spec:
                 enable:
                   description: Enable will mark this Helm release for tests.
                   type: boolean
+                ignoreFailures:
+                  description: IgnoreFailures will cause a Helm release to be rolled
+                    back if it fails otherwise it will be left in a released state
+                  type: boolean
                 timeout:
                   description: Timeout is the time to wait for any individual Kubernetes
                     operation (like Jobs for hooks) during test.

docs/helmrelease-guide/tests.md

@@ -23,6 +23,19 @@ spec:
 When tests are enabled, [resource waiting](release-configuration.md#wait-for-resources-to-be-ready)
 defaults to `true` since this is likely needed for test pre-conditions to be satisfied.
 
+## Uninstall/Rollback release on test failure
+
+The `spec.test.ignoreFailures` allows the helm release to be left in a released state if the tests fail.
+Setting ignoreFailures to false will automatically uninstall/rollback the Helm Release if any of the tests fail.
+If the tests are ignored, the `Released` condition will be left as true and `Tested` will be false.
+
+```yaml
+spec:
+  test:
+    enable: true
+    ignoreFailures: false
+```
+
 ## Test timeout
 
 Test timeout can be set via the `.test.timeout` option.

docs/references/helmrelease-custom-resource.md

@@ -1722,6 +1722,19 @@ bool
 </tr>
 <tr>
 <td>
+<code>ignoreFailures</code><br>
+<em>
+bool
+</em>
+</td>
+<td>
+<em>(Optional)</em>
+<p>IgnoreFailures will cause a Helm release to be rolled back
+if it fails otherwise it will be left in a released state</p>
+</td>
+</tr>
+<tr>
+<td>
 <code>timeout</code><br>
 <em>
 int64

pkg/apis/helm.fluxcd.io/v1/types_helmrelease.go

@@ -326,6 +326,10 @@ type Test struct {
 	// Enable will mark this Helm release for tests.
 	// +optional
 	Enable bool `json:"enable,omitempty"`
+	// IgnoreFailures will cause a Helm release to be rolled back
+	// if it fails otherwise it will be left in a released state
+	// +optional
+	IgnoreFailures *bool `json:"ignoreFailures,omitempty"`
 	// Timeout is the time to wait for any individual Kubernetes
 	// operation (like Jobs for hooks) during test.
 	// +optional
@@ -336,6 +340,17 @@ type Test struct {
 	Cleanup *bool `json:"cleanup,omitempty"`
 }
 
+// IgnoreFailures returns the configured ignoreFailures flag,
+// or the default of false to preserve backwards compatible
+func (t Test) GetIgnoreFailures() bool {
+	switch t.IgnoreFailures {
+	case nil:
+		return false
+	default:
+		return *t.IgnoreFailures
+	}
+}
+
 // GetTimeout returns the configured timout for the Helm release,
 // or the default of 300s.
 func (t Test) GetTimeout() time.Duration {

pkg/helm/v3/converter.go

@@ -1,6 +1,8 @@
 package v3
 
 import (
+	"strings"
+
 	"github.com/helm/helm-2to3/pkg/common"
 	helm2 "github.com/helm/helm-2to3/pkg/v2"
 	helm3 "github.com/helm/helm-2to3/pkg/v3"
@@ -22,7 +24,10 @@ func (c Converter) V2ReleaseExists(releaseName string) (bool, error) {
 		File: c.KubeConfig,
 	}
 	v2Releases, err := helm2.GetReleaseVersions(retrieveOpts, kubeConfig)
-	if err != nil {
+
+	// We check for the error message content because
+	// Helm 2to3 returns an error if it doesn't find release versions
+	if err != nil && !strings.Contains(err.Error(), "has no deployed releases") {
 		return false, err
 	}
 	return len(v2Releases) > 0, nil

pkg/install/templates/crds.yaml.tmpl

@@ -206,6 +206,10 @@ spec:
                 enable:
                   description: Enable will mark this Helm release for tests.
                   type: boolean
+                ignoreFailures:
+                  description: IgnoreFailures will cause a Helm release to be rolled
+                    back if it fails otherwise it will be left in a released state
+                  type: boolean
                 timeout:
                   description: Timeout is the time to wait for any individual Kubernetes
                     operation (like Jobs for hooks) during test.

pkg/release/release.go

@@ -218,7 +218,7 @@ func (r *Release) determineSyncAction(client helm.Client, hr *apiV1.HelmRelease,
 			case string(apiV1.HelmV3):
 				v2ReleaseExists, err := r.converter.V2ReleaseExists(hr.GetReleaseName())
 				if err != nil {
-					return SkipAction, nil, fmt.Errorf("failed to retrieve Helm v2 release whil attempting migration: %w", err)
+					return SkipAction, nil, fmt.Errorf("failed to retrieve Helm v2 release while attempting migration: %w", err)
 				}
 				if v2ReleaseExists {
 					return MigrateAction, nil, nil
@@ -364,19 +364,24 @@ next:
 	case TestAction:
 		if hr.Spec.Test.Enable {
 			logger.Log("info", "running test", "action", TestAction)
+
 			if err = r.test(client, hr); err != nil {
 				logger.Log("error", err, "action", TestAction)
 				errs = append(errs, err)
 
-				if curRel == nil {
-					action = UninstallAction
+				if !hr.Spec.Test.GetIgnoreFailures() {
+					if curRel == nil {
+						action = UninstallAction
+					} else {
+						action = RollbackAction
+					}
+					goto next
 				} else {
-					action = RollbackAction
+					logger.Log("info", "test failed - ignoring failures", "revision", chart.revision)
 				}
-				goto next
+			} else {
+				logger.Log("info", "test succeeded", "revision", chart.revision, "action", action)
 			}
-
-			logger.Log("info", "test succeeded", "revision", chart.revision, "action", action)
 		}
 
 		status.SetStatusPhaseWithRevision(r.hrClient.HelmReleases(hr.Namespace), hr, apiV1.HelmReleasePhaseSucceeded, chart.revision)

pkg/status/conditions.go

@@ -133,11 +133,13 @@ func ConditionsForPhase(hr *v1.HelmRelease, phase v1.HelmReleasePhase) ([]v1.Hel
 		condition.Type = v1.HelmReleaseTested
 		condition.Status = v1.ConditionFalse
 		condition.Message = message
-		conditions = append(conditions, &v1.HelmReleaseCondition{
-			Type:    v1.HelmReleaseReleased,
-			Status:  v1.ConditionFalse,
-			Message: message,
-		})
+		if !hr.Spec.Test.GetIgnoreFailures() {
+			conditions = append(conditions, &v1.HelmReleaseCondition{
+				Type:    v1.HelmReleaseReleased,
+				Status:  v1.ConditionFalse,
+				Message: message,
+			})
+		}
 	case v1.HelmReleasePhaseRollingBack:
 		condition.Type = v1.HelmReleaseRolledBack
 		condition.Status = v1.ConditionUnknown

test/e2e/40_tests.bats

@@ -36,7 +36,7 @@ function setup() {
   poll_until_equals 'release deploy' 'True' "kubectl -n $DEMO_NAMESPACE get helmrelease/podinfo-helm-repository -o jsonpath='{.status.conditions[?(@.type==\"Tested\")].status}'"
 }
 
-@test "When test.enable is set, releases with failed tests are uninstalled" {
+@test "When test.enable is set and test.ignoreFailures is false, releases with failed tests are uninstalled" {
   # Apply the HelmRelease
   kubectl apply -f "$FIXTURES_DIR/releases/test/fail.yaml" >&3
 
@@ -49,7 +49,7 @@ function setup() {
 }
 
 # TODO: Fail tests on install instead of upgrade once install retries can be disabled.
-@test "When tests fail, Tested and Released conditions are False" {
+@test "When tests fail and test.ignoreFailures is false, Tested and Released conditions are False" {
   # Apply the HelmRelease
   kubectl apply -f "$FIXTURES_DIR/releases/test/success.yaml" >&3
 
@@ -70,7 +70,24 @@ function setup() {
   [ "$output" = 'False' ]
 }
 
-@test "When test.enable and rollback.enable are set, releases with failed tests are rolled back" {
+@test "When tests fail and test.ignoreFailures is true, release has phase 'TestFailed' and Released condition is True & Tested condition is False" {
+  # Apply the HelmRelease that has test failure
+  kubectl apply -f "$FIXTURES_DIR/releases/test/fail-ignored.yaml" >&3
+
+  # Wait for test failure
+  poll_until_true 'test failure' "kubectl -n $E2E_NAMESPACE logs deploy/helm-operator | grep -E \"test failed\""
+
+  # Assert `Tested` condition is `False`
+  poll_until_equals 'tested condition False' 'False' "kubectl -n $DEMO_NAMESPACE get helmrelease/podinfo-helm-repository -o jsonpath='{.status.conditions[?(@.type==\"Tested\")].status}'"
+
+  # Assert test failures are ignored but logged
+  poll_until_true 'test failure - ignore failures' "kubectl -n $E2E_NAMESPACE logs deploy/helm-operator | grep -E \"test failed - ignoring failures\""
+
+  # Assert `Released` condition is `True`
+  poll_until_equals 'released condition True' 'True' "kubectl -n $DEMO_NAMESPACE get helmrelease/podinfo-helm-repository -o jsonpath='{.status.conditions[?(@.type==\"Released\")].status}'"
+}
+
+@test "When test.enable and rollback.enable are set and test.ignoreFailures is false, releases with failed tests are rolled back" {
   # Apply the HelmRelease
   kubectl apply -f "$FIXTURES_DIR/releases/test/success.yaml" >&3
 

test/e2e/45_convert_2to3.bats

@@ -15,7 +15,7 @@ function setup() {
   kubectl create namespace "$DEMO_NAMESPACE"
 }
 
-@test "When migrate annotations exist, migration succeeds" {
+@test "Migration succeeds from v2 to v3" {
   # Apply the HelmRelease
   kubectl apply -f "$FIXTURES_DIR/releases/convert-2to3-v2.yaml" >&3
 
@@ -28,6 +28,15 @@ function setup() {
   poll_until_equals 'helm2 no longer shows helm release' '0' "HELM_VERSION=v2 helm ls | grep podinfo-helm-repository | wc -l | awk '{\$1=\$1};1'"
   poll_until_equals 'helm3 shows helm release' 'podinfo-helm-repository' "HELM_VERSION=v3 helm ls -n $DEMO_NAMESPACE | grep podinfo-helm-repository | awk '{print \$1}'"
   poll_until_equals 'release migrated' 'True' "kubectl -n $DEMO_NAMESPACE get helmrelease/podinfo-helm-repository -o jsonpath='{.status.conditions[?(@.type==\"Released\")].status}'"
+
+  kubectl apply -f "$FIXTURES_DIR/releases/convert-2to3-v3-upgrade.yaml" >&3
+  poll_until_equals 'upgrades work after migration' '1' "kubectl get deploy/podinfo-helm-repository -n "$DEMO_NAMESPACE" -o jsonpath='{.spec.replicas}'"
+}
+
+@test "Migration is skipped and install works when no v2 release exists" {
+  kubectl apply -f "$FIXTURES_DIR/releases/convert-2to3-v3.yaml" >&3
+  poll_until_equals 'helm3 shows helm release' 'podinfo-helm-repository' "HELM_VERSION=v3 helm ls -n $DEMO_NAMESPACE | grep podinfo-helm-repository | awk '{print \$1}'"
+  poll_until_equals 'install successful' 'True' "kubectl -n $DEMO_NAMESPACE get helmrelease/podinfo-helm-repository -o jsonpath='{.status.conditions[?(@.type==\"Released\")].status}'"
 }
 
 function teardown() {
@@ -42,4 +51,4 @@ function teardown() {
   kubectl delete namespace "$E2E_NAMESPACE"
   # Only remove the demo workloads after the operator, so that they cannot be recreated.
   kubectl delete namespace "$DEMO_NAMESPACE"
-}
+}

test/e2e/fixtures/releases/convert-2to3-v3-upgrade.yaml

@@ -0,0 +1,24 @@
+---
+apiVersion: helm.fluxcd.io/v1
+kind: HelmRelease
+metadata:
+  name: podinfo-helm-repository
+  namespace: demo
+  annotations:
+    helm.fluxcd.io/migrate: "true"
+spec:
+  helmVersion: v3
+  releaseName: podinfo-helm-repository
+  timeout: 30
+  test:
+    enable: true
+    timeout: 30
+  rollback:
+    enable: true
+    wait: true
+  chart:
+    repository: https://stefanprodan.github.io/podinfo
+    name: podinfo
+    version: 4.0.1
+  values:
+    replicaCount: 1

test/e2e/fixtures/releases/test/fail-no-rollback.yaml → test/e2e/fixtures/releases/test/fail-ignored.yaml

@@ -9,7 +9,11 @@ spec:
   timeout: 30
   test:
     enable: true
+    ignoreFailures: true
     timeout: 30
+  rollback:
+    enable: true
+    wait: true
   chart:
     repository: https://stefanprodan.github.io/podinfo
     name: podinfo

test/e2e/fixtures/releases/test/fail.yaml

@@ -9,6 +9,7 @@ spec:
   timeout: 30
   test:
     enable: true
+    ignoreFailures: false
     timeout: 30
   rollback:
     enable: true