Provide optional rollback support for HelmReleases

chart/flux/templates/helm-operator-crd.yaml

@@ -44,6 +44,21 @@ spec:
               type: boolean
             forceUpgrade:
               type: boolean
+            rollback:
+              type: object
+              properties:
+                enable:
+                  type: boolean
+                force:
+                  type: boolean
+                recreate:
+                  type: boolean
+                disableHooks:
+                  type: boolean
+                timeout:
+                  type: int64
+                wait:
+                  type: boolean
             valueFileSecrets:
               type: array
               items:

deploy-helm/flux-helm-release-crd.yaml

@@ -35,6 +35,21 @@ spec:
               type: boolean
             forceUpgrade:
               type: boolean
+            rollback:
+              type: object
+              properties:
+                enable:
+                  type: boolean
+                force:
+                  type: boolean
+                recreate:
+                  type: boolean
+                disableHooks:
+                  type: boolean
+                timeout:
+                  type: int64
+                wait:
+                  type: boolean
             valueFileSecrets:
               type: array
               items:

integrations/apis/flux.weave.works/v1beta1/types.go

@@ -103,6 +103,22 @@ func (s RepoChartSource) CleanRepoURL() string {
 	return cleanURL + "/"
 }
 
+type Rollback struct {
+	Enable 		 bool 	`json:"enable,omitempty"`
+	Force  		 bool 	`json:"force,omitempty"`
+	Recreate 	 bool 	`json:"recreate,omitempty"`
+	DisableHooks bool 	`json:"disableHooks,omitempty"`
+	Timeout 	 *int64 `json:"timeout,omitempty"`
+	Wait 		 bool   `json:"wait,omitempty"`
+}
+
+func (r Rollback) GetTimeout() int64 {
+	if r.Timeout == nil {
+		return 300
+	}
+	return *r.Timeout
+}
+
 // HelmReleaseSpec is the spec for a HelmRelease resource
 type HelmReleaseSpec struct {
 	ChartSource      `json:"chart"`
@@ -119,6 +135,9 @@ type HelmReleaseSpec struct {
 	// Force resource update through delete/recreate, allows recovery from a failed state
 	// +optional
 	ForceUpgrade bool `json:"forceUpgrade,omitempty"`
+	// Enable automatic rollbacks
+	// +optional
+	Rollback Rollback `json:"rollback,omitempty"`
 }
 
 // GetTimeout returns the install or upgrade timeout (defaults to 300s)

integrations/helm/chartsync/chartsync.go

@@ -409,6 +409,20 @@ func (chs *ChartChangeSync) reconcileReleaseDef(fhr fluxv1beta1.HelmRelease) {
 	}
 }
 
+// RollbackRelease rolls back a helm release
+func (chs *ChartChangeSync) RollbackRelease(fhr fluxv1beta1.HelmRelease) {
+	if !fhr.Spec.Rollback.Enable {
+		return
+	}
+
+	name := release.GetReleaseName(fhr)
+	err := chs.release.Rollback(name, fhr.Spec.Rollback.GetTimeout(), fhr.Spec.Rollback.Force,
+		fhr.Spec.Rollback.Recreate, fhr.Spec.Rollback.DisableHooks, fhr.Spec.Rollback.Wait)
+	if err != nil {
+		chs.logger.Log("warning", "unable to rollback chart release", "resource", fhr.ResourceID().String(), "release", name, "err", err)
+	}
+}
+
 // DeleteRelease deletes the helm release associated with a
 // HelmRelease. This exists mainly so that the operator code can
 // call it when it is handling a resource deletion.

integrations/helm/operator/operator.go

@@ -18,6 +18,7 @@ import (
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/client-go/util/workqueue"
+	"k8s.io/helm/pkg/proto/hapi/release"
 
 	flux_v1beta1 "github.com/weaveworks/flux/integrations/apis/flux.weave.works/v1beta1"
 	ifscheme "github.com/weaveworks/flux/integrations/client/clientset/versioned/scheme"
@@ -233,6 +234,12 @@ func (c *Controller) syncHandler(key string) error {
 		return err
 	}
 
+	// Attempt a rollback if the release status is FAILED.
+	if fhr.Status.ReleaseStatus == release.Status_FAILED.String() {
+		c.sync.RollbackRelease(*fhr)
+		return nil
+	}
+
 	c.sync.ReconcileReleaseDef(*fhr)
 	c.recorder.Event(fhr, corev1.EventTypeNormal, ChartSynced, MessageChartSynced)
 	return nil
@@ -282,6 +289,16 @@ func (c *Controller) enqueueUpdateJob(old, new interface{}) {
 		return
 	}
 
+	// Enqueue rollback if the status of the release has changed to
+	// FAILED, and rollbacks for this HelmRelease are enabled.
+	if oldFhr.Status.ReleaseStatus != newFhr.Status.ReleaseStatus {
+		if newFhr.Spec.Rollback.Enable && newFhr.Status.ReleaseStatus == release.Status_FAILED.String() {
+			c.logger.Log("info", "enqueing rollback", "resource", newFhr.ResourceID().String())
+			c.enqueueJob(new)
+			return
+		}
+	}
+
 	diff := cmp.Diff(oldFhr.Spec, newFhr.Spec)
 
 	// Filter out any update notifications that are due to status

integrations/helm/release/release.go

@@ -94,11 +94,29 @@ func (r *Release) GetDeployedRelease(name string) (*hapi_release.Release, error)
 	return nil, nil
 }
 
+func (r *Release) mustRollback(name string) (bool, error) {
+	rls, err := r.HelmClient.ReleaseStatus(name)
+	if err != nil {
+		return false, err
+	}
+
+	status := rls.GetInfo().GetStatus()
+	switch status.Code {
+	case hapi_release.Status_FAILED:
+		r.logger.Log("info", "rolling back release", "release", name)
+		return true, nil
+	case hapi_release.Status_PENDING_ROLLBACK:
+		r.logger.Log("info", "release already has a rollback pending", "release", name)
+		return false, nil
+	default:
+		return false, fmt.Errorf("release with status %s cannot be rolled back", status.Code.String())
+	}
+}
+
 func (r *Release) canDelete(name string) (bool, error) {
 	rls, err := r.HelmClient.ReleaseStatus(name)
 
 	if err != nil {
-		r.logger.Log("error", fmt.Sprintf("Error finding status for release (%s): %#v", name, err))
 		return false, err
 	}
 	/*
@@ -121,7 +139,6 @@ func (r *Release) canDelete(name string) (bool, error) {
 		r.logger.Log("info", fmt.Sprintf("Release %s already deleted", name))
 		return false, nil
 	default:
-		r.logger.Log("info", fmt.Sprintf("Release %s with status %s cannot be deleted", name, status.Code.String()))
 		return false, fmt.Errorf("release %s with status %s cannot be deleted", name, status.Code.String())
 	}
 }
@@ -230,6 +247,43 @@ func (r *Release) Install(chartPath, releaseName string, fhr flux_v1beta1.HelmRe
 	}
 }
 
+// Rollback rolls back a Chart release if required
+func (r *Release) Rollback(name string, timeout int64, force, recreate, disableHooks, wait bool) error {
+	ok, err := r.mustRollback(name)
+	if !ok {
+		if err != nil {
+			return err
+		}
+		return nil
+	}
+
+	history, err := r.HelmClient.ReleaseHistory(name, k8shelm.WithMaxHistory(5))
+	if err != nil {
+		return err
+	}
+
+	var version int32
+	for _, rls := range history.GetReleases() {
+		if rls.Info.Status.Code == hapi_release.Status_DEPLOYED {
+			version = rls.Version
+			break
+		}
+	}
+
+	if version == 0 {
+		return fmt.Errorf("failed to determine what version to rollback to")
+	}
+
+	_, err = r.HelmClient.RollbackRelease(name, k8shelm.RollbackVersion(version), k8shelm.RollbackTimeout(timeout),
+		k8shelm.RollbackForce(force), k8shelm.RollbackRecreate(recreate), k8shelm.RollbackDisableHooks(disableHooks),
+		k8shelm.RollbackWait(wait), k8shelm.RollbackDescription("Automated rollback by Helm operator"))
+	if err != nil {
+		return err
+	}
+	r.logger.Log("info", "rolled back release", "release", name, "version", version)
+	return err
+}
+
 // Delete purges a Chart release
 func (r *Release) Delete(name string) error {
 	ok, err := r.canDelete(name)