Add support for ETCD restore flow (#1392)

Signed-off-by: Emruz Hossain <emruz@appscode.com>

go.mod

@@ -45,7 +45,7 @@ require (
 	kmodules.xyz/openshift v0.0.0-20210618001443-f2507caa512f
 	kmodules.xyz/prober v0.0.0-20210618020259-5836fb959027
 	kmodules.xyz/webhook-runtime v0.0.0-20210928141616-7f73c2ab318a
-	stash.appscode.dev/apimachinery v0.15.1-0.20211007085001-4e2a9445b1d7
+	stash.appscode.dev/apimachinery v0.15.1-0.20211008114243-3ddabb572a0a
 )
 
 replace bitbucket.org/ww/goautoneg => gomodules.xyz/goautoneg v0.0.0-20120707110453-a547fc61f48d

go.sum

@@ -1263,5 +1263,5 @@ sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
 sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
 sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
 sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU=
-stash.appscode.dev/apimachinery v0.15.1-0.20211007085001-4e2a9445b1d7 h1:7C+oucVxKRDgKnNkm5DYfFkFNM4eRk8r+tgCQ/bmZvc=
-stash.appscode.dev/apimachinery v0.15.1-0.20211007085001-4e2a9445b1d7/go.mod h1:owQaNhtUMh6s727PX9dGIoLlqblMvMOtYLF2jy9CUek=
+stash.appscode.dev/apimachinery v0.15.1-0.20211008114243-3ddabb572a0a h1:MYMUAvHKN+UFtB0HDX+oZjpfFQdetDchiEe3DsFqtXg=
+stash.appscode.dev/apimachinery v0.15.1-0.20211008114243-3ddabb572a0a/go.mod h1:owQaNhtUMh6s727PX9dGIoLlqblMvMOtYLF2jy9CUek=

pkg/controller/restore_session.go

@@ -1049,6 +1049,13 @@ func (c *StashController) getRestorePhase(status invoker.RestoreInvokerStatus) (
 		if target.Phase == api_v1beta1.TargetRestoreFailed {
 			errList = append(errList, fmt.Errorf("restore failed for target: %s/%s", target.Ref.Kind, target.Ref.Name))
 		}
+		if target.Phase == api_v1beta1.TargetRestoreRunning {
+			return api_v1beta1.RestoreRunning, nil
+		}
+	}
+
+	if errList != nil {
+		return api_v1beta1.RestoreFailed, errors.NewAggregate(errList)
 	}
 
 	// check if any of the target phase is "Unknown". if any of their phase is "Unknown", then consider entire restore process phase is unknown.
@@ -1062,10 +1069,6 @@ func (c *StashController) getRestorePhase(status invoker.RestoreInvokerStatus) (
 		return api_v1beta1.RestoreRunning, nil
 	}
 
-	if errList != nil {
-		return api_v1beta1.RestoreFailed, errors.NewAggregate(errList)
-	}
-
 	// Restore has been completed successfully for all targets.
 	return api_v1beta1.RestoreSucceeded, nil
 }
@@ -1149,9 +1152,10 @@ func (c *StashController) ensureRestoreTargetPhases(inv invoker.RestoreInvoker)
 			targetStats[i].Phase = api_v1beta1.TargetRestorePending
 			continue
 		}
-		// check if any host failed to restore or it's phase 'Unknown'
+		// check if any host failed to restore, running, or it's phase 'Unknown'
 		anyHostFailed := false
 		anyHostPhaseUnknown := false
+		anyHostRunning := false
 		for _, hostStats := range target.Stats {
 			if hostStats.Phase == api_v1beta1.HostRestoreFailed {
 				anyHostFailed = true
@@ -1161,6 +1165,10 @@ func (c *StashController) ensureRestoreTargetPhases(inv invoker.RestoreInvoker)
 				anyHostPhaseUnknown = true
 				break
 			}
+			if hostStats.Phase == api_v1beta1.HostRestoreRunning {
+				anyHostRunning = true
+				break
+			}
 		}
 		// if any host fail to restore, the overall target phase should be "Failed"
 		if anyHostFailed {
@@ -1172,13 +1180,23 @@ func (c *StashController) ensureRestoreTargetPhases(inv invoker.RestoreInvoker)
 			targetStats[i].Phase = api_v1beta1.TargetRestorePhaseUnknown
 			continue
 		}
+
+		// Restore should be succeeded only if all the conditions of this restore target are true
+		allConditionTrue := true
+		for _, c := range target.Conditions {
+			if c.Status == core.ConditionFalse {
+				allConditionTrue = false
+				break
+			}
+		}
+
 		// if some host hasn't completed their restore yet, phase should be "Running"
-		if target.TotalHosts != nil && *target.TotalHosts != int32(len(target.Stats)) {
-			targetStats[i].Phase = api_v1beta1.TargetRestoreRunning
+		if !anyHostRunning && *target.TotalHosts <= int32(len(target.Stats)) && allConditionTrue {
+			targetStats[i].Phase = api_v1beta1.TargetRestoreSucceeded
 			continue
 		}
 		// all host completed their restore process and none of them failed. so, phase should be "Succeeded".
-		targetStats[i].Phase = api_v1beta1.TargetRestoreSucceeded
+		targetStats[i].Phase = api_v1beta1.TargetRestoreRunning
 	}
 	return inv.UpdateRestoreInvokerStatus(invoker.RestoreInvokerStatus{TargetStatus: targetStats})
 }

pkg/rbac/restore_job.go

@@ -24,6 +24,7 @@ import (
 	api_v1alpha1 "stash.appscode.dev/apimachinery/apis/stash/v1alpha1"
 	api_v1beta1 "stash.appscode.dev/apimachinery/apis/stash/v1beta1"
 
+	apps "k8s.io/api/apps/v1"
 	core "k8s.io/api/core/v1"
 	policy "k8s.io/api/policy/v1beta1"
 	rbac "k8s.io/api/rbac/v1"
@@ -76,13 +77,28 @@ func ensureRestoreJobClusterRole(kc kubernetes.Interface, psps []string, labels
 			},
 			{
 				APIGroups: []string{core.SchemeGroupVersion.Group},
-				Resources: []string{"secrets", "endpoints", "pods"},
+				Resources: []string{"secrets", "endpoints", "persistentvolumeclaims"},
 				Verbs:     []string{"get"},
 			},
 			{
 				APIGroups: []string{core.SchemeGroupVersion.Group},
-				Resources: []string{"pods/exec"},
-				Verbs:     []string{"get", "create"},
+				Resources: []string{"pods", "pods/exec"},
+				Verbs:     []string{"get", "create", "list"},
+			},
+			{
+				APIGroups: []string{core.SchemeGroupVersion.Group},
+				Resources: []string{"serviceaccounts"},
+				Verbs:     []string{"get", "create", "patch"},
+			},
+			{
+				APIGroups: []string{apps.SchemeGroupVersion.Group},
+				Resources: []string{"statefulsets"},
+				Verbs:     []string{"get", "patch"},
+			},
+			{
+				APIGroups: []string{rbac.SchemeGroupVersion.Group},
+				Resources: []string{"roles", "rolebindings"},
+				Verbs:     []string{"get", "create", "patch"},
 			},
 			{
 				APIGroups: []string{core.GroupName},

pkg/resolve/task.go

@@ -79,6 +79,11 @@ func (o TaskResolver) GetPodSpec(invokerType, invokerName, targetKind, targetNam
 		// merge/replace backup config inputs
 		inputs = core_util.UpsertMap(inputs, o.Inputs)
 
+		//Add addon image as input
+		inputs = core_util.UpsertMap(inputs, map[string]string{
+			apis.AddonImage: function.Spec.Image,
+		})
+
 		// resolve Function with inputs, modify in place
 		if err = resolveWithInputs(function, inputs); err != nil {
 			return core.PodSpec{}, fmt.Errorf("can't resolve Function %s for Task %s, reason: %s", fn.Name, task.Name, err)

vendor/modules.txt

@@ -1482,7 +1482,7 @@ sigs.k8s.io/structured-merge-diff/v4/typed
 sigs.k8s.io/structured-merge-diff/v4/value
 # sigs.k8s.io/yaml v1.2.0
 sigs.k8s.io/yaml
-# stash.appscode.dev/apimachinery v0.15.1-0.20211007085001-4e2a9445b1d7
+# stash.appscode.dev/apimachinery v0.15.1-0.20211008114243-3ddabb572a0a
 ## explicit
 stash.appscode.dev/apimachinery/apis
 stash.appscode.dev/apimachinery/apis/repositories

vendor/stash.appscode.dev/apimachinery/apis/stash/v1beta1/restore_session_types.go

@@ -121,12 +121,13 @@ const (
 	RestorePhaseUnknown RestorePhase = "Unknown"
 )
 
-// +kubebuilder:validation:Enum=Succeeded;Failed;Unknown
+// +kubebuilder:validation:Enum=Succeeded;Failed;Running;Unknown
 type HostRestorePhase string
 
 const (
 	HostRestoreSucceeded HostRestorePhase = "Succeeded"
 	HostRestoreFailed    HostRestorePhase = "Failed"
+	HostRestoreRunning   HostRestorePhase = "Running"
 	HostRestoreUnknown   HostRestorePhase = "Unknown"
 )
 

vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_restorebatches.yaml

@@ -3833,6 +3833,7 @@ spec:
                             enum:
                             - Succeeded
                             - Failed
+                            - Running
                             - Unknown
                             type: string
                         type: object

vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_restoresessions.yaml

@@ -3326,6 +3326,7 @@ spec:
                       enum:
                       - Succeeded
                       - Failed
+                      - Running
                       - Unknown
                       type: string
                   type: object