From 8f63f724db66e3e002cadfdea457764a5a492455 Mon Sep 17 00:00:00 2001 From: Emruz Hossain Date: Fri, 8 Oct 2021 20:43:26 +0600 Subject: [PATCH] Add support for ETCD restore flow (#1392) Signed-off-by: Emruz Hossain --- go.mod | 2 +- go.sum | 4 +-- pkg/controller/restore_session.go | 34 ++++++++++++++----- pkg/rbac/restore_job.go | 22 ++++++++++-- pkg/resolve/task.go | 5 +++ vendor/modules.txt | 2 +- .../stash/v1beta1/restore_session_types.go | 3 +- .../stash.appscode.com_restorebatches.yaml | 1 + .../stash.appscode.com_restoresessions.yaml | 1 + 9 files changed, 58 insertions(+), 16 deletions(-) diff --git a/go.mod b/go.mod index c9840fe7d..550ca12af 100644 --- a/go.mod +++ b/go.mod @@ -45,7 +45,7 @@ require ( kmodules.xyz/openshift v0.0.0-20210618001443-f2507caa512f kmodules.xyz/prober v0.0.0-20210618020259-5836fb959027 kmodules.xyz/webhook-runtime v0.0.0-20210928141616-7f73c2ab318a - stash.appscode.dev/apimachinery v0.15.1-0.20211007085001-4e2a9445b1d7 + stash.appscode.dev/apimachinery v0.15.1-0.20211008114243-3ddabb572a0a ) replace bitbucket.org/ww/goautoneg => gomodules.xyz/goautoneg v0.0.0-20120707110453-a547fc61f48d diff --git a/go.sum b/go.sum index fe9e8e97a..fbf7ba4a8 100644 --- a/go.sum +++ b/go.sum @@ -1263,5 +1263,5 @@ sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= -stash.appscode.dev/apimachinery v0.15.1-0.20211007085001-4e2a9445b1d7 h1:7C+oucVxKRDgKnNkm5DYfFkFNM4eRk8r+tgCQ/bmZvc= -stash.appscode.dev/apimachinery v0.15.1-0.20211007085001-4e2a9445b1d7/go.mod h1:owQaNhtUMh6s727PX9dGIoLlqblMvMOtYLF2jy9CUek= +stash.appscode.dev/apimachinery v0.15.1-0.20211008114243-3ddabb572a0a h1:MYMUAvHKN+UFtB0HDX+oZjpfFQdetDchiEe3DsFqtXg= +stash.appscode.dev/apimachinery v0.15.1-0.20211008114243-3ddabb572a0a/go.mod h1:owQaNhtUMh6s727PX9dGIoLlqblMvMOtYLF2jy9CUek= diff --git a/pkg/controller/restore_session.go b/pkg/controller/restore_session.go index 1047e1819..e5522207e 100644 --- a/pkg/controller/restore_session.go +++ b/pkg/controller/restore_session.go @@ -1049,6 +1049,13 @@ func (c *StashController) getRestorePhase(status invoker.RestoreInvokerStatus) ( if target.Phase == api_v1beta1.TargetRestoreFailed { errList = append(errList, fmt.Errorf("restore failed for target: %s/%s", target.Ref.Kind, target.Ref.Name)) } + if target.Phase == api_v1beta1.TargetRestoreRunning { + return api_v1beta1.RestoreRunning, nil + } + } + + if errList != nil { + return api_v1beta1.RestoreFailed, errors.NewAggregate(errList) } // check if any of the target phase is "Unknown". if any of their phase is "Unknown", then consider entire restore process phase is unknown. @@ -1062,10 +1069,6 @@ func (c *StashController) getRestorePhase(status invoker.RestoreInvokerStatus) ( return api_v1beta1.RestoreRunning, nil } - if errList != nil { - return api_v1beta1.RestoreFailed, errors.NewAggregate(errList) - } - // Restore has been completed successfully for all targets. return api_v1beta1.RestoreSucceeded, nil } @@ -1149,9 +1152,10 @@ func (c *StashController) ensureRestoreTargetPhases(inv invoker.RestoreInvoker) targetStats[i].Phase = api_v1beta1.TargetRestorePending continue } - // check if any host failed to restore or it's phase 'Unknown' + // check if any host failed to restore, running, or it's phase 'Unknown' anyHostFailed := false anyHostPhaseUnknown := false + anyHostRunning := false for _, hostStats := range target.Stats { if hostStats.Phase == api_v1beta1.HostRestoreFailed { anyHostFailed = true @@ -1161,6 +1165,10 @@ func (c *StashController) ensureRestoreTargetPhases(inv invoker.RestoreInvoker) anyHostPhaseUnknown = true break } + if hostStats.Phase == api_v1beta1.HostRestoreRunning { + anyHostRunning = true + break + } } // if any host fail to restore, the overall target phase should be "Failed" if anyHostFailed { @@ -1172,13 +1180,23 @@ func (c *StashController) ensureRestoreTargetPhases(inv invoker.RestoreInvoker) targetStats[i].Phase = api_v1beta1.TargetRestorePhaseUnknown continue } + + // Restore should be succeeded only if all the conditions of this restore target are true + allConditionTrue := true + for _, c := range target.Conditions { + if c.Status == core.ConditionFalse { + allConditionTrue = false + break + } + } + // if some host hasn't completed their restore yet, phase should be "Running" - if target.TotalHosts != nil && *target.TotalHosts != int32(len(target.Stats)) { - targetStats[i].Phase = api_v1beta1.TargetRestoreRunning + if !anyHostRunning && *target.TotalHosts <= int32(len(target.Stats)) && allConditionTrue { + targetStats[i].Phase = api_v1beta1.TargetRestoreSucceeded continue } // all host completed their restore process and none of them failed. so, phase should be "Succeeded". - targetStats[i].Phase = api_v1beta1.TargetRestoreSucceeded + targetStats[i].Phase = api_v1beta1.TargetRestoreRunning } return inv.UpdateRestoreInvokerStatus(invoker.RestoreInvokerStatus{TargetStatus: targetStats}) } diff --git a/pkg/rbac/restore_job.go b/pkg/rbac/restore_job.go index 51be42df6..bcb3cf238 100644 --- a/pkg/rbac/restore_job.go +++ b/pkg/rbac/restore_job.go @@ -24,6 +24,7 @@ import ( api_v1alpha1 "stash.appscode.dev/apimachinery/apis/stash/v1alpha1" api_v1beta1 "stash.appscode.dev/apimachinery/apis/stash/v1beta1" + apps "k8s.io/api/apps/v1" core "k8s.io/api/core/v1" policy "k8s.io/api/policy/v1beta1" rbac "k8s.io/api/rbac/v1" @@ -76,13 +77,28 @@ func ensureRestoreJobClusterRole(kc kubernetes.Interface, psps []string, labels }, { APIGroups: []string{core.SchemeGroupVersion.Group}, - Resources: []string{"secrets", "endpoints", "pods"}, + Resources: []string{"secrets", "endpoints", "persistentvolumeclaims"}, Verbs: []string{"get"}, }, { APIGroups: []string{core.SchemeGroupVersion.Group}, - Resources: []string{"pods/exec"}, - Verbs: []string{"get", "create"}, + Resources: []string{"pods", "pods/exec"}, + Verbs: []string{"get", "create", "list"}, + }, + { + APIGroups: []string{core.SchemeGroupVersion.Group}, + Resources: []string{"serviceaccounts"}, + Verbs: []string{"get", "create", "patch"}, + }, + { + APIGroups: []string{apps.SchemeGroupVersion.Group}, + Resources: []string{"statefulsets"}, + Verbs: []string{"get", "patch"}, + }, + { + APIGroups: []string{rbac.SchemeGroupVersion.Group}, + Resources: []string{"roles", "rolebindings"}, + Verbs: []string{"get", "create", "patch"}, }, { APIGroups: []string{core.GroupName}, diff --git a/pkg/resolve/task.go b/pkg/resolve/task.go index 98dedc5f3..26831ef6d 100644 --- a/pkg/resolve/task.go +++ b/pkg/resolve/task.go @@ -79,6 +79,11 @@ func (o TaskResolver) GetPodSpec(invokerType, invokerName, targetKind, targetNam // merge/replace backup config inputs inputs = core_util.UpsertMap(inputs, o.Inputs) + //Add addon image as input + inputs = core_util.UpsertMap(inputs, map[string]string{ + apis.AddonImage: function.Spec.Image, + }) + // resolve Function with inputs, modify in place if err = resolveWithInputs(function, inputs); err != nil { return core.PodSpec{}, fmt.Errorf("can't resolve Function %s for Task %s, reason: %s", fn.Name, task.Name, err) diff --git a/vendor/modules.txt b/vendor/modules.txt index cc0cf062a..22ca364f7 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1482,7 +1482,7 @@ sigs.k8s.io/structured-merge-diff/v4/typed sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.2.0 sigs.k8s.io/yaml -# stash.appscode.dev/apimachinery v0.15.1-0.20211007085001-4e2a9445b1d7 +# stash.appscode.dev/apimachinery v0.15.1-0.20211008114243-3ddabb572a0a ## explicit stash.appscode.dev/apimachinery/apis stash.appscode.dev/apimachinery/apis/repositories diff --git a/vendor/stash.appscode.dev/apimachinery/apis/stash/v1beta1/restore_session_types.go b/vendor/stash.appscode.dev/apimachinery/apis/stash/v1beta1/restore_session_types.go index a00028f7f..63e518ac6 100644 --- a/vendor/stash.appscode.dev/apimachinery/apis/stash/v1beta1/restore_session_types.go +++ b/vendor/stash.appscode.dev/apimachinery/apis/stash/v1beta1/restore_session_types.go @@ -121,12 +121,13 @@ const ( RestorePhaseUnknown RestorePhase = "Unknown" ) -// +kubebuilder:validation:Enum=Succeeded;Failed;Unknown +// +kubebuilder:validation:Enum=Succeeded;Failed;Running;Unknown type HostRestorePhase string const ( HostRestoreSucceeded HostRestorePhase = "Succeeded" HostRestoreFailed HostRestorePhase = "Failed" + HostRestoreRunning HostRestorePhase = "Running" HostRestoreUnknown HostRestorePhase = "Unknown" ) diff --git a/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_restorebatches.yaml b/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_restorebatches.yaml index ac6ca992a..44ef63263 100644 --- a/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_restorebatches.yaml +++ b/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_restorebatches.yaml @@ -3833,6 +3833,7 @@ spec: enum: - Succeeded - Failed + - Running - Unknown type: string type: object diff --git a/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_restoresessions.yaml b/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_restoresessions.yaml index 4e77405ac..84074a6eb 100644 --- a/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_restoresessions.yaml +++ b/vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_restoresessions.yaml @@ -3326,6 +3326,7 @@ spec: enum: - Succeeded - Failed + - Running - Unknown type: string type: object