Implement Pause Restic (#315)

apis/stash/types.go

@@ -37,6 +37,9 @@ type ResticSpec struct {
 	RetentionPolicies []RetentionPolicy         `json:"retentionPolicies,omitempty"`
 	// https://github.com/appscode/stash/issues/225
 	Type BackupType `json:"type,omitempty"`
+	//Indicates that the Restic is paused from taking backup. Default value is 'false'
+	// +optional
+	Paused bool `json:"paused,omitempty"`
 	// ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec.
 	// If specified, these secrets will be passed to individual puller implementations for them to use. For example,
 	// in the case of docker, only DockerConfig type secrets are honored.

apis/stash/v1alpha1/types.go

@@ -37,6 +37,9 @@ type ResticSpec struct {
 	RetentionPolicies []RetentionPolicy         `json:"retentionPolicies,omitempty"`
 	// https://github.com/appscode/stash/issues/225
 	Type BackupType `json:"type,omitempty"`
+	//Indicates that the Restic is paused from taking backup. Default value is 'false'
+	// +optional
+	Paused bool `json:"paused,omitempty"`
 	// ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling any of the images used by this PodSpec.
 	// If specified, these secrets will be passed to individual puller implementations for them to use. For example,
 	// in the case of docker, only DockerConfig type secrets are honored.

apis/stash/v1alpha1/zz_generated.conversion.go

@@ -452,6 +452,7 @@ func autoConvert_v1alpha1_ResticSpec_To_stash_ResticSpec(in *ResticSpec, out *st
 	out.Resources = in.Resources
 	out.RetentionPolicies = *(*[]stash.RetentionPolicy)(unsafe.Pointer(&in.RetentionPolicies))
 	out.Type = stash.BackupType(in.Type)
+	out.Paused = in.Paused
 	out.ImagePullSecrets = *(*[]v1.LocalObjectReference)(unsafe.Pointer(&in.ImagePullSecrets))
 	return nil
 }
@@ -472,6 +473,7 @@ func autoConvert_stash_ResticSpec_To_v1alpha1_ResticSpec(in *stash.ResticSpec, o
 	out.Resources = in.Resources
 	out.RetentionPolicies = *(*[]RetentionPolicy)(unsafe.Pointer(&in.RetentionPolicies))
 	out.Type = BackupType(in.Type)
+	out.Paused = in.Paused
 	out.ImagePullSecrets = *(*[]v1.LocalObjectReference)(unsafe.Pointer(&in.ImagePullSecrets))
 	return nil
 }

docs/concepts/crds/restic.md

@@ -92,6 +92,9 @@ To learn how to configure various backends for Restic, please visit [here](/docs
 `spec.schedule` is a [cron expression](https://github.com/robfig/cron/blob/v2/doc.go#L26) that indicates how often `restic` commands are invoked for file groups.
 At each tick, `restic backup` and `restic forget` commands are run for each of the configured file groups.
 
+### spec.paused
+`spec.paused` can be used as `enable/disable` switch for Restic. The default value is `false`. To stop restic from taking backup set `spec.paused: true`. For more details see [here](/docs/guides/backup.md#disable-backup).
+
 ### spec.resources
 `spec.resources` refers to compute resources required by the `stash` sidecar container. To learn more, visit [here](http://kubernetes.io/docs/user-guide/compute-resources/).
 
@@ -105,7 +108,7 @@ At each tick, `restic backup` and `restic forget` commands are run for each of t
  - For workload kind `Daemonset` restic repository is created in the sub-directory `<WORKLOAD_KIND>/<WORKLOAD_NAME>/<NODE_NAME>`. For multiple replicas, multiple repositories are created and sidecar is added to all pods.
 
 ## Restic Status
-Stash operator updates `.status` of a Restic CRD every time a backup operation is completed. 
+Stash operator updates `.status` of a Restic CRD every time a backup operation is completed.
 
  - `status.backupCount` indicated the total number of backup operation completed for this Restic CRD.
  - `status.firstBackupTime` indicates the timestamp of first backup operation.
@@ -122,11 +125,19 @@ For each workload where a sidecar container is added by Stash operator, the foll
 ## Updating Restic
 The sidecar container watches for changes in the Restic fileGroups, backend and schedule. These changes are automatically applied on the next run of `restic` commands. If the selector of a Restic CRD is changed, Stash operator will update workload accordingly by adding/removing sidecars as required.
 
-## Disable Backup
-To stop taking backup, you can do 2 things:
+## Disable Restic
+To stop Restic from taking backup, you can do following things:
+
+* Set `spec.paused: true` in Restic `yaml` and then update the Restic object. This means:
+
+  - Paused Restic CRDs will not applied to newly created wrokloads.
+  - Stash sidecar containers will not be removed from existing workloads but the sidecar will stop taking backup.
+
+* Delete the Restic CRD. Stash operator will remove the sidecar container from all matching workloads.
+
+* Change the labels of a workload. Stash operator will remove sidecar container from that workload. This way you can selectively stop backup of a Deployment, ReplicaSet etc.
 
-- Delete the Restic CRD. Stash operator will remove the sidecar container from all matching workloads.
-- Change the labels of a workload. Stash operator will remove sidecar container from that workload. This way you can selectively stop backup of a Deployment, ReplicaSet, etc.
+For more details about how to disable and resume Restic see [here](/docs/guides/backup.md#disable-backup). 
 
 ## Next Steps
 

docs/guides/backup.md

@@ -345,17 +345,63 @@ ID        Date                 Host        Tags        Directory
 ```
 
 ## Disable Backup
-To stop taking backup of `/source/data` folder, delete the `stash-demo` Restic CRD. As a result, Stash operator will remove the sidecar container from `busybox` Deployment.
-```console
+To stop Restic from taking backup, you can do following things:
+
+* Set `spec.paused: true` in Restic `yaml` and then apply the update. This means:
+
+  - Paused Restic CRDs will not applied to newly created wrokloads.
+  - Stash sidecar containers will not be removed from existing workloads but the sidecar will stop taking backup.
+
+```command
+$ kubectl patch restic stash-demo --type="merge" --patch='{"spec": {"paused": true}}'
+restic "stash-demo" patched
+```
+
+```yaml
+apiVersion: stash.appscode.com/v1alpha1
+kind: Restic
+metadata:
+  name: stash-demo
+  namespace: default
+spec:
+  selector:
+    matchLabels:
+      app: stash-demo
+  fileGroups:
+  - path: /source/data
+    retentionPolicyName: 'keep-last-5'
+  backend:
+    local:
+      mountPath: /safe/data
+      hostPath:
+        path: /data/stash-test/restic-repo
+    storageSecretName: stash-demo
+  schedule: '@every 1m'
+  paused: true
+  volumeMounts:
+  - mountPath: /source/data
+    name: source-data
+  retentionPolicies:
+  - name: 'keep-last-5'
+    keepLast: 5
+    prune: true
+```
+
+* Delete the Restic CRD. Stash operator will remove the sidecar container from all matching workloads.
+
+```commands
 $ kubectl delete restic stash-demo
 restic "stash-demo" deleted
+```
+* Change the labels of a workload. Stash operator will remove sidecar container from that workload. This way you can selectively stop backup of a Deployment, ReplicaSet etc.
 
-$ kubectl get pods -l app=stash-demo
-NAME                          READY     STATUS        RESTARTS   AGE
-stash-demo-79554ff97b-wsdx2   2/2       Terminating   0          3m
-stash-demo-788ffcf9c6-p47p7   1/1       Running       0          5s
+### Resume Backup
+You can resume Restic to backup by setting `spec.paused: false` in Restic `yaml` and applying the update or you can patch Restic using,
+```command
+$ kubectl patch restic stash-demo --type="merge" --patch='{"spec": {"paused": false}}'
 ```
 
+
 ## Cleaning up
 
 To cleanup the Kubernetes resources created by this tutorial, run:
@@ -378,4 +424,4 @@ If you would like to uninstall Stash operator, please follow the steps [here](/d
 - Thinking about monitoring your backup operations? Stash works [out-of-the-box with Prometheus](/docs/guides/monitoring.md).
 - Learn about how to configure [RBAC roles](/docs/guides/rbac.md).
 - Learn about how to configure Stash operator as workload initializer [here](/docs/guides/initializer.md).
-- Want to hack on Stash? Check our [contribution guidelines](/docs/CONTRIBUTING.md).
+- Want to hack on Stash? Check our [contribution guidelines](/docs/CONTRIBUTING.md).

pkg/backup/backup.go

@@ -189,6 +189,10 @@ func (c *Controller) setup() (*api.Restic, error) {
 }
 
 func (c *Controller) runResticBackup(resource *api.Restic) (err error) {
+	if resource.Spec.Paused == true {
+		log.Infoln("skipped logging since restic is paused.")
+		return nil
+	}
 	startTime := metav1.Now()
 	var (
 		restic_session_success = prometheus.NewGauge(prometheus.GaugeOpts{

pkg/controller/daemonsets.go

@@ -59,7 +59,9 @@ func (c *StashController) runDaemonSetInjector(key string) error {
 		}
 
 		if newRestic != nil && !util.ResticEqual(oldRestic, newRestic) {
-			return c.EnsureDaemonSetSidecar(ds, oldRestic, newRestic)
+			if !newRestic.Spec.Paused {
+				return c.EnsureDaemonSetSidecar(ds, oldRestic, newRestic)
+			}
 		} else if oldRestic != nil && newRestic == nil {
 			return c.EnsureDaemonSetSidecarDeleted(ds, oldRestic)
 		}

pkg/controller/deployment.go

@@ -65,10 +65,12 @@ func (c *StashController) runDeploymentInjector(key string) error {
 			return err
 		}
 		if newRestic != nil && !util.ResticEqual(oldRestic, newRestic) {
-			if newRestic.Spec.Type == api.BackupOffline && *dp.Spec.Replicas > 1 {
-				return fmt.Errorf("cannot perform offline backup for deployment with replicas > 1")
+			if !newRestic.Spec.Paused {
+				if newRestic.Spec.Type == api.BackupOffline && *dp.Spec.Replicas > 1 {
+					return fmt.Errorf("cannot perform offline backup for deployment with replicas > 1")
+				}
+				return c.EnsureDeploymentSidecar(dp, oldRestic, newRestic)
 			}
-			return c.EnsureDeploymentSidecar(dp, oldRestic, newRestic)
 		} else if oldRestic != nil && newRestic == nil {
 			return c.EnsureDeploymentSidecarDeleted(dp, oldRestic)
 		}

pkg/controller/rcs.go

@@ -65,10 +65,12 @@ func (c *StashController) runRCInjector(key string) error {
 		}
 
 		if newRestic != nil && !util.ResticEqual(oldRestic, newRestic) {
-			if newRestic.Spec.Type == api.BackupOffline && *rc.Spec.Replicas > 1 {
-				return fmt.Errorf("cannot perform offline backup for rc with replicas > 1")
+			if !newRestic.Spec.Paused {
+				if newRestic.Spec.Type == api.BackupOffline && *rc.Spec.Replicas > 1 {
+					return fmt.Errorf("cannot perform offline backup for rc with replicas > 1")
+				}
+				return c.EnsureReplicationControllerSidecar(rc, oldRestic, newRestic)
 			}
-			return c.EnsureReplicationControllerSidecar(rc, oldRestic, newRestic)
 		} else if oldRestic != nil && newRestic == nil {
 			return c.EnsureReplicationControllerSidecarDeleted(rc, oldRestic)
 		}

pkg/controller/replicasets.go

@@ -66,10 +66,12 @@ func (c *StashController) runReplicaSetInjector(key string) error {
 				return err
 			}
 			if newRestic != nil && !util.ResticEqual(oldRestic, newRestic) {
-				if newRestic.Spec.Type == api.BackupOffline && *rs.Spec.Replicas > 1 {
-					return fmt.Errorf("cannot perform offline backup for rs with replicas > 1")
+				if !newRestic.Spec.Paused {
+					if newRestic.Spec.Type == api.BackupOffline && *rs.Spec.Replicas > 1 {
+						return fmt.Errorf("cannot perform offline backup for rs with replicas > 1")
+					}
+					return c.EnsureReplicaSetSidecar(rs, oldRestic, newRestic)
 				}
-				return c.EnsureReplicaSetSidecar(rs, oldRestic, newRestic)
 			} else if oldRestic != nil && newRestic == nil {
 				return c.EnsureReplicaSetSidecarDeleted(rs, oldRestic)
 			}

pkg/controller/statefulsets.go

@@ -61,7 +61,9 @@ func (c *StashController) runStatefulSetInjector(key string) error {
 			}
 
 			if newRestic != nil && !util.ResticEqual(oldRestic, newRestic) {
-				return c.EnsureStatefulSetSidecar(ss, oldRestic, newRestic)
+				if !newRestic.Spec.Paused {
+					return c.EnsureStatefulSetSidecar(ss, oldRestic, newRestic)
+				}
 			} else if oldRestic != nil && newRestic == nil {
 				return c.EnsureStatefulSetSidecarDeleted(ss, oldRestic)
 			}

test/e2e/daemonset_test.go

@@ -485,4 +485,79 @@ var _ = Describe("DaemonSet", func() {
 			})
 		})
 	})
+
+	Describe("Pause Restic to stop backup", func() {
+		Context(`"Local" backend`, func() {
+			AfterEach(func() {
+				f.DeleteDaemonSet(daemon.ObjectMeta)
+				f.DeleteRestic(restic.ObjectMeta)
+				f.DeleteSecret(cred.ObjectMeta)
+			})
+			BeforeEach(func() {
+				cred = f.SecretForLocalBackend()
+				restic = f.ResticForLocalBackend()
+			})
+			It(`should be able to Pause and Resume backup`, func() {
+				By("Creating repository Secret " + cred.Name)
+				err = f.CreateSecret(cred)
+				Expect(err).NotTo(HaveOccurred())
+
+				By("Creating restic")
+				err = f.CreateRestic(restic)
+				Expect(err).NotTo(HaveOccurred())
+
+				By("Creating Daemonset " + daemon.Name)
+				_, err = f.CreateDaemonSet(daemon)
+				Expect(err).NotTo(HaveOccurred())
+
+				By("Waiting for sidecar")
+				f.EventuallyDaemonSet(daemon.ObjectMeta).Should(HaveSidecar(util.StashContainer))
+
+				By("Waiting for backup to complete")
+				f.EventuallyRestic(restic.ObjectMeta).Should(WithTransform(func(r *api.Restic) int64 {
+					return r.Status.BackupCount
+				}, BeNumerically(">=", 1)))
+
+				By("Waiting for backup event")
+				f.EventualEvent(restic.ObjectMeta).Should(WithTransform(f.CountSuccessfulBackups, BeNumerically(">=", 1)))
+
+				By(`Patching Restic with "paused: true"`)
+				err = f.CreateOrPatchRestic(restic.ObjectMeta, func(in *api.Restic) *api.Restic {
+					in.Spec.Paused = true
+					return in
+				})
+				Expect(err).NotTo(HaveOccurred())
+
+				resticObj, err := f.StashClient.StashV1alpha1().Restics(restic.Namespace).Get(restic.Name, metav1.GetOptions{})
+				Expect(err).NotTo(HaveOccurred())
+
+				previousBackupCount := resticObj.Status.BackupCount
+
+				By("Wating 2 minutes")
+				time.Sleep(2 * time.Minute)
+
+				By("Checking that Backup count has not changed")
+				resticObj, err = f.StashClient.StashV1alpha1().Restics(restic.Namespace).Get(restic.Name, metav1.GetOptions{})
+				Expect(err).NotTo(HaveOccurred())
+				Expect(resticObj.Status.BackupCount).Should(BeNumerically("==", previousBackupCount))
+
+				By(`Patching Restic with "paused: false"`)
+				err = f.CreateOrPatchRestic(restic.ObjectMeta, func(in *api.Restic) *api.Restic {
+					in.Spec.Paused = false
+					return in
+				})
+				Expect(err).NotTo(HaveOccurred())
+
+				By("Waiting for backup to complete")
+				f.EventuallyRestic(restic.ObjectMeta).Should(WithTransform(func(r *api.Restic) int64 {
+					return r.Status.BackupCount
+				}, BeNumerically(">", previousBackupCount)))
+
+				By("Waiting for backup event")
+				f.EventualEvent(restic.ObjectMeta).Should(WithTransform(f.CountSuccessfulBackups, BeNumerically(">", previousBackupCount)))
+
+			})
+
+		})
+	})
 })

test/e2e/deployment_test.go

@@ -741,4 +741,79 @@ var _ = Describe("Deployment", func() {
 		})
 		It(`should backup new Deployment`, shouldBackupNewDeployment)
 	})
+
+	Describe("Pause Restic to stop backup", func() {
+		Context(`"Local" backend`, func() {
+			AfterEach(func() {
+				f.DeleteDeployment(deployment.ObjectMeta)
+				f.DeleteRestic(restic.ObjectMeta)
+				f.DeleteSecret(cred.ObjectMeta)
+			})
+			BeforeEach(func() {
+				cred = f.SecretForLocalBackend()
+				restic = f.ResticForLocalBackend()
+			})
+			It(`should be able to Pause and Resume backup`, func() {
+				By("Creating repository Secret " + cred.Name)
+				err = f.CreateSecret(cred)
+				Expect(err).NotTo(HaveOccurred())
+
+				By("Creating restic")
+				err = f.CreateRestic(restic)
+				Expect(err).NotTo(HaveOccurred())
+
+				By("Creating Deployment " + deployment.Name)
+				_, err = f.CreateDeployment(deployment)
+				Expect(err).NotTo(HaveOccurred())
+
+				By("Waiting for sidecar")
+				f.EventuallyDeployment(deployment.ObjectMeta).Should(HaveSidecar(util.StashContainer))
+
+				By("Waiting for backup to complete")
+				f.EventuallyRestic(restic.ObjectMeta).Should(WithTransform(func(r *api.Restic) int64 {
+					return r.Status.BackupCount
+				}, BeNumerically(">=", 1)))
+
+				By("Waiting for backup event")
+				f.EventualEvent(restic.ObjectMeta).Should(WithTransform(f.CountSuccessfulBackups, BeNumerically(">=", 1)))
+
+				By(`Patching Restic with "paused: true"`)
+				err = f.CreateOrPatchRestic(restic.ObjectMeta, func(in *api.Restic) *api.Restic {
+					in.Spec.Paused = true
+					return in
+				})
+				Expect(err).NotTo(HaveOccurred())
+
+				resticObj, err := f.StashClient.StashV1alpha1().Restics(restic.Namespace).Get(restic.Name, metav1.GetOptions{})
+				Expect(err).NotTo(HaveOccurred())
+
+				previousBackupCount := resticObj.Status.BackupCount
+
+				By("Wating 2 minutes")
+				time.Sleep(2 * time.Minute)
+
+				By("Checking that Backup count has not changed")
+				resticObj, err = f.StashClient.StashV1alpha1().Restics(restic.Namespace).Get(restic.Name, metav1.GetOptions{})
+				Expect(err).NotTo(HaveOccurred())
+				Expect(resticObj.Status.BackupCount).Should(BeNumerically("==", previousBackupCount))
+
+				By(`Patching Restic with "paused: false"`)
+				err = f.CreateOrPatchRestic(restic.ObjectMeta, func(in *api.Restic) *api.Restic {
+					in.Spec.Paused = false
+					return in
+				})
+				Expect(err).NotTo(HaveOccurred())
+
+				By("Waiting for backup to complete")
+				f.EventuallyRestic(restic.ObjectMeta).Should(WithTransform(func(r *api.Restic) int64 {
+					return r.Status.BackupCount
+				}, BeNumerically(">", previousBackupCount)))
+
+				By("Waiting for backup event")
+				f.EventualEvent(restic.ObjectMeta).Should(WithTransform(f.CountSuccessfulBackups, BeNumerically(">", previousBackupCount)))
+
+			})
+
+		})
+	})
 })