Support passing arguments to restic backup/restore command (#121)

Signed-off-by: Emruz Hossain <emruz@appscode.com>

Fixes stashed/stash#1138
Fixes stashed/stash#1346

.config/api-rules/violation_exceptions.list

@@ -178,6 +178,7 @@ API rule violation: list_type_missing,stash.appscode.dev/apimachinery/apis/stash
 API rule violation: list_type_missing,stash.appscode.dev/apimachinery/apis/stash/v1beta1,BackupConfigurationStatus,Conditions
 API rule violation: list_type_missing,stash.appscode.dev/apimachinery/apis/stash/v1beta1,BackupSessionStatus,Conditions
 API rule violation: list_type_missing,stash.appscode.dev/apimachinery/apis/stash/v1beta1,BackupSessionStatus,Targets
+API rule violation: list_type_missing,stash.appscode.dev/apimachinery/apis/stash/v1beta1,BackupTarget,Args
 API rule violation: list_type_missing,stash.appscode.dev/apimachinery/apis/stash/v1beta1,BackupTarget,Exclude
 API rule violation: list_type_missing,stash.appscode.dev/apimachinery/apis/stash/v1beta1,BackupTarget,Paths
 API rule violation: list_type_missing,stash.appscode.dev/apimachinery/apis/stash/v1beta1,BackupTarget,VolumeMounts
@@ -200,6 +201,7 @@ API rule violation: list_type_missing,stash.appscode.dev/apimachinery/apis/stash
 API rule violation: list_type_missing,stash.appscode.dev/apimachinery/apis/stash/v1beta1,RestoreSessionSpec,Rules
 API rule violation: list_type_missing,stash.appscode.dev/apimachinery/apis/stash/v1beta1,RestoreSessionStatus,Conditions
 API rule violation: list_type_missing,stash.appscode.dev/apimachinery/apis/stash/v1beta1,RestoreSessionStatus,Stats
+API rule violation: list_type_missing,stash.appscode.dev/apimachinery/apis/stash/v1beta1,RestoreTarget,Args
 API rule violation: list_type_missing,stash.appscode.dev/apimachinery/apis/stash/v1beta1,RestoreTarget,Rules
 API rule violation: list_type_missing,stash.appscode.dev/apimachinery/apis/stash/v1beta1,RestoreTarget,VolumeClaimTemplates
 API rule violation: list_type_missing,stash.appscode.dev/apimachinery/apis/stash/v1beta1,RestoreTarget,VolumeMounts

apis/stash/v1beta1/types.go

@@ -75,6 +75,9 @@ type BackupTarget struct {
 	// Supported only for "Restic" driver
 	// +optional
 	Exclude []string `json:"exclude,omitempty" protobuf:"bytes,7,rep,name=exclude"`
+	// Args specifies a list of arguments to pass to the backup driver.
+	// +optional
+	Args []string `json:"args,omitempty" protobuf:"bytes,8,rep,name=args"`
 }
 
 type RestoreTarget struct {
@@ -100,6 +103,9 @@ type RestoreTarget struct {
 	// Rules specifies different restore options for different hosts
 	// +optional
 	Rules []Rule `json:"rules,omitempty" protobuf:"bytes,6,rep,name=rules"`
+	// Args specifies a list of arguments to pass to the restore driver.
+	// +optional
+	Args []string `json:"args,omitempty" protobuf:"bytes,7,rep,name=args"`
 }
 
 type Rule struct {

crds/stash.appscode.com_backupbatches.yaml

@@ -3175,6 +3175,12 @@ spec:
                             or will be used to generate the `hostname` for the restic
                             repository.
                           type: string
+                        args:
+                          description: Args specifies a list of arguments to pass
+                            to the backup driver.
+                          items:
+                            type: string
+                          type: array
                         exclude:
                           description: Exclude specifies a list of patterns for the
                             files to ignore during backup. Stash will ignore those

crds/stash.appscode.com_backupconfigurations.yaml

@@ -2762,6 +2762,12 @@ spec:
                       data in the repository. This will be used as `hostname` or will
                       be used to generate the `hostname` for the restic repository.
                     type: string
+                  args:
+                    description: Args specifies a list of arguments to pass to the
+                      backup driver.
+                    items:
+                      type: string
+                    type: array
                   exclude:
                     description: Exclude specifies a list of patterns for the files
                       to ignore during backup. Stash will ignore those files that

crds/stash.appscode.com_restorebatches.yaml

@@ -3169,6 +3169,12 @@ spec:
                             up data in the repository. This will be used as `sourceHost`
                             and `targetHosts` or will be used to generate them.
                           type: string
+                        args:
+                          description: Args specifies a list of arguments to pass
+                            to the restore driver.
+                          items:
+                            type: string
+                          type: array
                         ref:
                           description: Ref refers to the restore,target
                           properties:

crds/stash.appscode.com_restoresessions.yaml

@@ -2761,6 +2761,12 @@ spec:
                       data in the repository. This will be used as `sourceHost` and
                       `targetHosts` or will be used to generate them.
                     type: string
+                  args:
+                    description: Args specifies a list of arguments to pass to the
+                      restore driver.
+                    items:
+                      type: string
+                    type: array
                   ref:
                     description: Ref refers to the restore,target
                     properties:

openapi/swagger.json

@@ -11953,6 +11953,14 @@
           "description": "Alias represents the identifier of the backed up data in the repository. This will be used as `hostname` or will be used to generate the `hostname` for the restic repository.",
           "type": "string"
         },
+        "args": {
+          "description": "Args specifies a list of arguments to pass to the backup driver.",
+          "type": "array",
+          "items": {
+            "type": "string",
+            "default": ""
+          }
+        },
         "exclude": {
           "description": "Exclude specifies a list of patterns for the files to ignore during backup. Stash will ignore those files that match the specified patterns. Supported only for \"Restic\" driver",
           "type": "array",
@@ -12635,6 +12643,14 @@
           "description": "Alias represents the identifier of the backed up data in the repository. This will be used as `sourceHost` and `targetHosts` or will be used to generate them.",
           "type": "string"
         },
+        "args": {
+          "description": "Args specifies a list of arguments to pass to the restore driver.",
+          "type": "array",
+          "items": {
+            "type": "string",
+            "default": ""
+          }
+        },
         "ref": {
           "description": "Ref refers to the restore,target",
           "default": {},

pkg/restic/backup.go

@@ -148,6 +148,7 @@ func (w *ResticWrapper) runBackup(backupOption BackupOptions) (api_v1beta1.HostB
 			path:     path,
 			host:     backupOption.Host,
 			excludes: backupOption.Exclude,
+			args:     backupOption.Args,
 		}
 		out, err := w.backup(params)
 		if err != nil {

pkg/restic/commands.go

@@ -56,6 +56,7 @@ type backupParams struct {
 	host     string
 	tags     []string
 	excludes []string
+	args     []string
 }
 
 type restoreParams struct {
@@ -65,6 +66,7 @@ type restoreParams struct {
 	destination string
 	excludes    []string
 	includes    []string
+	args        []string
 }
 
 func (w *ResticWrapper) listSnapshots(snapshotIDs []string) ([]Snapshot, error) {
@@ -131,6 +133,10 @@ func (w *ResticWrapper) backup(params backupParams) ([]byte, error) {
 		args = append(args, "--exclude")
 		args = append(args, exclude)
 	}
+	// add additional arguments passed by user to the backup process
+	for i := range params.args {
+		args = append(args, params.args[i])
+	}
 	args = w.appendCacheDirFlag(args)
 	args = w.appendCleanupCacheFlag(args)
 	args = w.appendCaCertFlag(args)
@@ -256,6 +262,10 @@ func (w *ResticWrapper) restore(params restoreParams) ([]byte, error) {
 		args = append(args, "--exclude")
 		args = append(args, exclude)
 	}
+	// add additional arguments passed by user to the restore process
+	for i := range params.args {
+		args = append(args, params.args[i])
+	}
 	args = w.appendCacheDirFlag(args)
 	args = w.appendCaCertFlag(args)
 	args = w.appendMaxConnectionsFlag(args)

pkg/restic/config.go

@@ -54,6 +54,7 @@ type BackupOptions struct {
 	StdinFileName     string // default "stdin"
 	RetentionPolicy   v1alpha1.RetentionPolicy
 	Exclude           []string
+	Args              []string
 }
 
 // RestoreOptions specifies restore information
@@ -65,6 +66,7 @@ type RestoreOptions struct {
 	Destination  string   // destination path where snapshot will be restored, used in cli
 	Exclude      []string
 	Include      []string
+	Args         []string
 }
 
 type DumpOptions struct {

pkg/restic/restic_test.go

@@ -665,6 +665,88 @@ func TestIncludeExcludePattern(t *testing.T) {
 	}
 }
 
+func TestBackupRestoreWithArgs(t *testing.T) {
+	retentionPolicy := api_v1alpha1.RetentionPolicy{
+		Name:     "keep-last-1",
+		KeepLast: 1,
+		Prune:    true,
+		DryRun:   false,
+	}
+
+	testCases := []struct {
+		name       string
+		backupOpt  BackupOptions
+		restoreOpt RestoreOptions
+	}{
+		{
+			name: "pass --ignore-inode flag during backup",
+			backupOpt: BackupOptions{
+				RetentionPolicy: retentionPolicy,
+				Args:            []string{"--ignore-inode"},
+			},
+		},
+		{
+			name: "pass --tags during backup and restore",
+			backupOpt: BackupOptions{
+				RetentionPolicy: retentionPolicy,
+				Args:            []string{"--tag=t1,t2"},
+			},
+			restoreOpt: RestoreOptions{
+				Args: []string{"--tag=t1,t2"},
+			},
+		},
+	}
+	for _, test := range testCases {
+		t.Run(test.name, func(t *testing.T) {
+			tempDir, err := ioutil.TempDir("", "stash-unit-test-")
+			if err != nil {
+				t.Error(err)
+				return
+			}
+
+			w, err := setupTest(tempDir)
+			if err != nil {
+				t.Error(err)
+				return
+			}
+			defer cleanup(tempDir)
+
+			// Initialize Repository
+			err = w.InitializeRepository()
+			if err != nil {
+				t.Error(err)
+			}
+
+			// create the source files
+			err = os.Remove(filepath.Join(targetPath, fileName))
+			if err != nil {
+				t.Error(err)
+				return
+			}
+			test.backupOpt.BackupPaths = []string{targetPath}
+
+			_, err = w.RunBackup(test.backupOpt, testTargetRef)
+			if err != nil {
+				t.Error(err)
+				return
+			}
+
+			// delete target then restore
+			if err = os.RemoveAll(targetPath); err != nil {
+				t.Error(err)
+				return
+			}
+			test.restoreOpt.RestorePaths = []string{targetPath}
+
+			_, err = w.RunRestore(test.restoreOpt, testTargetRef)
+			if err != nil {
+				t.Error(err)
+				return
+			}
+		})
+	}
+}
+
 func TestApplyRetentionPolicy(t *testing.T) {
 	tempDir, err := ioutil.TempDir("", "stash-unit-test-")
 	if err != nil {

pkg/restic/restore.go

@@ -233,6 +233,7 @@ func (w *ResticWrapper) runRestore(restoreOptions RestoreOptions) error {
 				snapshotId:  snapshot,
 				excludes:    restoreOptions.Exclude,
 				includes:    restoreOptions.Include,
+				args:        restoreOptions.Args,
 			}
 			if _, err := w.restore(params); err != nil {
 				return err
@@ -246,6 +247,7 @@ func (w *ResticWrapper) runRestore(restoreOptions RestoreOptions) error {
 				destination: restoreOptions.Destination,
 				excludes:    restoreOptions.Exclude,
 				includes:    restoreOptions.Include,
+				args:        restoreOptions.Args,
 			}
 			if _, err := w.restore(params); err != nil {
 				return err