Add support Disabling TLS Certificate Verification for Secure S3 Stor…

…age (#2017) (#2025)

Signed-off-by: Anisur Rahman <anisur@appscode.com>

go.mod

@@ -19,7 +19,7 @@ require (
 	kmodules.xyz/custom-resources v0.29.0
 	kmodules.xyz/offshoot-api v0.29.0
 	kubedb.dev/apimachinery v0.40.1-0.20240101000103-032b27211164
-	stash.appscode.dev/apimachinery v0.32.1-0.20240101013736-ef308633d8b2
+	stash.appscode.dev/apimachinery v0.32.1-0.20240202121916-8f4a855a72c8
 )
 
 require (

go.sum

@@ -558,5 +558,5 @@ sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+s
 sigs.k8s.io/structured-merge-diff/v4 v4.4.1/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08=
 sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E=
 sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY=
-stash.appscode.dev/apimachinery v0.32.1-0.20240101013736-ef308633d8b2 h1:dePrbjp7o57sKe33K1ppaWQK/Ely4QgxmCcab0sDOpY=
-stash.appscode.dev/apimachinery v0.32.1-0.20240101013736-ef308633d8b2/go.mod h1:hTslVqyx20fF2i2s/m0rqXD+pZwnI2oG3k5zPzsDnXQ=
+stash.appscode.dev/apimachinery v0.32.1-0.20240202121916-8f4a855a72c8 h1:p3kd1saM5Ehz9mkqJhVrcFDhY0Bjv8S0XYuq2dZb3KI=
+stash.appscode.dev/apimachinery v0.32.1-0.20240202121916-8f4a855a72c8/go.mod h1:cPdkM9Upe5hxgxrV+WlIQCq7K0Tpx//VUwFs0NKB7ek=

pkg/backup.go

@@ -173,6 +173,7 @@ func NewCmdBackup() *cobra.Command {
 	cmd.Flags().StringVar(&opt.setupOptions.Provider, "provider", opt.setupOptions.Provider, "Backend provider (i.e. gcs, s3, azure etc)")
 	cmd.Flags().StringVar(&opt.setupOptions.Bucket, "bucket", opt.setupOptions.Bucket, "Name of the cloud bucket/container (keep empty for local backend)")
 	cmd.Flags().StringVar(&opt.setupOptions.Endpoint, "endpoint", opt.setupOptions.Endpoint, "Endpoint for s3/s3 compatible backend or REST server URL")
+	cmd.Flags().BoolVar(&opt.setupOptions.InsecureTLS, "insecure-tls", opt.setupOptions.InsecureTLS, "InsecureTLS for TLS secure s3/s3 compatible backend")
 	cmd.Flags().StringVar(&opt.setupOptions.Region, "region", opt.setupOptions.Region, "Region for s3/s3 compatible backend")
 	cmd.Flags().StringVar(&opt.setupOptions.Path, "path", opt.setupOptions.Path, "Directory inside the bucket where backup will be stored")
 	cmd.Flags().StringVar(&opt.setupOptions.ScratchDir, "scratch-dir", opt.setupOptions.ScratchDir, "Temporary directory")

pkg/restore.go

@@ -134,6 +134,7 @@ func NewCmdRestore() *cobra.Command {
 	cmd.Flags().StringVar(&opt.setupOptions.Provider, "provider", opt.setupOptions.Provider, "Backend provider (i.e. gcs, s3, azure etc)")
 	cmd.Flags().StringVar(&opt.setupOptions.Bucket, "bucket", opt.setupOptions.Bucket, "Name of the cloud bucket/container (keep empty for local backend)")
 	cmd.Flags().StringVar(&opt.setupOptions.Endpoint, "endpoint", opt.setupOptions.Endpoint, "Endpoint for s3/s3 compatible backend or REST server URL")
+	cmd.Flags().BoolVar(&opt.setupOptions.InsecureTLS, "insecure-tls", opt.setupOptions.InsecureTLS, "InsecureTLS for TLS secure s3/s3 compatible backend")
 	cmd.Flags().StringVar(&opt.setupOptions.Region, "region", opt.setupOptions.Region, "Region for s3/s3 compatible backend")
 	cmd.Flags().StringVar(&opt.setupOptions.Path, "path", opt.setupOptions.Path, "Directory inside the bucket where backup will be stored")
 	cmd.Flags().StringVar(&opt.setupOptions.ScratchDir, "scratch-dir", opt.setupOptions.ScratchDir, "Temporary directory")

vendor/modules.txt

@@ -705,7 +705,7 @@ sigs.k8s.io/structured-merge-diff/v4/value
 ## explicit; go 1.12
 sigs.k8s.io/yaml
 sigs.k8s.io/yaml/goyaml.v2
-# stash.appscode.dev/apimachinery v0.32.1-0.20240101013736-ef308633d8b2
+# stash.appscode.dev/apimachinery v0.32.1-0.20240202121916-8f4a855a72c8
 ## explicit; go 1.21.5
 stash.appscode.dev/apimachinery/apis
 stash.appscode.dev/apimachinery/apis/repositories

vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_backupblueprints.yaml

@@ -1794,6 +1794,8 @@ spec:
                         type: string
                       endpoint:
                         type: string
+                      insecureTLS:
+                        type: boolean
                       prefix:
                         type: string
                       region:

vendor/stash.appscode.dev/apimachinery/crds/stash.appscode.com_repositories.yaml

@@ -1801,6 +1801,8 @@ spec:
                         type: string
                       endpoint:
                         type: string
+                      insecureTLS:
+                        type: boolean
                       prefix:
                         type: string
                       region:

vendor/stash.appscode.dev/apimachinery/pkg/restic/commands.go

@@ -80,6 +80,7 @@ func (w *ResticWrapper) listSnapshots(snapshotIDs []string) ([]Snapshot, error)
 	result := make([]Snapshot, 0)
 	args := w.appendCacheDirFlag([]interface{}{"snapshots", "--json", "--quiet", "--no-lock"})
 	args = w.appendCaCertFlag(args)
+	args = w.appendInsecureTLSFlag(args)
 	args = w.appendMaxConnectionsFlag(args)
 	for _, id := range snapshotIDs {
 		args = append(args, id)
@@ -95,6 +96,7 @@ func (w *ResticWrapper) listSnapshots(snapshotIDs []string) ([]Snapshot, error)
 func (w *ResticWrapper) deleteSnapshots(snapshotIDs []string) ([]byte, error) {
 	args := w.appendCacheDirFlag([]interface{}{"forget", "--quiet", "--prune"})
 	args = w.appendCaCertFlag(args)
+	args = w.appendInsecureTLSFlag(args)
 	args = w.appendMaxConnectionsFlag(args)
 	for _, id := range snapshotIDs {
 		args = append(args, id)
@@ -107,6 +109,7 @@ func (w *ResticWrapper) repositoryExist() bool {
 	klog.Infoln("Checking whether the backend repository exist or not....")
 	args := w.appendCacheDirFlag([]interface{}{"snapshots", "--json", "--no-lock"})
 	args = w.appendCaCertFlag(args)
+	args = w.appendInsecureTLSFlag(args)
 	args = w.appendMaxConnectionsFlag(args)
 	if _, err := w.run(Command{Name: ResticCMD, Args: args}); err == nil {
 		return true
@@ -122,6 +125,7 @@ func (w *ResticWrapper) initRepository() error {
 
 	args := w.appendCacheDirFlag([]interface{}{"init"})
 	args = w.appendCaCertFlag(args)
+	args = w.appendInsecureTLSFlag(args)
 	args = w.appendMaxConnectionsFlag(args)
 	_, err := w.run(Command{Name: ResticCMD, Args: args})
 	return err
@@ -151,6 +155,7 @@ func (w *ResticWrapper) backup(params backupParams) ([]byte, error) {
 	args = w.appendCacheDirFlag(args)
 	args = w.appendCleanupCacheFlag(args)
 	args = w.appendCaCertFlag(args)
+	args = w.appendInsecureTLSFlag(args)
 	args = w.appendMaxConnectionsFlag(args)
 
 	return w.run(Command{Name: ResticCMD, Args: args})
@@ -174,6 +179,7 @@ func (w *ResticWrapper) backupFromStdin(options BackupOptions) ([]byte, error) {
 	args = w.appendCacheDirFlag(args)
 	args = w.appendCleanupCacheFlag(args)
 	args = w.appendCaCertFlag(args)
+	args = w.appendInsecureTLSFlag(args)
 	args = w.appendMaxConnectionsFlag(args)
 
 	commands = append(commands, Command{Name: ResticCMD, Args: args})
@@ -248,6 +254,7 @@ func (w *ResticWrapper) tryCleanup(retentionPolicy v1alpha1.RetentionPolicy, hos
 	if len(args) > 1 {
 		args = w.appendCacheDirFlag(args)
 		args = w.appendCaCertFlag(args)
+		args = w.appendInsecureTLSFlag(args)
 		args = w.appendMaxConnectionsFlag(args)
 
 		return w.run(Command{Name: ResticCMD, Args: args})
@@ -294,6 +301,7 @@ func (w *ResticWrapper) restore(params restoreParams) ([]byte, error) {
 	}
 	args = w.appendCacheDirFlag(args)
 	args = w.appendCaCertFlag(args)
+	args = w.appendInsecureTLSFlag(args)
 	args = w.appendMaxConnectionsFlag(args)
 
 	return w.run(Command{Name: ResticCMD, Args: args})
@@ -325,6 +333,7 @@ func (w *ResticWrapper) DumpOnce(dumpOptions DumpOptions) ([]byte, error) {
 
 	args = w.appendCacheDirFlag(args)
 	args = w.appendCaCertFlag(args)
+	args = w.appendInsecureTLSFlag(args)
 	args = w.appendMaxConnectionsFlag(args)
 
 	// first add restic command, then add StdoutPipeCommands
@@ -339,6 +348,7 @@ func (w *ResticWrapper) check() ([]byte, error) {
 	klog.Infoln("Checking integrity of repository")
 	args := w.appendCacheDirFlag([]interface{}{"check", "--no-lock"})
 	args = w.appendCaCertFlag(args)
+	args = w.appendInsecureTLSFlag(args)
 	args = w.appendMaxConnectionsFlag(args)
 
 	return w.run(Command{Name: ResticCMD, Args: args})
@@ -353,6 +363,7 @@ func (w *ResticWrapper) stats(snapshotID string) ([]byte, error) {
 	args = w.appendMaxConnectionsFlag(args)
 	args = append(args, "--quiet", "--json", "--mode", "raw-data", "--no-lock")
 	args = w.appendCaCertFlag(args)
+	args = w.appendInsecureTLSFlag(args)
 
 	return w.run(Command{Name: ResticCMD, Args: args})
 }
@@ -362,6 +373,7 @@ func (w *ResticWrapper) unlock() ([]byte, error) {
 	args := w.appendCacheDirFlag([]interface{}{"unlock", "--remove-all"})
 	args = w.appendMaxConnectionsFlag(args)
 	args = w.appendCaCertFlag(args)
+	args = w.appendInsecureTLSFlag(args)
 
 	return w.run(Command{Name: ResticCMD, Args: args})
 }
@@ -515,6 +527,7 @@ func (w *ResticWrapper) addKey(params keyParams) ([]byte, error) {
 	args = w.appendCacheDirFlag(args)
 	args = w.appendMaxConnectionsFlag(args)
 	args = w.appendCaCertFlag(args)
+	args = w.appendInsecureTLSFlag(args)
 
 	return w.run(Command{Name: ResticCMD, Args: args})
 }
@@ -527,6 +540,7 @@ func (w *ResticWrapper) listKey() ([]byte, error) {
 	args = w.appendCacheDirFlag(args)
 	args = w.appendMaxConnectionsFlag(args)
 	args = w.appendCaCertFlag(args)
+	args = w.appendInsecureTLSFlag(args)
 
 	return w.run(Command{Name: ResticCMD, Args: args})
 }
@@ -543,6 +557,7 @@ func (w *ResticWrapper) updateKey(params keyParams) ([]byte, error) {
 	args = w.appendCacheDirFlag(args)
 	args = w.appendMaxConnectionsFlag(args)
 	args = w.appendCaCertFlag(args)
+	args = w.appendInsecureTLSFlag(args)
 
 	return w.run(Command{Name: ResticCMD, Args: args})
 }
@@ -555,6 +570,14 @@ func (w *ResticWrapper) removeKey(params keyParams) ([]byte, error) {
 	args = w.appendCacheDirFlag(args)
 	args = w.appendMaxConnectionsFlag(args)
 	args = w.appendCaCertFlag(args)
+	args = w.appendInsecureTLSFlag(args)
 
 	return w.run(Command{Name: ResticCMD, Args: args})
 }
+
+func (w *ResticWrapper) appendInsecureTLSFlag(args []interface{}) []interface{} {
+	if w.config.InsecureTLS {
+		return append(args, "--insecure-tls")
+	}
+	return args
+}

vendor/stash.appscode.dev/apimachinery/pkg/restic/config.go

@@ -87,6 +87,7 @@ type SetupOptions struct {
 	CacertFile     string
 	ScratchDir     string
 	EnableCache    bool
+	InsecureTLS    bool
 	MaxConnections int64
 	StorageSecret  *core.Secret
 	Nice           *ofst.NiceSettings

vendor/stash.appscode.dev/apimachinery/pkg/util/addon.go

@@ -60,8 +60,8 @@ func ExtractAddonInfo(appClient appcatalog_cs.Interface, task v1beta1.TaskRef, t
 		addon.RestoreTask.Name = task.Name
 	}
 	if len(task.Params) != 0 {
-		addon.BackupTask.Params = getTaskParams(task)
-		addon.RestoreTask.Params = getTaskParams(task)
+		addon.BackupTask.Params = upsertParams(addon.BackupTask.Params, getTaskParams(task))
+		addon.RestoreTask.Params = upsertParams(addon.BackupTask.Params, getTaskParams(task))
 	}
 
 	return &addon, nil
@@ -75,3 +75,24 @@ func getTaskParams(task v1beta1.TaskRef) []appcat.Param {
 	}
 	return params
 }
+
+func upsertParams(oldParams, newParams []appcat.Param) []appcat.Param {
+	paramMap := make(map[string]appcat.Param)
+
+	for _, newParam := range newParams {
+		paramMap[newParam.Name] = newParam
+	}
+
+	for _, oldParam := range oldParams {
+		if _, found := paramMap[oldParam.Name]; !found {
+			paramMap[oldParam.Name] = oldParam
+		}
+	}
+
+	var updatedParams []appcat.Param
+	for _, param := range paramMap {
+		updatedParams = append(updatedParams, param)
+	}
+
+	return updatedParams
+}