Skip to content

Commit

Permalink
Fix MongoDB url connection str for external databases (#2138) (#2142)
Browse files Browse the repository at this point in the history 
Signed-off-by: Anisur Rahman <anisur@appscode.com>
Signed-off-by: sayedppqq <sayed@appscode.com>
Co-authored-by: sayedppqq <sayed@appscode.com>
  • Loading branch information
@1gtm @sayedppqq
1gtm and sayedppqq committed Aug 27, 2024
1 parent 1d63b61 commit d440be0
Show file tree
Hide file tree
Showing 3 changed files with 174 additions and 35 deletions.
77 changes: 61 additions & 16 deletions pkg/backup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -267,11 +267,24 @@ func (opt *mongoOptions) backupMongoDB(targetRef api_v1beta1.TargetRef) (*restic
return nil, err
}

port, err := appBinding.Port()
if err != nil {
return nil, err
var isSrv bool
port := int32(27017)
if appBinding.Spec.ClientConfig.URL != nil {
isSrv, err = isSrvConnection(*appBinding.Spec.ClientConfig.URL)
if err != nil {
return nil, err
}
}

// Checked for Altlas and DigitalOcean srv format connection string don't give port.
// mongodump not support both --uri and --port.

if !isSrv {
port, err = appBinding.Port()
if err != nil {
return nil, err
}
}
waitForDBReady(hostname, port, opt.waitTimeout)

// unmarshal parameter is the field has value
Expand Down Expand Up @@ -318,7 +331,12 @@ func (opt *mongoOptions) backupMongoDB(targetRef api_v1beta1.TargetRef) (*restic
}
}

var tlsEnable bool
if appBinding.Spec.ClientConfig.CABundle != nil {
tlsEnable = true
}

if tlsEnable {
if tlsSecret == nil {
return nil, errors.Wrap(err, "spec.tlsSecret needs to be set in appbinding for TLS secured database.")
}
Expand All @@ -333,8 +351,8 @@ func (opt *mongoOptions) backupMongoDB(targetRef api_v1beta1.TargetRef) (*restic
}
dumpCreds = []interface{}{
"--ssl",
"--sslCAFile", filepath.Join(opt.setupOptions.ScratchDir, MongoTLSCertFileName),
"--sslPEMKeyFile", filepath.Join(opt.setupOptions.ScratchDir, MongoClientPemFileName),
fmt.Sprintf("--sslCAFile=%s", filepath.Join(opt.setupOptions.ScratchDir, MongoTLSCertFileName)),
fmt.Sprintf("--sslPEMKeyFile=%s", filepath.Join(opt.setupOptions.ScratchDir, MongoClientPemFileName)),
}

// get certificate secret to get client certificate
Expand All @@ -361,9 +379,9 @@ func (opt *mongoOptions) backupMongoDB(targetRef api_v1beta1.TargetRef) (*restic
return nil, errors.Wrap(err, "unable to get user from ssl.")
}
userAuth := []interface{}{
"-u", user,
"--authenticationMechanism", "MONGODB-X509",
"--authenticationDatabase", "$external",
fmt.Sprintf("--username=%s", user),
"--authenticationMechanism=MONGODB-X509",
"--authenticationDatabase=$external",
}
mongoCreds = append(mongoCreds, userAuth...)
dumpCreds = append(dumpCreds, userAuth...)
Expand All @@ -372,7 +390,7 @@ func (opt *mongoOptions) backupMongoDB(targetRef api_v1beta1.TargetRef) (*restic
userAuth := []interface{}{
fmt.Sprintf("--username=%s", authSecret.Data[MongoUserKey]),
fmt.Sprintf("--password=%s", authSecret.Data[MongoPasswordKey]),
"--authenticationDatabase", opt.authenticationDatabase,
fmt.Sprintf("--authenticationDatabase=%s", opt.authenticationDatabase),
}
mongoCreds = append(mongoCreds, userAuth...)
dumpCreds = append(dumpCreds, userAuth...)
Expand All @@ -387,19 +405,32 @@ func (opt *mongoOptions) backupMongoDB(targetRef api_v1beta1.TargetRef) (*restic
BackupPaths: opt.defaultBackupOptions.BackupPaths,
}

uri := opt.buildMongoURI(mongoDSN, port, isStandalone, isSrv, tlsEnable)

// setup pipe command
backupCmd := restic.Command{
Name: MongoDumpCMD,
Args: append([]interface{}{
"--host", mongoDSN,
Args: []interface{}{
"--uri", fmt.Sprintf("\"%s\"", uri),
"--archive",
}, dumpCreds...),
},
}
userArgs := strings.Fields(opt.mongoArgs)

if isStandalone {
backupCmd.Args = append(backupCmd.Args, fmt.Sprintf("--port=%d", port))
} else {
if tlsEnable {
backupCmd.Args = append(backupCmd.Args,
fmt.Sprintf("--sslCAFile=%s", getOptionValue(dumpCreds, "--sslCAFile")),
fmt.Sprintf("--sslPEMKeyFile=%s", getOptionValue(dumpCreds, "--sslPEMKeyFile")))
}

var userArgs []string
for _, arg := range strings.Fields(opt.mongoArgs) {
// illegal argument combination: cannot specify --db and --uri
if !strings.Contains(arg, "--db") {
userArgs = append(userArgs, arg)
}
}

if !isStandalone {
// - port is already added in mongoDSN with replicasetName/host:port format.
// - oplog is enabled automatically for replicasets.
// Don't use --oplog if user specify any of these arguments through opt.mongoArgs
Expand Down Expand Up @@ -558,6 +589,20 @@ func cleanup() {
}
}

func getOptionValue(args []interface{}, option string) string {
for _, arg := range args {
strArg, ok := arg.(string)
if !ok {
continue
}
// assuming value has '='
if strings.HasPrefix(strArg, option+"=") {
return strings.TrimPrefix(strArg, option+"=")
}
}
return ""
}

func (opt *mongoOptions) getHostBackupStats(err error) []api_v1beta1.HostBackupStats {
var backupStats []api_v1beta1.HostBackupStats

Expand Down
69 changes: 50 additions & 19 deletions pkg/restore.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -204,9 +204,23 @@ func (opt *mongoOptions) restoreMongoDB(targetRef api_v1beta1.TargetRef) (*resti
return nil, err
}

port, err := appBinding.Port()
if err != nil {
return nil, err
var isSrv bool
port := int32(27017)
if appBinding.Spec.ClientConfig.URL != nil {
isSrv, err = isSrvConnection(*appBinding.Spec.ClientConfig.URL)
if err != nil {
return nil, err
}
}

// Checked for Altlas and DigitalOcean srv format connection string don't give port.
// mongodump --uri format not support port.

if !isSrv {
port, err = appBinding.Port()
if err != nil {
return nil, err
}
}

// unmarshal parameter is the field has value
Expand Down Expand Up @@ -249,8 +263,12 @@ func (opt *mongoOptions) restoreMongoDB(targetRef api_v1beta1.TargetRef) (*resti
return nil, err
}
}

var tlsEnable bool
if appBinding.Spec.ClientConfig.CABundle != nil {
tlsEnable = true
}

if tlsEnable {
if err := os.WriteFile(filepath.Join(opt.setupOptions.ScratchDir, MongoTLSCertFileName), appBinding.Spec.ClientConfig.CABundle, os.ModePerm); err != nil {
return nil, errors.Wrap(err, "failed to write key for CA certificate")
}
Expand All @@ -261,8 +279,8 @@ func (opt *mongoOptions) restoreMongoDB(targetRef api_v1beta1.TargetRef) (*resti
}
dumpCreds = []interface{}{
"--ssl",
"--sslCAFile", filepath.Join(opt.setupOptions.ScratchDir, MongoTLSCertFileName),
"--sslPEMKeyFile", filepath.Join(opt.setupOptions.ScratchDir, MongoClientPemFileName),
fmt.Sprintf("--sslCAFile=%s", filepath.Join(opt.setupOptions.ScratchDir, MongoTLSCertFileName)),
fmt.Sprintf("--sslPEMKeyFile=%s", filepath.Join(opt.setupOptions.ScratchDir, MongoClientPemFileName)),
}

// get certificate secret to get client certificate
Expand All @@ -289,9 +307,9 @@ func (opt *mongoOptions) restoreMongoDB(targetRef api_v1beta1.TargetRef) (*resti
return nil, errors.Wrap(err, "unable to get user from ssl.")
}
userAuth := []interface{}{
"-u", user,
"--authenticationMechanism", "MONGODB-X509",
"--authenticationDatabase", "$external",
fmt.Sprintf("--username=%s", user),
"--authenticationMechanism=MONGODB-X509",
"--authenticationDatabase=$external",
}
mongoCreds = append(mongoCreds, userAuth...)
dumpCreds = append(dumpCreds, userAuth...)
Expand All @@ -300,7 +318,7 @@ func (opt *mongoOptions) restoreMongoDB(targetRef api_v1beta1.TargetRef) (*resti
userAuth := []interface{}{
fmt.Sprintf("--username=%s", authSecret.Data[MongoUserKey]),
fmt.Sprintf("--password=%s", authSecret.Data[MongoPasswordKey]),
"--authenticationDatabase", opt.authenticationDatabase,
fmt.Sprintf("--authenticationDatabase=%s", opt.authenticationDatabase),
}
mongoCreds = append(mongoCreds, userAuth...)
dumpCreds = append(dumpCreds, userAuth...)
Expand All @@ -314,19 +332,32 @@ func (opt *mongoOptions) restoreMongoDB(targetRef api_v1beta1.TargetRef) (*resti
FileName: opt.defaultDumpOptions.FileName,
Snapshot: opt.getSnapshotForHost(hostKey, restoreSession.Spec.Target.Rules),
}

uri := opt.buildMongoURI(mongoDSN, port, isStandalone, isSrv, tlsEnable)

// setup pipe command
restoreCmd := restic.Command{
Name: MongoRestoreCMD,
Args: append([]interface{}{
"--host", mongoDSN,
Args: []interface{}{
"--uri", fmt.Sprintf("\"%s\"", uri),
"--archive",
}, dumpCreds...),
},
}
if tlsEnable {
restoreCmd.Args = append(restoreCmd.Args,
fmt.Sprintf("--sslCAFile=%s", getOptionValue(dumpCreds, "--sslCAFile")),
fmt.Sprintf("--sslPEMKeyFile=%s", getOptionValue(dumpCreds, "--sslPEMKeyFile")))
}

var userArgs []string
for _, arg := range strings.Fields(opt.mongoArgs) {
// illegal argument combination: cannot specify --db and --uri
if !strings.Contains(arg, "--db") {
userArgs = append(userArgs, arg)
}
}

userArgs := strings.Fields(opt.mongoArgs)
if isStandalone {
restoreCmd.Args = append(restoreCmd.Args, fmt.Sprintf("--port=%d", port))
} else {
if !isStandalone {
// - port is already added in mongoDSN with replicasetName/host:port format.
// - oplog is enabled automatically for replicasets.
// Don't use --oplogReplay if user specify any of these arguments through opt.mongoArgs
Expand Down Expand Up @@ -369,11 +400,11 @@ func (opt *mongoOptions) restoreMongoDB(targetRef api_v1beta1.TargetRef) (*resti
// ref: https://docs.mongodb.com/manual/tutorial/backup-sharded-cluster-with-database-dumps/

if parameters.ConfigServer != "" {
opt.dumpOptions = append(opt.dumpOptions, getDumpOpts(parameters.ConfigServer, MongoConfigSVRHostKey, false))
opt.dumpOptions = append(opt.dumpOptions, getDumpOpts(extractHost(parameters.ConfigServer), MongoConfigSVRHostKey, false))
}

for key, host := range parameters.ReplicaSets {
opt.dumpOptions = append(opt.dumpOptions, getDumpOpts(host, key, false))
opt.dumpOptions = append(opt.dumpOptions, getDumpOpts(extractHost(host), key, false))
}

// if parameters.ReplicaSets is nil, then perform normal backup with clientconfig.Service.Name mongo dsn
Expand Down
63 changes: 63 additions & 0 deletions pkg/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package pkg

import (
"fmt"
"net/url"
"os/exec"
"strings"
"time"
Expand Down Expand Up @@ -114,3 +115,65 @@ func getTime(t string) (time.Time, error) {
}
return parsedTime, nil
}

func isSrvConnection(connectionString string) (bool, error) {
parsedURL, err := url.Parse(connectionString)
if err != nil {
return false, err
}

// Check if the scheme is "mongodb+srv"
return parsedURL.Scheme == "mongodb+srv", nil
}

func (opt *mongoOptions) buildMongoURI(mongoDSN string, port int32, isStandalone, isSrv, tlsEnable bool) string {
prefix, ssl := "mongodb", ""
portStr := fmt.Sprintf(":%d", port)
if isSrv {
prefix += "+srv"
}
if !isStandalone || isSrv {
portStr = ""
}

backupDb := getBackupDB(opt.mongoArgs) // "" stands for all databases.
authDbName := getOptionValue(dumpCreds, "--authenticationDatabase")
userName := getOptionValue(dumpCreds, "--username")
password := getOptionValue(dumpCreds, "--password")
authMechanism := getOptionValue(dumpCreds, "--authenticationMechanism")

if password != "" {
password = fmt.Sprintf(":%s", password)
}
if authMechanism == "" {
authMechanism = "SCRAM-SHA-256"
}
if tlsEnable {
ssl = "&ssl=true"
}

return fmt.Sprintf("%s://%s%s@%s%s/%s?authSource=%s&authMechanism=%s%s",
prefix, userName, password, mongoDSN, portStr, backupDb, authDbName, authMechanism, ssl)
}

// remove "shard0/" prefix from shard0/simple-shard0-0.simple-shard0-pods.demo.svc:27017,simple-shard0-1.simple-shard0-pods.demo.svc:27017
func extractHost(host string) string {
index := strings.Index(host, "/")
if index != -1 {
host = host[index+1:]
}
return host
}

func getBackupDB(mongoArgs string) string {
backupdb := "" // full
if strings.Contains(mongoArgs, "--db") {
args := strings.Fields(mongoArgs)
for _, arg := range args {
if strings.Contains(arg, "--db") {
backupdb = strings.Split(arg, "=")[1]
}
}
}
return backupdb
}

0 comments on commit d440be0

Please sign in to comment.