Skip to content

Commit

Permalink
Make Redis addon ready to use (#1)
Browse files Browse the repository at this point in the history 
Signed-off-by: hossainemruz <emruz@appscode.com>
  • Loading branch information
Emruz Hossain authored Jul 29, 2021
1 parent f5cf21d commit 7d4bfee
Show file tree
Hide file tree
Showing 6 changed files with 103 additions and 32 deletions.
4 changes: 2 additions & 2 deletions Dockerfile.dbg
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,12 @@ RUN set -x \
&& apt-get install -y --no-install-recommends apt-transport-https ca-certificates curl bzip2 tar

RUN set -x \
&& curl -fsSL -o restic.bz2 https://github.com/restic/restic/releases/download/v{RESTIC_VER}/restic_{RESTIC_VER}_{ARG_OS}_{ARG_ARCH}.bz2 \
&& curl -fsSL -o restic.bz2 https://github.com/stashed/restic/releases/download/v{RESTIC_VER}/restic_{RESTIC_VER}_{ARG_OS}_{ARG_ARCH}.bz2 \
&& bzip2 -d restic.bz2 \
&& chmod 755 restic

RUN set -x \
&& curl -fsSL -o redis-dump-go.tar.gz https://github.com/yannh/redis-dump-go/releases/download/v{REDIS_DUMP_VER}/redis-dump-go-{ARG_OS}-{ARG_ARCH}.tar.gz \
&& curl -fsSL -o redis-dump-go.tar.gz https://github.com/yannh/redis-dump-go/releases/download/v{REDIS_DUMP_VER}/redis-dump-go_{REDIS_DUMP_VER}_{ARG_OS}_{ARG_ARCH}.tar.gz \
&& tar -xzf redis-dump-go.tar.gz \
&& chmod 755 redis-dump-go

Expand Down
4 changes: 2 additions & 2 deletions Dockerfile.in
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,12 @@ RUN set -x \
&& apt-get install -y --no-install-recommends apt-transport-https ca-certificates curl bzip2 tar

RUN set -x \
&& curl -fsSL -o restic.bz2 https://github.com/restic/restic/releases/download/v{RESTIC_VER}/restic_{RESTIC_VER}_{ARG_OS}_{ARG_ARCH}.bz2 \
&& curl -fsSL -o restic.bz2 https://github.com/stashed/restic/releases/download/v{RESTIC_VER}/restic_{RESTIC_VER}_{ARG_OS}_{ARG_ARCH}.bz2 \
&& bzip2 -d restic.bz2 \
&& chmod 755 restic

RUN set -x \
&& curl -fsSL -o redis-dump-go.tar.gz https://github.com/yannh/redis-dump-go/releases/download/v{REDIS_DUMP_VER}/redis-dump-go-{ARG_OS}-{ARG_ARCH}.tar.gz \
&& curl -fsSL -o redis-dump-go.tar.gz https://github.com/yannh/redis-dump-go/releases/download/v{REDIS_DUMP_VER}/redis-dump-go_{REDIS_DUMP_VER}_{ARG_OS}_{ARG_ARCH}.tar.gz \
&& tar -xzf redis-dump-go.tar.gz \
&& chmod 755 redis-dump-go

Expand Down
4 changes: 2 additions & 2 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,8 @@ BIN_PLATFORMS := $(DOCKER_PLATFORMS)
OS := $(if $(GOOS),$(GOOS),$(shell go env GOOS))
ARCH := $(if $(GOARCH),$(GOARCH),$(shell go env GOARCH))

BASEIMAGE_PROD ?= redis:5.0.3
BASEIMAGE_DBG ?= redis:5.0.3
BASEIMAGE_PROD ?= redis:6.2.5
BASEIMAGE_DBG ?= redis:6.2.5

IMAGE := $(REGISTRY)/$(BIN)
VERSION_PROD := $(VERSION)
Expand Down
24 changes: 16 additions & 8 deletions pkg/backup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,6 @@ func NewCmdBackup() *cobra.Command {
masterURL string
kubeconfigPath string
opt = redisOptions{

myArgs: "--all-databases",
waitTimeout: 300,
setupOptions: restic.SetupOptions{
ScratchDir: restic.DefaultScratchDir,
Expand Down Expand Up @@ -116,7 +114,7 @@ func NewCmdBackup() *cobra.Command {
},
}

cmd.Flags().StringVar(&opt.myArgs, "redis-args", opt.myArgs, "Additional arguments")
cmd.Flags().StringVar(&opt.redisArgs, "redis-args", opt.redisArgs, "Additional arguments")
cmd.Flags().Int32Var(&opt.waitTimeout, "wait-timeout", opt.waitTimeout, "Time limit to wait for the database to be ready")

cmd.Flags().StringVar(&masterURL, "master", masterURL, "The address of the Kubernetes API server (overrides any value in kubeconfig)")
Expand Down Expand Up @@ -192,27 +190,37 @@ func (opt *redisOptions) backupRedis(targetRef api_v1beta1.TargetRef) (*restic.B
return nil, err
}

// set access credentials
err = opt.setCredentials(resticWrapper, appBinding)
if err != nil {
return nil, err
}

// setup pipe command
opt.backupOptions.StdinPipeCommand = restic.Command{
backupCmd := restic.Command{
Name: RedisDumpCMD,
Args: []interface{}{
"-host", appBinding.Spec.ClientConfig.Service.Name,
},
}
for _, arg := range strings.Fields(opt.myArgs) {
opt.backupOptions.StdinPipeCommand.Args = append(opt.backupOptions.StdinPipeCommand.Args, arg)
for _, arg := range strings.Fields(opt.redisArgs) {
backupCmd.Args = append(backupCmd.Args, arg)
}

// if port is specified, append port in the arguments
if appBinding.Spec.ClientConfig.Service.Port != 0 {
opt.backupOptions.StdinPipeCommand.Args = append(opt.backupOptions.StdinPipeCommand.Args, "-port", strconv.Itoa(int(appBinding.Spec.ClientConfig.Service.Port)))
backupCmd.Args = append(backupCmd.Args, "-port", strconv.Itoa(int(appBinding.Spec.ClientConfig.Service.Port)))
}

// wait for DB ready
err = waitForDBReady(appBinding)
err = opt.waitForDBReady(appBinding)
if err != nil {
return nil, err
}

// add backup command in the pipeline
opt.backupOptions.StdinPipeCommands = append(opt.backupOptions.StdinPipeCommands, backupCmd)

// Run backup
return resticWrapper.RunBackup(opt.backupOptions, targetRef)
}
21 changes: 15 additions & 6 deletions pkg/restore.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ func NewCmdRestore() *cobra.Command {
},
}

cmd.Flags().StringVar(&opt.myArgs, "redis-args", opt.myArgs, "Additional arguments")
cmd.Flags().StringVar(&opt.redisArgs, "redis-args", opt.redisArgs, "Additional arguments")
cmd.Flags().Int32Var(&opt.waitTimeout, "wait-timeout", opt.waitTimeout, "Time limit to wait for the database to be ready")

cmd.Flags().StringVar(&masterURL, "master", masterURL, "The address of the Kubernetes API server (overrides any value in kubeconfig)")
Expand Down Expand Up @@ -161,28 +161,37 @@ func (opt *redisOptions) restoreRedis(targetRef api_v1beta1.TargetRef) (*restic.
return nil, err
}

// set access credentials
err = opt.setCredentials(resticWrapper, appBinding)
if err != nil {
return nil, err
}

// setup pipe command
opt.dumpOptions.StdoutPipeCommand = restic.Command{
restoreCmd := restic.Command{
Name: RedisRestoreCMD,
Args: []interface{}{
"--pipe",
"-h", appBinding.Spec.ClientConfig.Service.Name,
},
}
for _, arg := range strings.Fields(opt.myArgs) {
opt.dumpOptions.StdoutPipeCommand.Args = append(opt.dumpOptions.StdoutPipeCommand.Args, arg)
for _, arg := range strings.Fields(opt.redisArgs) {
restoreCmd.Args = append(restoreCmd.Args, arg)
}
// if port is specified, append port in the arguments
if appBinding.Spec.ClientConfig.Service.Port != 0 {
opt.backupOptions.StdinPipeCommand.Args = append(opt.backupOptions.StdinPipeCommand.Args, "-p", strconv.Itoa(int(appBinding.Spec.ClientConfig.Service.Port)))
restoreCmd.Args = append(restoreCmd.Args, "-p", strconv.Itoa(int(appBinding.Spec.ClientConfig.Service.Port)))
}

// wait for DB ready
err = waitForDBReady(appBinding)
err = opt.waitForDBReady(appBinding)
if err != nil {
return nil, err
}

// append the restore command to the pipeline
opt.dumpOptions.StdoutPipeCommands = append(opt.dumpOptions.StdoutPipeCommands, restoreCmd)

// Run dump
return resticWrapper.Dump(opt.dumpOptions, targetRef)
}
78 changes: 66 additions & 12 deletions pkg/util.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,26 +17,29 @@ limitations under the License.
package pkg

import (
"context"
"time"

stash "stash.appscode.dev/apimachinery/client/clientset/versioned"
"stash.appscode.dev/apimachinery/pkg/restic"

"gomodules.xyz/go-sh"
shell "gomodules.xyz/go-sh"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
"k8s.io/klog/v2"
"kmodules.xyz/custom-resources/apis/appcatalog/v1alpha1"
appcatalog "kmodules.xyz/custom-resources/apis/appcatalog/v1alpha1"
appcatalog_cs "kmodules.xyz/custom-resources/client/clientset/versioned"
)

const (
RedisUser = "username"
RedisPassword = "password"
RedisDumpFile = "dumpfile.resp"
RedisDumpCMD = "redis-dump-go"
RedisRestoreCMD = "redis-cli"
EnvRedisPassword = "REDIS_PWD"
RedisUser = "username"
RedisPassword = "password"
RedisDumpFile = "dumpfile.resp"
RedisDumpCMD = "redis-dump-go"
RedisRestoreCMD = "redis-cli"
EnvRedisCLIAuth = "REDISCLI_AUTH"
EnvRedisDumpGoAuth = "REDISDUMPGO_AUTH"
)

type redisOptions struct {
Expand All @@ -47,7 +50,7 @@ type redisOptions struct {
namespace string
backupSessionName string
appBindingName string
myArgs string
redisArgs string
waitTimeout int32
outputDir string

Expand All @@ -56,22 +59,73 @@ type redisOptions struct {
dumpOptions restic.DumpOptions
}

func waitForDBReady(appBinding *v1alpha1.AppBinding) error {
type Shell interface {
SetEnv(key, value string)
}

type SessionWrapper struct {
*shell.Session
}

func NewSessionWrapper() *SessionWrapper {
return &SessionWrapper{
shell.NewSession(),
}
}
func (wrapper *SessionWrapper) SetEnv(key, value string) {
wrapper.Session.SetEnv(key, value)
}

func (opt *redisOptions) waitForDBReady(appBinding *appcatalog.AppBinding) error {
klog.Infoln("Waiting for the database to be ready.....")
shell := sh.NewSession()
sh := NewSessionWrapper()
args := []interface{}{
"-h", appBinding.Spec.ClientConfig.Service.Name,
"ping",
}

//if port is specified, append port in the arguments
if appBinding.Spec.ClientConfig.Service.Port != 0 {
args = append(args, "-p", appBinding.Spec.ClientConfig.Service.Port)
}

// set access credentials
err := opt.setCredentials(sh, appBinding)
if err != nil {
return err
}

return wait.PollImmediate(time.Second*5, time.Minute*5, func() (bool, error) {
err := shell.Command("redis-cli", args...).Run()
err := sh.Command("redis-cli", args...).Run()
if err != nil {
return false, nil
}
return true, nil
})
}

func (opt *redisOptions) setCredentials(sh Shell, appBinding *appcatalog.AppBinding) error {
// if credential secret is not provided in AppBinding, then nothing to do.
if appBinding.Spec.Secret == nil {
return nil
}

// get the Secret
secret, err := opt.kubeClient.CoreV1().Secrets(opt.namespace).Get(context.TODO(), appBinding.Spec.Secret.Name, metav1.GetOptions{})
if err != nil {
return err
}

// perform necessary transform if secretTransforms section is provided in the AppBinding
err = appBinding.TransformSecret(opt.kubeClient, secret.Data)
if err != nil {
return err
}

// set auth env for redis-cli
sh.SetEnv(EnvRedisCLIAuth, string(secret.Data[RedisPassword]))

// set auth env for redis-dump-go
sh.SetEnv(EnvRedisDumpGoAuth, string(secret.Data[RedisPassword]))
return nil
}

0 comments on commit 7d4bfee

Please sign in to comment.