diff --git a/backup_mysql.go b/backup_mysql.go new file mode 100644 index 000000000..1d5e90ed5 --- /dev/null +++ b/backup_mysql.go @@ -0,0 +1,156 @@ +package cmds + +import ( + "path/filepath" + + "github.com/appscode/go/flags" + "github.com/appscode/stash/pkg/restic" + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + appcatalog_cs "kmodules.xyz/custom-resources/client/clientset/versioned" +) + +const ( + JobMySqlBackup = "stash-mysql-backup" + MySqlUser = "username" + MySqlPassword = "password" + MySqlDumpFile = "dumpfile.sql" + MySqlDumpCMD = "mysqldump" + MySqlRestoreCMD = "mysql" + EnvMySqlPassword = "MYSQL_PWD" +) + +func NewCmdBackupMySql() *cobra.Command { + var ( + masterURL string + kubeconfigPath string + namespace string + appBindingName string + outputDir string + mysqlArgs = "--all-databases" + setupOpt = restic.SetupOptions{ + ScratchDir: restic.DefaultScratchDir, + EnableCache: false, + } + backupOpt = restic.BackupOptions{ + StdinFileName: MySqlDumpFile, + } + metrics = restic.MetricsOptions{ + JobName: JobMySqlBackup, + } + ) + + cmd := &cobra.Command{ + Use: "backup-mysql", + Short: "Takes a backup of MySQL DB", + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + flags.EnsureRequiredFlags(cmd, "app-binding", "provider", "secret-dir") + + // prepare client + config, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfigPath) + if err != nil { + return err + } + kubeClient, err := kubernetes.NewForConfig(config) + if err != nil { + return err + } + appCatalogClient, err := appcatalog_cs.NewForConfig(config) + if err != nil { + return err + } + + // get app binding + appBinding, err := appCatalogClient.AppcatalogV1alpha1().AppBindings(namespace).Get(appBindingName, metav1.GetOptions{}) + if err != nil { + return err + } + // get secret + appBindingSecret, err := kubeClient.CoreV1().Secrets(namespace).Get(appBinding.Spec.Secret.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + // init restic wrapper + resticWrapper, err := restic.NewResticWrapper(setupOpt) + if err != nil { + return err + } + + // set env for mysqldump + resticWrapper.SetEnv(EnvMySqlPassword, string(appBindingSecret.Data[MySqlPassword])) + // setup pipe command + backupOpt.StdinPipeCommand = restic.Command{ + Name: MySqlDumpCMD, + Args: []interface{}{ + "-u", string(appBindingSecret.Data[MySqlUser]), + "-h", appBinding.Spec.ClientConfig.Service.Name, + }, + } + if mysqlArgs != "" { + backupOpt.StdinPipeCommand.Args = append(backupOpt.StdinPipeCommand.Args, mysqlArgs) + } + + // wait for DB ready + waitForDBReady(appBinding.Spec.ClientConfig.Service.Name, appBinding.Spec.ClientConfig.Service.Port) + + // Run backup + backupOutput, backupErr := resticWrapper.RunBackup(backupOpt) + // If metrics are enabled then generate metrics + if metrics.Enabled { + err := backupOutput.HandleMetrics(&metrics, backupErr) + if err != nil { + return errors.NewAggregate([]error{backupErr, err}) + } + } + // If output directory specified, then write the output in "output.json" file in the specified directory + if backupErr == nil && outputDir != "" { + err := backupOutput.WriteOutput(filepath.Join(outputDir, restic.DefaultOutputFileName)) + if err != nil { + return err + } + } + return backupErr + }, + } + + cmd.Flags().StringVar(&mysqlArgs, "mysql-args", mysqlArgs, "Additional arguments") + + cmd.Flags().StringVar(&masterURL, "master", masterURL, "The address of the Kubernetes API server (overrides any value in kubeconfig)") + cmd.Flags().StringVar(&kubeconfigPath, "kubeconfig", kubeconfigPath, "Path to kubeconfig file with authorization information (the master location is set by the master flag).") + cmd.Flags().StringVar(&namespace, "namespace", "default", "Namespace of Backup/Restore Session") + cmd.Flags().StringVar(&appBindingName, "app-binding", appBindingName, "Name of the app binding") + + cmd.Flags().StringVar(&setupOpt.Provider, "provider", setupOpt.Provider, "Backend provider (i.e. gcs, s3, azure etc)") + cmd.Flags().StringVar(&setupOpt.Bucket, "bucket", setupOpt.Bucket, "Name of the cloud bucket/container (keep empty for local backend)") + cmd.Flags().StringVar(&setupOpt.Endpoint, "endpoint", setupOpt.Endpoint, "Endpoint for s3/s3 compatible backend") + cmd.Flags().StringVar(&setupOpt.Path, "path", setupOpt.Path, "Directory inside the bucket where backup will be stored") + cmd.Flags().StringVar(&setupOpt.SecretDir, "secret-dir", setupOpt.SecretDir, "Directory where storage secret has been mounted") + cmd.Flags().StringVar(&setupOpt.ScratchDir, "scratch-dir", setupOpt.ScratchDir, "Temporary directory") + cmd.Flags().BoolVar(&setupOpt.EnableCache, "enable-cache", setupOpt.EnableCache, "Specify weather to enable caching for restic") + + cmd.Flags().StringVar(&backupOpt.Host, "hostname", backupOpt.Host, "Name of the host machine") + + cmd.Flags().IntVar(&backupOpt.RetentionPolicy.KeepLast, "retention-keep-last", backupOpt.RetentionPolicy.KeepLast, "Specify value for retention strategy") + cmd.Flags().IntVar(&backupOpt.RetentionPolicy.KeepHourly, "retention-keep-hourly", backupOpt.RetentionPolicy.KeepHourly, "Specify value for retention strategy") + cmd.Flags().IntVar(&backupOpt.RetentionPolicy.KeepDaily, "retention-keep-daily", backupOpt.RetentionPolicy.KeepDaily, "Specify value for retention strategy") + cmd.Flags().IntVar(&backupOpt.RetentionPolicy.KeepWeekly, "retention-keep-weekly", backupOpt.RetentionPolicy.KeepWeekly, "Specify value for retention strategy") + cmd.Flags().IntVar(&backupOpt.RetentionPolicy.KeepMonthly, "retention-keep-monthly", backupOpt.RetentionPolicy.KeepMonthly, "Specify value for retention strategy") + cmd.Flags().IntVar(&backupOpt.RetentionPolicy.KeepYearly, "retention-keep-yearly", backupOpt.RetentionPolicy.KeepYearly, "Specify value for retention strategy") + cmd.Flags().StringSliceVar(&backupOpt.RetentionPolicy.KeepTags, "retention-keep-tags", backupOpt.RetentionPolicy.KeepTags, "Specify value for retention strategy") + cmd.Flags().BoolVar(&backupOpt.RetentionPolicy.Prune, "retention-prune", backupOpt.RetentionPolicy.Prune, "Specify weather to prune old snapshot data") + cmd.Flags().BoolVar(&backupOpt.RetentionPolicy.DryRun, "retention-dry-run", backupOpt.RetentionPolicy.DryRun, "Specify weather to test retention policy without deleting actual data") + + cmd.Flags().StringVar(&outputDir, "output-dir", outputDir, "Directory where output.json file will be written (keep empty if you don't need to write output in file)") + + cmd.Flags().BoolVar(&metrics.Enabled, "metrics-enabled", metrics.Enabled, "Specify weather to export Prometheus metrics") + cmd.Flags().StringVar(&metrics.PushgatewayURL, "metrics-pushgateway-url", metrics.PushgatewayURL, "Pushgateway URL where the metrics will be pushed") + cmd.Flags().StringVar(&metrics.MetricFileDir, "metrics-dir", metrics.MetricFileDir, "Directory where to write metric.prom file (keep empty if you don't want to write metric in a text file)") + cmd.Flags().StringSliceVar(&metrics.Labels, "metrics-labels", metrics.Labels, "Labels to apply in exported metrics") + + return cmd +} diff --git a/restore_mysql.go b/restore_mysql.go new file mode 100644 index 000000000..5658643e6 --- /dev/null +++ b/restore_mysql.go @@ -0,0 +1,138 @@ +package cmds + +import ( + "path/filepath" + + "github.com/appscode/go/flags" + "github.com/appscode/stash/pkg/restic" + "github.com/spf13/cobra" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + appcatalog_cs "kmodules.xyz/custom-resources/client/clientset/versioned" +) + +func NewCmdRestoreMySql() *cobra.Command { + var ( + masterURL string + kubeconfigPath string + namespace string + appBindingName string + outputDir string + mysqlArgs string + setupOpt = restic.SetupOptions{ + ScratchDir: restic.DefaultScratchDir, + EnableCache: false, + } + dumpOpt = restic.DumpOptions{ + FileName: MySqlDumpFile, + } + metrics = restic.MetricsOptions{ + JobName: JobMySqlBackup, + } + ) + + cmd := &cobra.Command{ + Use: "restore-mysql", + Short: "Restores MySQL DB Backup", + DisableAutoGenTag: true, + RunE: func(cmd *cobra.Command, args []string) error { + flags.EnsureRequiredFlags(cmd, "app-binding", "provider", "secret-dir") + + // prepare client + config, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfigPath) + if err != nil { + return err + } + kubeClient, err := kubernetes.NewForConfig(config) + if err != nil { + return err + } + appCatalogClient, err := appcatalog_cs.NewForConfig(config) + if err != nil { + return err + } + + // get app binding + appBinding, err := appCatalogClient.AppcatalogV1alpha1().AppBindings(namespace).Get(appBindingName, metav1.GetOptions{}) + if err != nil { + return err + } + // get secret + appBindingSecret, err := kubeClient.CoreV1().Secrets(namespace).Get(appBinding.Spec.Secret.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + // init restic wrapper + resticWrapper, err := restic.NewResticWrapper(setupOpt) + if err != nil { + return err + } + + // set env for mysql + resticWrapper.SetEnv(EnvMySqlPassword, string(appBindingSecret.Data[MySqlPassword])) + // setup pipe command + dumpOpt.StdoutPipeCommand = restic.Command{ + Name: MySqlRestoreCMD, + Args: []interface{}{ + "-u", string(appBindingSecret.Data[MySqlUser]), + "-h", appBinding.Spec.ClientConfig.Service.Name, + }, + } + if mysqlArgs != "" { + dumpOpt.StdoutPipeCommand.Args = append(dumpOpt.StdoutPipeCommand.Args, mysqlArgs) + } + + // wait for DB ready + waitForDBReady(appBinding.Spec.ClientConfig.Service.Name, appBinding.Spec.ClientConfig.Service.Port) + + // Run dump + dumpOutput, backupErr := resticWrapper.Dump(dumpOpt) + // If metrics are enabled then generate metrics + if metrics.Enabled { + err := dumpOutput.HandleMetrics(&metrics, backupErr) + if err != nil { + return errors.NewAggregate([]error{backupErr, err}) + } + } + // If output directory specified, then write the output in "output.json" file in the specified directory + if backupErr == nil && outputDir != "" { + err := dumpOutput.WriteOutput(filepath.Join(outputDir, restic.DefaultOutputFileName)) + if err != nil { + return err + } + } + return backupErr + }, + } + + cmd.Flags().StringVar(&mysqlArgs, "mysql-args", mysqlArgs, "Additional arguments") + + cmd.Flags().StringVar(&masterURL, "master", masterURL, "The address of the Kubernetes API server (overrides any value in kubeconfig)") + cmd.Flags().StringVar(&kubeconfigPath, "kubeconfig", kubeconfigPath, "Path to kubeconfig file with authorization information (the master location is set by the master flag).") + cmd.Flags().StringVar(&namespace, "namespace", "default", "Namespace of Backup/Restore Session") + cmd.Flags().StringVar(&appBindingName, "app-binding", appBindingName, "Name of the app binding") + + cmd.Flags().StringVar(&setupOpt.Provider, "provider", setupOpt.Provider, "Backend provider (i.e. gcs, s3, azure etc)") + cmd.Flags().StringVar(&setupOpt.Bucket, "bucket", setupOpt.Bucket, "Name of the cloud bucket/container (keep empty for local backend)") + cmd.Flags().StringVar(&setupOpt.Endpoint, "endpoint", setupOpt.Endpoint, "Endpoint for s3/s3 compatible backend") + cmd.Flags().StringVar(&setupOpt.Path, "path", setupOpt.Path, "Directory inside the bucket where backup will be stored") + cmd.Flags().StringVar(&setupOpt.SecretDir, "secret-dir", setupOpt.SecretDir, "Directory where storage secret has been mounted") + cmd.Flags().StringVar(&setupOpt.ScratchDir, "scratch-dir", setupOpt.ScratchDir, "Temporary directory") + cmd.Flags().BoolVar(&setupOpt.EnableCache, "enable-cache", setupOpt.EnableCache, "Specify weather to enable caching for restic") + + cmd.Flags().StringVar(&dumpOpt.Host, "hostname", dumpOpt.Host, "Name of the host machine") + // TODO: sliceVar + cmd.Flags().StringVar(&dumpOpt.Snapshot, "snapshot", dumpOpt.Snapshot, "Snapshot to dump") + + cmd.Flags().StringVar(&outputDir, "output-dir", outputDir, "Directory where output.json file will be written (keep empty if you don't need to write output in file)") + + cmd.Flags().BoolVar(&metrics.Enabled, "metrics-enabled", metrics.Enabled, "Specify weather to export Prometheus metrics") + cmd.Flags().StringVar(&metrics.PushgatewayURL, "metrics-pushgateway-url", metrics.PushgatewayURL, "Pushgateway URL where the metrics will be pushed") + cmd.Flags().StringVar(&metrics.MetricFileDir, "metrics-dir", metrics.MetricFileDir, "Directory where to write metric.prom file (keep empty if you don't want to write metric in a text file)") + cmd.Flags().StringSliceVar(&metrics.Labels, "metrics-labels", metrics.Labels, "Labels to apply in exported metrics") + + return cmd +} diff --git a/root.go b/root.go index 6b1166552..566ee44cc 100644 --- a/root.go +++ b/root.go @@ -59,6 +59,9 @@ func NewRootCmd() *cobra.Command { rootCmd.AddCommand(NewCmdBackupPG()) rootCmd.AddCommand(NewCmdRestorePG()) + rootCmd.AddCommand(NewCmdBackupMySql()) + rootCmd.AddCommand(NewCmdRestoreMySql()) + rootCmd.AddCommand(NewCmdUpdateStatus()) return rootCmd