Merge pull request #277 from databacker/retention

enable retention

README.md

@@ -4,7 +4,7 @@ Back up mysql databases to... anywhere!
 
 ## Overview
 
-mysql-backup is a simple way to do MySQL database backups and restores.
+mysql-backup is a simple way to do MySQL database backups and restores, as well as manage your backups.
 
 It has the following features:
 
@@ -14,6 +14,7 @@ It has the following features:
 * connect to any container running on the same system
 * select how often to run a dump
 * select when to start the first dump, whether time of day or relative to container start time
+* prune backups older than a specific time period or quantity
 
 Please see [CONTRIBUTORS.md](./CONTRIBUTORS.md) for a list of contributors.
 

cmd/common_test.go

@@ -17,12 +17,25 @@ func newMockExecs() *mockExecs {
 	return m
 }
 
-func (m *mockExecs) timerDump(opts core.DumpOptions, timerOpts core.TimerOptions) error {
-	args := m.Called(opts, timerOpts)
+func (m *mockExecs) dump(opts core.DumpOptions) error {
+	args := m.Called(opts)
 	return args.Error(0)
 }
 
 func (m *mockExecs) restore(target storage.Storage, targetFile string, dbconn database.Connection, databasesMap map[string]string, compressor compression.Compressor) error {
 	args := m.Called(target, targetFile, dbconn, databasesMap, compressor)
 	return args.Error(0)
 }
+
+func (m *mockExecs) prune(opts core.PruneOptions) error {
+	args := m.Called(opts)
+	return args.Error(0)
+}
+func (m *mockExecs) timer(timerOpts core.TimerOptions, cmd func() error) error {
+	args := m.Called(timerOpts)
+	err := args.Error(0)
+	if err != nil {
+		return err
+	}
+	return cmd()
+}

cmd/dump.go

@@ -20,7 +20,10 @@ const (
 	defaultMaxAllowedPacket = 4194304
 )
 
-func dumpCmd(execs execs) (*cobra.Command, error) {
+func dumpCmd(execs execs, cmdConfig *cmdConfiguration) (*cobra.Command, error) {
+	if cmdConfig == nil {
+		return nil, fmt.Errorf("cmdConfig is nil")
+	}
 	var v *viper.Viper
 	var cmd = &cobra.Command{
 		Use:     "dump",
@@ -43,18 +46,18 @@ func dumpCmd(execs execs) (*cobra.Command, error) {
 			)
 			if len(targetURLs) > 0 {
 				for _, t := range targetURLs {
-					store, err := storage.ParseURL(t, creds)
+					store, err := storage.ParseURL(t, cmdConfig.creds)
 					if err != nil {
 						return fmt.Errorf("invalid target url: %v", err)
 					}
 					targets = append(targets, store)
 				}
 			} else {
 				// try the config file
-				if configuration != nil {
+				if cmdConfig.configuration != nil {
 					// parse the target objects, then the ones listed for the backup
-					targetStructures := configuration.Targets
-					dumpTargets := configuration.Dump.Targets
+					targetStructures := cmdConfig.configuration.Targets
+					dumpTargets := cmdConfig.configuration.Dump.Targets
 					for _, t := range dumpTargets {
 						var store storage.Storage
 						if target, ok := targetStructures[t]; !ok {
@@ -73,49 +76,49 @@ func dumpCmd(execs execs) (*cobra.Command, error) {
 				return fmt.Errorf("no targets specified")
 			}
 			safechars := v.GetBool("safechars")
-			if !v.IsSet("safechars") && configuration != nil {
-				safechars = configuration.Dump.Safechars
+			if !v.IsSet("safechars") && cmdConfig.configuration != nil {
+				safechars = cmdConfig.configuration.Dump.Safechars
 			}
 			include := v.GetStringSlice("include")
-			if len(include) == 0 && configuration != nil {
-				include = configuration.Dump.Include
+			if len(include) == 0 && cmdConfig.configuration != nil {
+				include = cmdConfig.configuration.Dump.Include
 			}
 			// make this slice nil if it's empty, so it is consistent; used mainly for test consistency
 			if len(include) == 0 {
 				include = nil
 			}
 			exclude := v.GetStringSlice("exclude")
-			if len(exclude) == 0 && configuration != nil {
-				exclude = configuration.Dump.Exclude
+			if len(exclude) == 0 && cmdConfig.configuration != nil {
+				exclude = cmdConfig.configuration.Dump.Exclude
 			}
 			// make this slice nil if it's empty, so it is consistent; used mainly for test consistency
 			if len(exclude) == 0 {
 				exclude = nil
 			}
 			preBackupScripts := v.GetString("pre-backup-scripts")
-			if preBackupScripts == "" && configuration != nil {
-				preBackupScripts = configuration.Dump.Scripts.PreBackup
+			if preBackupScripts == "" && cmdConfig.configuration != nil {
+				preBackupScripts = cmdConfig.configuration.Dump.Scripts.PreBackup
 			}
 			noDatabaseName := v.GetBool("no-database-name")
-			if !v.IsSet("no-database-name") && configuration != nil {
-				noDatabaseName = configuration.Dump.NoDatabaseName
+			if !v.IsSet("no-database-name") && cmdConfig.configuration != nil {
+				noDatabaseName = cmdConfig.configuration.Dump.NoDatabaseName
 			}
 			compact := v.GetBool("compact")
-			if !v.IsSet("compact") && configuration != nil {
-				compact = configuration.Dump.Compact
+			if !v.IsSet("compact") && cmdConfig.configuration != nil {
+				compact = cmdConfig.configuration.Dump.Compact
 			}
 			maxAllowedPacket := v.GetInt("max-allowed-packet")
-			if !v.IsSet("max-allowed-packet") && configuration != nil && configuration.Dump.MaxAllowedPacket != 0 {
-				maxAllowedPacket = configuration.Dump.MaxAllowedPacket
+			if !v.IsSet("max-allowed-packet") && cmdConfig.configuration != nil && cmdConfig.configuration.Dump.MaxAllowedPacket != 0 {
+				maxAllowedPacket = cmdConfig.configuration.Dump.MaxAllowedPacket
 			}
 
 			// compression algorithm: check config, then CLI/env var overrides
 			var (
 				compressionAlgo string
 				compressor      compression.Compressor
 			)
-			if configuration != nil {
-				compressionAlgo = configuration.Dump.Compression
+			if cmdConfig.configuration != nil {
+				compressionAlgo = cmdConfig.configuration.Dump.Compression
 			}
 			compressionVar := v.GetString("compression")
 			if compressionVar != "" {
@@ -131,7 +134,7 @@ func dumpCmd(execs execs) (*cobra.Command, error) {
 				Targets:             targets,
 				Safechars:           safechars,
 				DBNames:             include,
-				DBConn:              dbconn,
+				DBConn:              cmdConfig.dbconn,
 				Compressor:          compressor,
 				Exclude:             exclude,
 				PreBackupScripts:    preBackupScripts,
@@ -141,35 +144,56 @@ func dumpCmd(execs execs) (*cobra.Command, error) {
 				MaxAllowedPacket:    maxAllowedPacket,
 			}
 
+			// retention, if enabled
+			retention := v.GetString("retention")
+			if retention == "" && cmdConfig.configuration != nil {
+				retention = cmdConfig.configuration.Prune.Retention
+			}
+
 			// timer options
 			once := v.GetBool("once")
-			if !v.IsSet("once") && configuration != nil {
-				once = configuration.Dump.Schedule.Once
+			if !v.IsSet("once") && cmdConfig.configuration != nil {
+				once = cmdConfig.configuration.Dump.Schedule.Once
 			}
 			cron := v.GetString("cron")
-			if cron == "" && configuration != nil {
-				cron = configuration.Dump.Schedule.Cron
+			if cron == "" && cmdConfig.configuration != nil {
+				cron = cmdConfig.configuration.Dump.Schedule.Cron
 			}
 			begin := v.GetString("begin")
-			if begin == "" && configuration != nil {
-				begin = configuration.Dump.Schedule.Begin
+			if begin == "" && cmdConfig.configuration != nil {
+				begin = cmdConfig.configuration.Dump.Schedule.Begin
 			}
 			frequency := v.GetInt("frequency")
-			if frequency == 0 && configuration != nil {
-				frequency = configuration.Dump.Schedule.Frequency
+			if frequency == 0 && cmdConfig.configuration != nil {
+				frequency = cmdConfig.configuration.Dump.Schedule.Frequency
 			}
 			timerOpts := core.TimerOptions{
 				Once:      once,
 				Cron:      cron,
 				Begin:     begin,
 				Frequency: frequency,
 			}
-			dump := core.TimerDump
+			dump := core.Dump
+			prune := core.Prune
+			timer := core.TimerCommand
 			if execs != nil {
-				dump = execs.timerDump
+				dump = execs.dump
+				prune = execs.prune
+				timer = execs.timer
 			}
-			if err := dump(dumpOpts, timerOpts); err != nil {
-				return err
+			if err := timer(timerOpts, func() error {
+				err := dump(dumpOpts)
+				if err != nil {
+					return fmt.Errorf("error running dump: %w", err)
+				}
+				if retention != "" {
+					if err := prune(core.PruneOptions{Targets: targets, Retention: retention}); err != nil {
+						return fmt.Errorf("error running prune: %w", err)
+					}
+				}
+				return nil
+			}); err != nil {
+				return fmt.Errorf("error running command: %w", err)
 			}
 			log.Info("Backup complete")
 			return nil
@@ -232,6 +256,8 @@ S3: If it is a URL of the format s3://bucketname/path then it will connect via S
 	cmd.MarkFlagsMutuallyExclusive("once", "frequency")
 	cmd.MarkFlagsMutuallyExclusive("cron", "begin")
 	cmd.MarkFlagsMutuallyExclusive("cron", "frequency")
+	// retention
+	flags.String("retention", "", "Retention period for backups. Optional. If not specified, no pruning will be done. Can be number of backups or time-based. For time-based, the format is: 1d, 1w, 1m, 1y for days, weeks, months, years, respectively. For number-based, the format is: 1c, 2c, 3c, etc. for the count of backups to keep.")
 
 	return cmd, nil
 }

cmd/dump_test.go

@@ -25,70 +25,97 @@ func TestDumpCmd(t *testing.T) {
 		wantErr              bool
 		expectedDumpOptions  core.DumpOptions
 		expectedTimerOptions core.TimerOptions
+		expectedPruneOptions *core.PruneOptions
 	}{
-		{"missing server and target options", []string{""}, "", true, core.DumpOptions{}, core.TimerOptions{}},
-		{"invalid target URL", []string{"--server", "abc", "--target", "def"}, "", true, core.DumpOptions{DBConn: database.Connection{Host: "abc"}}, core.TimerOptions{}},
+		// invalid ones
+		{"missing server and target options", []string{""}, "", true, core.DumpOptions{}, core.TimerOptions{}, nil},
+		{"invalid target URL", []string{"--server", "abc", "--target", "def"}, "", true, core.DumpOptions{DBConn: database.Connection{Host: "abc"}}, core.TimerOptions{}, nil},
+
+		// file URL
 		{"file URL", []string{"--server", "abc", "--target", "file:///foo/bar"}, "", false, core.DumpOptions{
 			Targets:          []storage.Storage{file.New(*fileTargetURL)},
 			MaxAllowedPacket: defaultMaxAllowedPacket,
 			Compressor:       &compression.GzipCompressor{},
 			DBConn:           database.Connection{Host: "abc"},
-		}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}},
+		}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, nil},
+		{"file URL with prune", []string{"--server", "abc", "--target", "file:///foo/bar", "--retention", "1h"}, "", false, core.DumpOptions{
+			Targets:          []storage.Storage{file.New(*fileTargetURL)},
+			MaxAllowedPacket: defaultMaxAllowedPacket,
+			Compressor:       &compression.GzipCompressor{},
+			DBConn:           database.Connection{Host: "abc"},
+		}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, &core.PruneOptions{Targets: []storage.Storage{file.New(*fileTargetURL)}, Retention: "1h"}},
+
+		// config file
+		{"config file", []string{"--config-file", "testdata/config.yml"}, "", false, core.DumpOptions{
+			Targets:          []storage.Storage{file.New(*fileTargetURL)},
+			MaxAllowedPacket: defaultMaxAllowedPacket,
+			Compressor:       &compression.GzipCompressor{},
+			DBConn:           database.Connection{Host: "abcd", Port: 3306, User: "user2", Pass: "xxxx2"},
+		}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, &core.PruneOptions{Targets: []storage.Storage{file.New(*fileTargetURL)}, Retention: "1h"}},
+
+		// timer options
 		{"once flag", []string{"--server", "abc", "--target", "file:///foo/bar", "--once"}, "", false, core.DumpOptions{
 			Targets:          []storage.Storage{file.New(*fileTargetURL)},
 			MaxAllowedPacket: defaultMaxAllowedPacket,
 			Compressor:       &compression.GzipCompressor{},
 			DBConn:           database.Connection{Host: "abc"},
-		}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin, Once: true}},
+		}, core.TimerOptions{Once: true, Frequency: defaultFrequency, Begin: defaultBegin}, nil},
 		{"cron flag", []string{"--server", "abc", "--target", "file:///foo/bar", "--cron", "0 0 * * *"}, "", false, core.DumpOptions{
 			Targets:          []storage.Storage{file.New(*fileTargetURL)},
 			MaxAllowedPacket: defaultMaxAllowedPacket,
 			Compressor:       &compression.GzipCompressor{},
 			DBConn:           database.Connection{Host: "abc"},
-		}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin, Cron: "0 0 * * *"}},
+		}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin, Cron: "0 0 * * *"}, nil},
 		{"begin flag", []string{"--server", "abc", "--target", "file:///foo/bar", "--begin", "1234"}, "", false, core.DumpOptions{
 			Targets:          []storage.Storage{file.New(*fileTargetURL)},
 			MaxAllowedPacket: defaultMaxAllowedPacket,
 			Compressor:       &compression.GzipCompressor{},
 			DBConn:           database.Connection{Host: "abc"},
-		}, core.TimerOptions{Frequency: defaultFrequency, Begin: "1234"}},
+		}, core.TimerOptions{Frequency: defaultFrequency, Begin: "1234"}, nil},
 		{"frequency flag", []string{"--server", "abc", "--target", "file:///foo/bar", "--frequency", "10"}, "", false, core.DumpOptions{
 			Targets:          []storage.Storage{file.New(*fileTargetURL)},
 			MaxAllowedPacket: defaultMaxAllowedPacket,
 			Compressor:       &compression.GzipCompressor{},
 			DBConn:           database.Connection{Host: "abc"},
-		}, core.TimerOptions{Frequency: 10, Begin: defaultBegin}},
-		{"config file", []string{"--config-file", "testdata/config.yml"}, "", false, core.DumpOptions{
-			Targets:          []storage.Storage{file.New(*fileTargetURL)},
-			MaxAllowedPacket: defaultMaxAllowedPacket,
-			Compressor:       &compression.GzipCompressor{},
-			DBConn:           database.Connection{Host: "abc", Port: 3306, User: "user", Pass: "xxxx"},
-		}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}},
-		{"incompatible flags: once/cron", []string{"--server", "abc", "--target", "file:///foo/bar", "--once", "--cron", "0 0 * * *"}, "", true, core.DumpOptions{}, core.TimerOptions{}},
-		{"incompatible flags: once/begin", []string{"--server", "abc", "--target", "file:///foo/bar", "--once", "--begin", "1234"}, "", true, core.DumpOptions{}, core.TimerOptions{}},
-		{"incompatible flags: once/frequency", []string{"--server", "abc", "--target", "file:///foo/bar", "--once", "--frequency", "10"}, "", true, core.DumpOptions{}, core.TimerOptions{}},
-		{"incompatible flags: cron/begin", []string{"--server", "abc", "--target", "file:///foo/bar", "--cron", "0 0 * * *", "--begin", "1234"}, "", true, core.DumpOptions{}, core.TimerOptions{}},
-		{"incompatible flags: cron/frequency", []string{"--server", "abc", "--target", "file:///foo/bar", "--cron", "0 0 * * *", "--frequency", "10"}, "", true, core.DumpOptions{}, core.TimerOptions{}},
+		}, core.TimerOptions{Frequency: 10, Begin: defaultBegin}, nil},
+		{"incompatible flags: once/cron", []string{"--server", "abc", "--target", "file:///foo/bar", "--once", "--cron", "0 0 * * *"}, "", true, core.DumpOptions{}, core.TimerOptions{}, nil},
+		{"incompatible flags: once/begin", []string{"--server", "abc", "--target", "file:///foo/bar", "--once", "--begin", "1234"}, "", true, core.DumpOptions{}, core.TimerOptions{}, nil},
+		{"incompatible flags: once/frequency", []string{"--server", "abc", "--target", "file:///foo/bar", "--once", "--frequency", "10"}, "", true, core.DumpOptions{}, core.TimerOptions{}, nil},
+		{"incompatible flags: cron/begin", []string{"--server", "abc", "--target", "file:///foo/bar", "--cron", "0 0 * * *", "--begin", "1234"}, "", true, core.DumpOptions{}, core.TimerOptions{}, nil},
+		{"incompatible flags: cron/frequency", []string{"--server", "abc", "--target", "file:///foo/bar", "--cron", "0 0 * * *", "--frequency", "10"}, "", true, core.DumpOptions{
+			DBConn: database.Connection{Host: "abcd", Port: 3306, User: "user2", Pass: "xxxx2"},
+		}, core.TimerOptions{Frequency: defaultFrequency, Begin: defaultBegin}, &core.PruneOptions{Targets: []storage.Storage{file.New(*fileTargetURL)}, Retention: "1h"}},
 	}
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			m := newMockExecs()
-			m.On("timerDump", mock.MatchedBy(func(dumpOpts core.DumpOptions) bool {
+			m.On("dump", mock.MatchedBy(func(dumpOpts core.DumpOptions) bool {
 				diff := deep.Equal(dumpOpts, tt.expectedDumpOptions)
 				if diff == nil {
 					return true
 				}
 				t.Errorf("dumpOpts compare failed: %v", diff)
 				return false
-			}), mock.MatchedBy(func(timerOpts core.TimerOptions) bool {
+			})).Return(nil)
+			m.On("timer", mock.MatchedBy(func(timerOpts core.TimerOptions) bool {
 				diff := deep.Equal(timerOpts, tt.expectedTimerOptions)
 				if diff == nil {
 					return true
 				}
 				t.Errorf("timerOpts compare failed: %v", diff)
 				return false
 			})).Return(nil)
+			if tt.expectedPruneOptions != nil {
+				m.On("prune", mock.MatchedBy(func(pruneOpts core.PruneOptions) bool {
+					diff := deep.Equal(pruneOpts, *tt.expectedPruneOptions)
+					if diff == nil {
+						return true
+					}
+					t.Errorf("pruneOpts compare failed: %v", diff)
+					return false
+				})).Return(nil)
+			}
 
 			cmd, err := rootCmd(m)
 			if err != nil {