feat: Online backup

api/api.go

@@ -97,6 +97,9 @@ type Boost interface {
 	DealsSetConsiderVerifiedStorageDeals(context.Context, bool) error   //perm:admin
 	DealsConsiderUnverifiedStorageDeals(context.Context) (bool, error)  //perm:admin
 	DealsSetConsiderUnverifiedStorageDeals(context.Context, bool) error //perm:admin
+
+	// MethodGroup: Misc
+	OnlineBackup(context.Context, string) error //perm:admin
 }
 
 // DagstoreShardInfo is the serialized form of dagstore.DagstoreShardInfo that

cli/util/api.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"strings"
 
+	"github.com/filecoin-project/boost/node/repo"
 	"github.com/mitchellh/go-homedir"
 	"github.com/urfave/cli/v2"
 
@@ -17,9 +18,7 @@ import (
 
 	"github.com/filecoin-project/boost/api"
 	"github.com/filecoin-project/boost/api/client"
-	"github.com/filecoin-project/boost/node"
 	"github.com/filecoin-project/boostd-data/shared/cliutil"
-	"github.com/filecoin-project/lotus/node/repo"
 	lotus_repo "github.com/filecoin-project/lotus/node/repo"
 )
 
@@ -192,7 +191,7 @@ func GetBoostAPI(ctx *cli.Context, opts ...GetBoostOption) (api.Boost, jsonrpc.C
 		return tn.(api.Boost), func() {}, nil
 	}
 
-	addr, headers, err := GetRawAPI(ctx, node.Boost, "v0")
+	addr, headers, err := GetRawAPI(ctx, repo.Boost, "v0")
 	if err != nil {
 		return nil, nil, err
 	}
@@ -220,7 +219,7 @@ func GetBoostAPI(ctx *cli.Context, opts ...GetBoostOption) (api.Boost, jsonrpc.C
 	return client.NewBoostRPCV0(ctx.Context, addr, headers)
 }
 
-func GetRawAPI(ctx *cli.Context, t repo.RepoType, version string) (string, http.Header, error) {
+func GetRawAPI(ctx *cli.Context, t lotus_repo.RepoType, version string) (string, http.Header, error) {
 	ainfo, err := GetAPIInfo(ctx, t)
 	if err != nil {
 		return "", nil, fmt.Errorf("could not get API info for %s: %w", t.Type(), err)

cmd/boostd/auth.go

@@ -4,12 +4,12 @@ import (
 	"errors"
 	"fmt"
 
+	"github.com/filecoin-project/boost/node/repo"
 	"github.com/urfave/cli/v2"
 
 	bcli "github.com/filecoin-project/boost/cli"
 	boostcliutil "github.com/filecoin-project/boost/cli/util"
 
-	"github.com/filecoin-project/boost/node"
 	"github.com/filecoin-project/go-jsonrpc/auth"
 	"github.com/filecoin-project/lotus/api"
 	cliutil "github.com/filecoin-project/lotus/cli/util"
@@ -113,12 +113,12 @@ var AuthApiInfoToken = &cli.Command{
 			return err
 		}
 
-		ainfo, err := cliutil.GetAPIInfo(cctx, node.Boost)
+		ainfo, err := cliutil.GetAPIInfo(cctx, repo.Boost)
 		if err != nil {
-			return fmt.Errorf("could not get API info for %s: %w", node.Boost, err)
+			return fmt.Errorf("could not get API info for %s: %w", repo.Boost, err)
 		}
 
-		currentEnv, _, _ := boostcliutil.EnvsForAPIInfos(node.Boost)
+		currentEnv, _, _ := boostcliutil.EnvsForAPIInfos(repo.Boost)
 		fmt.Printf("%s=%s:%s\n", currentEnv, string(token), ainfo.Addr)
 		return nil
 	},

cmd/boostd/backup.go

@@ -2,39 +2,63 @@ package main
 
 import (
 	"fmt"
-	"io/ioutil"
 	"os"
 	"path"
 	"path/filepath"
 	"time"
 
+	bcli "github.com/filecoin-project/boost/cli"
+	"github.com/filecoin-project/boost/node/impl/backupmgr"
+	"github.com/filecoin-project/boost/node/repo"
 	"github.com/mitchellh/go-homedir"
 	"github.com/urfave/cli/v2"
 	"gopkg.in/cheggaaa/pb.v1"
 
-	"github.com/filecoin-project/boost/node"
 	"github.com/filecoin-project/lotus/lib/backupds"
 	lotus_repo "github.com/filecoin-project/lotus/node/repo"
 )
 
-const metadaFileName = "metadata"
-
-var fm = []string{"boost.db",
-	"boost.logs.db",
-	"config.toml",
-	"storage.json",
-	"token"}
-
 var backupCmd = &cli.Command{
 	Name:        "backup",
 	Usage:       "boostd backup <backup directory>",
-	Description: "Performs offline backup of Boost",
+	Description: "Takes a backup of Boost",
 	Before:      before,
+	Flags: []cli.Flag{
+		&cli.BoolFlag{
+			Name:  "offline",
+			Usage: "Performs an offline backup of Boost. Boost must be stopped.",
+			Value: false,
+		},
+	},
 	Action: func(cctx *cli.Context) error {
 		if cctx.Args().Len() != 1 {
 			return fmt.Errorf("usage: boostd backup <backup directory>")
 		}
 
+		// Online backup
+		if !cctx.Bool("offline") {
+			api, closer, err := bcli.GetBoostAPI(cctx)
+			if err != nil {
+				return err
+			}
+			defer closer()
+
+			ctx := bcli.ReqContext(cctx)
+
+			bkpPath, err := homedir.Expand(cctx.Args().First())
+			if err != nil {
+				return fmt.Errorf("expanding backup directory path: %w", err)
+			}
+
+			bpath, err := filepath.Abs(bkpPath)
+			if err != nil {
+				return fmt.Errorf("failed get absolute path for backup directory: %w", err)
+			}
+
+			return api.OnlineBackup(ctx, bpath)
+		}
+
+		// Offline backup
 		boostRepoPath := cctx.String(FlagBoostRepo)
 
 		r, err := lotus_repo.NewFS(boostRepoPath)
@@ -49,7 +73,7 @@ var backupCmd = &cli.Command{
 			return fmt.Errorf("repo at '%s' is not initialized", cctx.String(FlagBoostRepo))
 		}
 
-		lr, err := r.LockRO(node.Boost)
+		lr, err := r.LockRO(repo.Boost)
 		if err != nil {
 			return fmt.Errorf("locking repo: %w. Please stop the boostd process to take backup", err)
 		}
@@ -60,11 +84,6 @@ var backupCmd = &cli.Command{
 			return fmt.Errorf("getting metadata datastore: %w", err)
 		}
 
-		bds, err := backupds.Wrap(mds, backupds.NoLogdir)
-		if err != nil {
-			return err
-		}
-
 		bkpPath, err := homedir.Expand(cctx.Args().First())
 		if err != nil {
 			return fmt.Errorf("expanding backup directory path: %w", err)
@@ -88,7 +107,7 @@ var backupCmd = &cli.Command{
 			return err
 		}
 
-		lb, err := b.Lock(node.Boost)
+		lb, err := b.Lock(repo.Boost)
 		if err != nil {
 			return err
 		}
@@ -100,15 +119,15 @@ var backupCmd = &cli.Command{
 			return fmt.Errorf("error creating keystore directory %s: %w", path.Join(bkpDir, "keystore"), err)
 		}
 
-		if err := migrateMarketsKeystore(lr, lb); err != nil {
+		if err := backupmgr.CopyKeysBetweenRepos(lr, lb); err != nil {
 			return fmt.Errorf("error copying keys: %w", err)
 		}
 
-		if err := os.Mkdir(path.Join(bkpDir, "config"), 0755); err != nil {
-			return fmt.Errorf("error creating config directory %s: %w", path.Join(bkpDir, "config"), err)
+		if err := os.Mkdir(path.Join(bkpDir, backupmgr.ConfigDirName), 0755); err != nil {
+			return fmt.Errorf("error creating config directory %s: %w", path.Join(bkpDir, backupmgr.ConfigDirName), err)
 		}
 
-		fpathName := path.Join(bkpDir, metadaFileName)
+		fpathName := path.Join(bkpDir, backupmgr.MetadataFileName)
 
 		fpath, err := homedir.Expand(fpathName)
 		if err != nil {
@@ -117,39 +136,19 @@ var backupCmd = &cli.Command{
 
 		fmt.Println("creating metadata backup")
 
-		out, err := os.OpenFile(fpath, os.O_CREATE|os.O_WRONLY, 0644)
+		err = backupmgr.BackupMetadata(cctx.Context, mds, fpath)
 		if err != nil {
-			return fmt.Errorf("opening backup file %s: %w", fpath, err)
-		}
-
-		defer func() {
-			if err := out.Close(); err != nil {
-				log.Errorw("closing backup file: %w", err)
-			}
-		}()
-
-		if err := bds.Backup(cctx.Context, out); err != nil {
-			return fmt.Errorf("backup error: %w", err)
+			return fmt.Errorf("failed to take metadata backup: %w", err)
 		}
 
-		cfgFiles, err := ioutil.ReadDir(path.Join(lr.Path(), "config"))
+		fl, err := backupmgr.GenerateBkpFileList(bkpDir, true)
 		if err != nil {
-			return fmt.Errorf("failed to read files from config directory: %w", err)
-		}
-
-		for _, cfgFile := range cfgFiles {
-			f := path.Join("config", cfgFile.Name())
-			fm = append(fm, f)
+			return fmt.Errorf("failed to generate list of files to be copied: %w", err)
 		}
 
 		fmt.Println("Copying the files to backup directory")
 
-		destPath, err := homedir.Expand(bkpDir)
-		if err != nil {
-			return fmt.Errorf("expanding destination file path %s: %w", bkpDir, err)
-		}
-
-		if err := copyFiles(lr.Path(), destPath, fm); err != nil {
+		if err := backupmgr.CopyFiles(lr.Path(), bkpDir, fl); err != nil {
 			return fmt.Errorf("error copying file: %w", err)
 		}
 
@@ -181,7 +180,7 @@ var restoreCmd = &cli.Command{
 
 		fmt.Printf("Checking backup directory %s\n", bpath)
 
-		flist := []string{"metadata", "boost.db", "boost.logs.db"}
+		flist := backupmgr.RestoreFileChk
 		for _, fileName := range flist {
 			_, err = os.Stat(path.Join(bpath, fileName))
 			if os.IsNotExist(err) {
@@ -208,11 +207,11 @@ var restoreCmd = &cli.Command{
 		}
 
 		fmt.Println("Creating boost repo")
-		if err := r.Init(node.Boost); err != nil {
+		if err := r.Init(repo.Boost); err != nil {
 			return err
 		}
 
-		lr, err := r.Lock(node.Boost)
+		lr, err := r.Lock(repo.Boost)
 		if err != nil {
 			return err
 		}
@@ -223,15 +222,15 @@ var restoreCmd = &cli.Command{
 			return err
 		}
 
-		lb, err := b.Lock(node.Boost)
+		lb, err := b.Lock(repo.Boost)
 		if err != nil {
 			return err
 		}
 		defer lb.Close()
 
 		fmt.Println("Copying keystore")
 
-		if err := migrateMarketsKeystore(lb, lr); err != nil {
+		if err := backupmgr.CopyKeysBetweenRepos(lb, lr); err != nil {
 			return fmt.Errorf("error copying keys: %w", err)
 		}
 
@@ -240,7 +239,7 @@ var restoreCmd = &cli.Command{
 			return err
 		}
 
-		fpathName := path.Join(bpath, metadaFileName)
+		fpathName := path.Join(bpath, backupmgr.MetadataFileName)
 
 		fpath, err := homedir.Expand(fpathName)
 		if err != nil {
@@ -277,26 +276,21 @@ var restoreCmd = &cli.Command{
 
 		fmt.Println("Restoring files")
 
-		if err := os.Mkdir(path.Join(lr.Path(), "config"), 0755); err != nil {
-			return fmt.Errorf("error creating config directory %s: %w", path.Join(lr.Path(), "config"), err)
+		if err := os.Mkdir(path.Join(lr.Path(), backupmgr.ConfigDirName), 0755); err != nil {
+			return fmt.Errorf("error creating config directory %s: %w", path.Join(lr.Path(), backupmgr.ConfigDirName), err)
 		}
 
-		cfgFiles, err := ioutil.ReadDir(path.Join(lb.Path(), "config"))
+		fl, err := backupmgr.GenerateBkpFileList(lr.Path(), true)
 		if err != nil {
-			return fmt.Errorf("failed to read files from config directory: %w", err)
-		}
-
-		for _, cfgFile := range cfgFiles {
-			f := path.Join("config", cfgFile.Name())
-			fm = append(fm, f)
+			return fmt.Errorf("failed to generate list of files to be copied: %w", err)
 		}
 
 		//Remove default config.toml created with repo to avoid conflict with symllink
 		if err = os.Remove(path.Join(lr.Path(), "config.toml")); err != nil {
 			return fmt.Errorf("failed to remove the default config file: %w", err)
 		}
 
-		if err := copyFiles(lb.Path(), lr.Path(), fm); err != nil {
+		if err := backupmgr.CopyFiles(lb.Path(), lr.Path(), fl); err != nil {
 			return fmt.Errorf("error copying file: %w", err)
 		}
 
@@ -305,44 +299,3 @@ var restoreCmd = &cli.Command{
 		return nil
 	},
 }
-
-func copyFiles(srcDir, destDir string, flist []string) error {
-
-	for _, fName := range flist {
-
-		f, err := os.Lstat(path.Join(srcDir, fName))
-
-		if os.IsNotExist(err) {
-			fmt.Printf("Not copying %s as file does not exists\n", path.Join(srcDir, fName))
-			return nil
-		}
-
-		if err != nil && !os.IsNotExist(err) {
-			return err
-		}
-
-		// Handle if symlinks
-		if f.Mode()&os.ModeSymlink == os.ModeSymlink {
-			linkDest, err := os.Readlink(path.Join(srcDir, fName))
-			if err != nil {
-				return err
-			}
-			if err = os.Symlink(linkDest, path.Join(destDir, fName)); err != nil {
-				return err
-			}
-		} else {
-
-			input, err := ioutil.ReadFile(path.Join(srcDir, fName))
-			if err != nil {
-				return err
-			}
-
-			err = ioutil.WriteFile(path.Join(destDir, fName), input, f.Mode())
-			if err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}