diff --git a/api/api.go b/api/api.go index 7e6da9565..6172ef613 100644 --- a/api/api.go +++ b/api/api.go @@ -97,6 +97,9 @@ type Boost interface { DealsSetConsiderVerifiedStorageDeals(context.Context, bool) error //perm:admin DealsConsiderUnverifiedStorageDeals(context.Context) (bool, error) //perm:admin DealsSetConsiderUnverifiedStorageDeals(context.Context, bool) error //perm:admin + + // MethodGroup: Misc + OnlineBackup(context.Context, string) error //perm:admin } // DagstoreShardInfo is the serialized form of dagstore.DagstoreShardInfo that diff --git a/api/proxy_gen.go b/api/proxy_gen.go index 79c161d0e..2f2688cec 100644 --- a/api/proxy_gen.go +++ b/api/proxy_gen.go @@ -124,6 +124,8 @@ type BoostStruct struct { MarketSetRetrievalAsk func(p0 context.Context, p1 *retrievalmarket.Ask) error `perm:"admin"` + OnlineBackup func(p0 context.Context, p1 string) error `perm:"admin"` + PiecesGetCIDInfo func(p0 context.Context, p1 cid.Cid) (*piecestore.CIDInfo, error) `perm:"read"` PiecesGetMaxOffset func(p0 context.Context, p1 cid.Cid) (uint64, error) `perm:"read"` @@ -747,6 +749,17 @@ func (s *BoostStub) MarketSetRetrievalAsk(p0 context.Context, p1 *retrievalmarke return ErrNotSupported } +func (s *BoostStruct) OnlineBackup(p0 context.Context, p1 string) error { + if s.Internal.OnlineBackup == nil { + return ErrNotSupported + } + return s.Internal.OnlineBackup(p0, p1) +} + +func (s *BoostStub) OnlineBackup(p0 context.Context, p1 string) error { + return ErrNotSupported +} + func (s *BoostStruct) PiecesGetCIDInfo(p0 context.Context, p1 cid.Cid) (*piecestore.CIDInfo, error) { if s.Internal.PiecesGetCIDInfo == nil { return nil, ErrNotSupported diff --git a/build/openrpc/boost.json.gz b/build/openrpc/boost.json.gz index 897015bca..3687702ae 100644 Binary files a/build/openrpc/boost.json.gz and b/build/openrpc/boost.json.gz differ diff --git a/cli/util/api.go b/cli/util/api.go index c857d5c1a..b7860533f 100644 --- a/cli/util/api.go +++ b/cli/util/api.go @@ -8,6 +8,7 @@ import ( "os" "strings" + "github.com/filecoin-project/boost/node/repo" "github.com/mitchellh/go-homedir" "github.com/urfave/cli/v2" @@ -17,9 +18,7 @@ import ( "github.com/filecoin-project/boost/api" "github.com/filecoin-project/boost/api/client" - "github.com/filecoin-project/boost/node" "github.com/filecoin-project/boostd-data/shared/cliutil" - "github.com/filecoin-project/lotus/node/repo" lotus_repo "github.com/filecoin-project/lotus/node/repo" ) @@ -192,7 +191,7 @@ func GetBoostAPI(ctx *cli.Context, opts ...GetBoostOption) (api.Boost, jsonrpc.C return tn.(api.Boost), func() {}, nil } - addr, headers, err := GetRawAPI(ctx, node.Boost, "v0") + addr, headers, err := GetRawAPI(ctx, repo.Boost, "v0") if err != nil { return nil, nil, err } @@ -220,7 +219,7 @@ func GetBoostAPI(ctx *cli.Context, opts ...GetBoostOption) (api.Boost, jsonrpc.C return client.NewBoostRPCV0(ctx.Context, addr, headers) } -func GetRawAPI(ctx *cli.Context, t repo.RepoType, version string) (string, http.Header, error) { +func GetRawAPI(ctx *cli.Context, t lotus_repo.RepoType, version string) (string, http.Header, error) { ainfo, err := GetAPIInfo(ctx, t) if err != nil { return "", nil, fmt.Errorf("could not get API info for %s: %w", t.Type(), err) diff --git a/cmd/boostd/auth.go b/cmd/boostd/auth.go index 713072adb..62dc0e99b 100644 --- a/cmd/boostd/auth.go +++ b/cmd/boostd/auth.go @@ -4,12 +4,12 @@ import ( "errors" "fmt" + "github.com/filecoin-project/boost/node/repo" "github.com/urfave/cli/v2" bcli "github.com/filecoin-project/boost/cli" boostcliutil "github.com/filecoin-project/boost/cli/util" - "github.com/filecoin-project/boost/node" "github.com/filecoin-project/go-jsonrpc/auth" "github.com/filecoin-project/lotus/api" cliutil "github.com/filecoin-project/lotus/cli/util" @@ -113,12 +113,12 @@ var AuthApiInfoToken = &cli.Command{ return err } - ainfo, err := cliutil.GetAPIInfo(cctx, node.Boost) + ainfo, err := cliutil.GetAPIInfo(cctx, repo.Boost) if err != nil { - return fmt.Errorf("could not get API info for %s: %w", node.Boost, err) + return fmt.Errorf("could not get API info for %s: %w", repo.Boost, err) } - currentEnv, _, _ := boostcliutil.EnvsForAPIInfos(node.Boost) + currentEnv, _, _ := boostcliutil.EnvsForAPIInfos(repo.Boost) fmt.Printf("%s=%s:%s\n", currentEnv, string(token), ainfo.Addr) return nil }, diff --git a/cmd/boostd/backup.go b/cmd/boostd/backup.go index da1826636..f327dcf4e 100644 --- a/cmd/boostd/backup.go +++ b/cmd/boostd/backup.go @@ -2,39 +2,63 @@ package main import ( "fmt" - "io/ioutil" "os" "path" "path/filepath" "time" + bcli "github.com/filecoin-project/boost/cli" + "github.com/filecoin-project/boost/node/impl/backupmgr" + "github.com/filecoin-project/boost/node/repo" "github.com/mitchellh/go-homedir" "github.com/urfave/cli/v2" "gopkg.in/cheggaaa/pb.v1" - "github.com/filecoin-project/boost/node" "github.com/filecoin-project/lotus/lib/backupds" lotus_repo "github.com/filecoin-project/lotus/node/repo" ) -const metadaFileName = "metadata" - -var fm = []string{"boost.db", - "boost.logs.db", - "config.toml", - "storage.json", - "token"} - var backupCmd = &cli.Command{ Name: "backup", Usage: "boostd backup ", - Description: "Performs offline backup of Boost", + Description: "Takes a backup of Boost", Before: before, + Flags: []cli.Flag{ + &cli.BoolFlag{ + Name: "offline", + Usage: "Performs an offline backup of Boost. Boost must be stopped.", + Value: false, + }, + }, Action: func(cctx *cli.Context) error { if cctx.Args().Len() != 1 { return fmt.Errorf("usage: boostd backup ") } + // Online backup + if !cctx.Bool("offline") { + api, closer, err := bcli.GetBoostAPI(cctx) + if err != nil { + return err + } + defer closer() + + ctx := bcli.ReqContext(cctx) + + bkpPath, err := homedir.Expand(cctx.Args().First()) + if err != nil { + return fmt.Errorf("expanding backup directory path: %w", err) + } + + bpath, err := filepath.Abs(bkpPath) + if err != nil { + return fmt.Errorf("failed get absolute path for backup directory: %w", err) + } + + return api.OnlineBackup(ctx, bpath) + } + + // Offline backup boostRepoPath := cctx.String(FlagBoostRepo) r, err := lotus_repo.NewFS(boostRepoPath) @@ -49,7 +73,7 @@ var backupCmd = &cli.Command{ return fmt.Errorf("repo at '%s' is not initialized", cctx.String(FlagBoostRepo)) } - lr, err := r.LockRO(node.Boost) + lr, err := r.LockRO(repo.Boost) if err != nil { return fmt.Errorf("locking repo: %w. Please stop the boostd process to take backup", err) } @@ -60,11 +84,6 @@ var backupCmd = &cli.Command{ return fmt.Errorf("getting metadata datastore: %w", err) } - bds, err := backupds.Wrap(mds, backupds.NoLogdir) - if err != nil { - return err - } - bkpPath, err := homedir.Expand(cctx.Args().First()) if err != nil { return fmt.Errorf("expanding backup directory path: %w", err) @@ -88,7 +107,7 @@ var backupCmd = &cli.Command{ return err } - lb, err := b.Lock(node.Boost) + lb, err := b.Lock(repo.Boost) if err != nil { return err } @@ -100,15 +119,15 @@ var backupCmd = &cli.Command{ return fmt.Errorf("error creating keystore directory %s: %w", path.Join(bkpDir, "keystore"), err) } - if err := migrateMarketsKeystore(lr, lb); err != nil { + if err := backupmgr.CopyKeysBetweenRepos(lr, lb); err != nil { return fmt.Errorf("error copying keys: %w", err) } - if err := os.Mkdir(path.Join(bkpDir, "config"), 0755); err != nil { - return fmt.Errorf("error creating config directory %s: %w", path.Join(bkpDir, "config"), err) + if err := os.Mkdir(path.Join(bkpDir, backupmgr.ConfigDirName), 0755); err != nil { + return fmt.Errorf("error creating config directory %s: %w", path.Join(bkpDir, backupmgr.ConfigDirName), err) } - fpathName := path.Join(bkpDir, metadaFileName) + fpathName := path.Join(bkpDir, backupmgr.MetadataFileName) fpath, err := homedir.Expand(fpathName) if err != nil { @@ -117,39 +136,19 @@ var backupCmd = &cli.Command{ fmt.Println("creating metadata backup") - out, err := os.OpenFile(fpath, os.O_CREATE|os.O_WRONLY, 0644) + err = backupmgr.BackupMetadata(cctx.Context, mds, fpath) if err != nil { - return fmt.Errorf("opening backup file %s: %w", fpath, err) - } - - defer func() { - if err := out.Close(); err != nil { - log.Errorw("closing backup file: %w", err) - } - }() - - if err := bds.Backup(cctx.Context, out); err != nil { - return fmt.Errorf("backup error: %w", err) + return fmt.Errorf("failed to take metadata backup: %w", err) } - cfgFiles, err := ioutil.ReadDir(path.Join(lr.Path(), "config")) + fl, err := backupmgr.GenerateBkpFileList(lr.Path(), true) if err != nil { - return fmt.Errorf("failed to read files from config directory: %w", err) - } - - for _, cfgFile := range cfgFiles { - f := path.Join("config", cfgFile.Name()) - fm = append(fm, f) + return fmt.Errorf("failed to generate list of files to be copied: %w", err) } fmt.Println("Copying the files to backup directory") - destPath, err := homedir.Expand(bkpDir) - if err != nil { - return fmt.Errorf("expanding destination file path %s: %w", bkpDir, err) - } - - if err := copyFiles(lr.Path(), destPath, fm); err != nil { + if err := backupmgr.CopyFiles(lr.Path(), bkpDir, fl); err != nil { return fmt.Errorf("error copying file: %w", err) } @@ -181,7 +180,7 @@ var restoreCmd = &cli.Command{ fmt.Printf("Checking backup directory %s\n", bpath) - flist := []string{"metadata", "boost.db", "boost.logs.db"} + flist := backupmgr.RestoreFileChk for _, fileName := range flist { _, err = os.Stat(path.Join(bpath, fileName)) if os.IsNotExist(err) { @@ -208,11 +207,11 @@ var restoreCmd = &cli.Command{ } fmt.Println("Creating boost repo") - if err := r.Init(node.Boost); err != nil { + if err := r.Init(repo.Boost); err != nil { return err } - lr, err := r.Lock(node.Boost) + lr, err := r.Lock(repo.Boost) if err != nil { return err } @@ -223,7 +222,7 @@ var restoreCmd = &cli.Command{ return err } - lb, err := b.Lock(node.Boost) + lb, err := b.Lock(repo.Boost) if err != nil { return err } @@ -231,7 +230,7 @@ var restoreCmd = &cli.Command{ fmt.Println("Copying keystore") - if err := migrateMarketsKeystore(lb, lr); err != nil { + if err := backupmgr.CopyKeysBetweenRepos(lb, lr); err != nil { return fmt.Errorf("error copying keys: %w", err) } @@ -240,7 +239,7 @@ var restoreCmd = &cli.Command{ return err } - fpathName := path.Join(bpath, metadaFileName) + fpathName := path.Join(bpath, backupmgr.MetadataFileName) fpath, err := homedir.Expand(fpathName) if err != nil { @@ -277,18 +276,13 @@ var restoreCmd = &cli.Command{ fmt.Println("Restoring files") - if err := os.Mkdir(path.Join(lr.Path(), "config"), 0755); err != nil { - return fmt.Errorf("error creating config directory %s: %w", path.Join(lr.Path(), "config"), err) + if err := os.Mkdir(path.Join(lr.Path(), backupmgr.ConfigDirName), 0755); err != nil { + return fmt.Errorf("error creating config directory %s: %w", path.Join(lr.Path(), backupmgr.ConfigDirName), err) } - cfgFiles, err := ioutil.ReadDir(path.Join(lb.Path(), "config")) + fl, err := backupmgr.GenerateBkpFileList(lb.Path(), true) if err != nil { - return fmt.Errorf("failed to read files from config directory: %w", err) - } - - for _, cfgFile := range cfgFiles { - f := path.Join("config", cfgFile.Name()) - fm = append(fm, f) + return fmt.Errorf("failed to generate list of files to be copied: %w", err) } //Remove default config.toml created with repo to avoid conflict with symllink @@ -296,7 +290,7 @@ var restoreCmd = &cli.Command{ return fmt.Errorf("failed to remove the default config file: %w", err) } - if err := copyFiles(lb.Path(), lr.Path(), fm); err != nil { + if err := backupmgr.CopyFiles(lb.Path(), lr.Path(), fl); err != nil { return fmt.Errorf("error copying file: %w", err) } @@ -305,44 +299,3 @@ var restoreCmd = &cli.Command{ return nil }, } - -func copyFiles(srcDir, destDir string, flist []string) error { - - for _, fName := range flist { - - f, err := os.Lstat(path.Join(srcDir, fName)) - - if os.IsNotExist(err) { - fmt.Printf("Not copying %s as file does not exists\n", path.Join(srcDir, fName)) - return nil - } - - if err != nil && !os.IsNotExist(err) { - return err - } - - // Handle if symlinks - if f.Mode()&os.ModeSymlink == os.ModeSymlink { - linkDest, err := os.Readlink(path.Join(srcDir, fName)) - if err != nil { - return err - } - if err = os.Symlink(linkDest, path.Join(destDir, fName)); err != nil { - return err - } - } else { - - input, err := ioutil.ReadFile(path.Join(srcDir, fName)) - if err != nil { - return err - } - - err = ioutil.WriteFile(path.Join(destDir, fName), input, f.Mode()) - if err != nil { - return err - } - } - } - - return nil -} diff --git a/cmd/boostd/init.go b/cmd/boostd/init.go index 010d6e89b..76ffc7838 100644 --- a/cmd/boostd/init.go +++ b/cmd/boostd/init.go @@ -15,8 +15,9 @@ import ( "github.com/dustin/go-humanize" "github.com/filecoin-project/boost/api" cliutil "github.com/filecoin-project/boost/cli/util" - "github.com/filecoin-project/boost/node" "github.com/filecoin-project/boost/node/config" + "github.com/filecoin-project/boost/node/impl/backupmgr" + "github.com/filecoin-project/boost/node/repo" "github.com/filecoin-project/boost/util" scliutil "github.com/filecoin-project/boostd-data/shared/cliutil" "github.com/filecoin-project/go-address" @@ -77,7 +78,7 @@ var initCmd = &cli.Command{ return err } - lr, err := bp.repo.Lock(node.Boost) + lr, err := bp.repo.Lock(repo.Boost) if err != nil { return err } @@ -201,7 +202,7 @@ func migrate(cctx *cli.Context, fromMonolith bool, mktsRepoPath string) error { return err } - boostRepo, err := bp.repo.Lock(node.Boost) + boostRepo, err := bp.repo.Lock(repo.Boost) if err != nil { return err } @@ -221,7 +222,7 @@ func migrate(cctx *cli.Context, fromMonolith bool, mktsRepoPath string) error { // Migrate keystore fmt.Println("Migrating keystore") - err = migrateMarketsKeystore(mktsRepo, boostRepo) + err = backupmgr.CopyKeysBetweenRepos(mktsRepo, boostRepo) if err != nil { return err } @@ -624,7 +625,7 @@ func initBoost(ctx context.Context, cctx *cli.Context, marketsRepo lotus_repo.Lo } fmt.Println("Creating boost repo") - if err := r.Init(node.Boost); err != nil { + if err := r.Init(repo.Boost); err != nil { return nil, err } @@ -803,36 +804,6 @@ func importPrefix(ctx context.Context, prefix string, mktsDS datastore.Batching, } } -func migrateMarketsKeystore(mktsRepo lotus_repo.LockedRepo, boostRepo lotus_repo.LockedRepo) error { - boostKS, err := boostRepo.KeyStore() - if err != nil { - return err - } - - mktsKS, err := mktsRepo.KeyStore() - if err != nil { - return err - } - - keys, err := mktsKS.List() - if err != nil { - return err - } - - for _, k := range keys { - ki, err := mktsKS.Get(k) - if err != nil { - return err - } - err = boostKS.Put(k, ki) - if err != nil { - return err - } - } - - return nil -} - // checkV1ApiSupport uses v0 api version to signal support for v1 API // trying to query the v1 api on older lotus versions would get a 404, which can happen for any number of other reasons func checkV1ApiSupport(ctx context.Context, cctx *cli.Context) error { diff --git a/cmd/boostd/net.go b/cmd/boostd/net.go index 545da45bd..bb30c99af 100644 --- a/cmd/boostd/net.go +++ b/cmd/boostd/net.go @@ -1,7 +1,7 @@ package main import ( - "github.com/filecoin-project/boost/node" + "github.com/filecoin-project/boost/node/repo" lcli "github.com/filecoin-project/lotus/cli" "github.com/urfave/cli/v2" ) @@ -10,7 +10,7 @@ var netCmd = &cli.Command{ Name: "net", Usage: "Manage P2P Network", Before: func(cctx *cli.Context) error { - cctx.App.Metadata["repoType"] = node.Boost + cctx.App.Metadata["repoType"] = repo.Boost return nil }, Subcommands: lcli.NetCmd.Subcommands, diff --git a/cmd/boostx/utils_cmd.go b/cmd/boostx/utils_cmd.go index 9286d13a8..436d8635a 100644 --- a/cmd/boostx/utils_cmd.go +++ b/cmd/boostx/utils_cmd.go @@ -11,8 +11,8 @@ import ( clinode "github.com/filecoin-project/boost/cli/node" "github.com/filecoin-project/boost/cmd" - "github.com/filecoin-project/boost/node" "github.com/filecoin-project/boost/node/config" + "github.com/filecoin-project/boost/node/repo" "github.com/filecoin-project/boost/testutil" "github.com/filecoin-project/go-commp-utils/writer" "github.com/filecoin-project/go-fil-markets/stores" @@ -329,7 +329,7 @@ var generatecarCmd = &cli.Command{ ctx := lcli.ReqContext(cctx) r := lotus_repo.NewMemory(nil) - lr, err := r.Lock(node.Boost) + lr, err := r.Lock(repo.Boost) if err != nil { return err } diff --git a/db/db.go b/db/db.go index e7a6c4b1e..c6ab13410 100644 --- a/db/db.go +++ b/db/db.go @@ -7,11 +7,16 @@ import ( "errors" "fmt" "io/ioutil" + "path" "testing" + "github.com/mattn/go-sqlite3" "github.com/stretchr/testify/require" ) +const DealsDBName = "boost.db" +const LogsDBName = "boost.logs.db" + var ErrNotFound = errors.New("not found") type Scannable interface { @@ -55,3 +60,97 @@ func CreateTestTmpDB(t *testing.T) *sql.DB { require.NoError(t, err) return d } + +func SqlBackup(srcDB *sql.DB, dstDir, dbFileName string) error { + dbPath := path.Join(dstDir, dbFileName+"?cache=shared") + dstDB, err := SqlDB(dbPath) + if err != nil { + return fmt.Errorf("failed to open source sql db for backup: %w", err) + } + + defer dstDB.Close() + + err = dstDB.Ping() + if err != nil { + return fmt.Errorf("failed to open source sql db for backup: %w", err) + } + + ctx := context.Background() + + destConn, err := dstDB.Conn(ctx) + if err != nil { + return err + } + + srcConn, err := srcDB.Conn(ctx) + if err != nil { + return err + } + + return destConn.Raw(func(destConn interface{}) error { + return srcConn.Raw(func(srcConn interface{}) error { + destSQLiteConn, ok := destConn.(*sqlite3.SQLiteConn) + if !ok { + return fmt.Errorf("can't convert destination connection to SQLiteConn") + } + + srcSQLiteConn, ok := srcConn.(*sqlite3.SQLiteConn) + if !ok { + return fmt.Errorf("can't convert source connection to SQLiteConn") + } + + b, err := destSQLiteConn.Backup("main", srcSQLiteConn, "main") + if err != nil { + return fmt.Errorf("error initializing SQLite backup: %w", err) + } + + // Allow the initial page count and remaining values to be retrieved + isDone, err := b.Step(0) + if err != nil { + return fmt.Errorf("unable to perform an initial 0-page backup step: %w", err) + } + if isDone { + return fmt.Errorf("backup is unexpectedly done") + } + + // Check that the page count and remaining values are reasonable. + initialPageCount := b.PageCount() + if initialPageCount <= 0 { + return fmt.Errorf("unexpected initial page count value: %v", initialPageCount) + } + initialRemaining := b.Remaining() + if initialRemaining <= 0 { + return fmt.Errorf("unexpected initial remaining value: %v", initialRemaining) + } + if initialRemaining != initialPageCount { + return fmt.Errorf("initial remaining value %v differs from the initial page count value %v", initialRemaining, initialPageCount) + } + + // Copy all the pages + isDone, err = b.Step(-1) + if err != nil { + return fmt.Errorf("failed to perform a backup step: %w", err) + } + if !isDone { + return fmt.Errorf("backup is unexpectedly not done") + } + + // Check that the page count and remaining values are reasonable. + finalPageCount := b.PageCount() + if finalPageCount != initialPageCount { + return fmt.Errorf("final page count %v differs from the initial page count %v", initialPageCount, finalPageCount) + } + finalRemaining := b.Remaining() + if finalRemaining != 0 { + return fmt.Errorf("unexpected remaining value: %v", finalRemaining) + } + + err = b.Finish() + if err != nil { + return fmt.Errorf("error finishing backup: %w", err) + } + + return err + }) + }) +} diff --git a/db/deals_test.go b/db/deals_test.go index f67961550..c122c4ea2 100644 --- a/db/deals_test.go +++ b/db/deals_test.go @@ -2,11 +2,12 @@ package db import ( "context" + "io/ioutil" + "path" "testing" "time" "github.com/filecoin-project/boost/db/migrations" - "github.com/filecoin-project/boost/storagemarket/types" "github.com/filecoin-project/boost/storagemarket/types/dealcheckpoints" cborutil "github.com/filecoin-project/go-cbor-util" @@ -270,3 +271,39 @@ func TestWithSearchFilter(t *testing.T) { req.Equal("", where) req.Equal(0, len(whereArgs)) } + +func TestSqlDbBkp(t *testing.T) { + req := require.New(t) + ctx := context.Background() + + sqldb := CreateTestTmpDB(t) + require.NoError(t, CreateAllBoostTables(ctx, sqldb, sqldb)) + require.NoError(t, migrations.Migrate(sqldb)) + + db := NewDealsDB(sqldb) + deals, err := GenerateDeals() + req.NoError(err) + + for _, deal := range deals { + err = db.Insert(ctx, &deal) + req.NoError(err) + } + + f, err := ioutil.TempFile(t.TempDir(), "*.db") + require.NoError(t, err) + require.NoError(t, f.Close()) + + dir := t.TempDir() + err = SqlBackup(sqldb, dir, "test_db.db") + require.NoError(t, err) + + bdb, err := SqlDB(path.Join(dir, "test_db.db")) + require.NoError(t, err) + + bkdb := NewDealsDB(bdb) + + dealList, err := bkdb.List(ctx, "", nil, nil, 0, 0) + req.NoError(err) + req.Len(dealList, len(deals)) + +} diff --git a/documentation/en/api-v1-methods.md b/documentation/en/api-v1-methods.md index bf8f34a00..b752e996a 100644 --- a/documentation/en/api-v1-methods.md +++ b/documentation/en/api-v1-methods.md @@ -80,6 +80,8 @@ * [NetPubsubScores](#netpubsubscores) * [NetSetLimit](#netsetlimit) * [NetStat](#netstat) +* [Online](#online) + * [OnlineBackup](#onlinebackup) * [Pieces](#pieces) * [PiecesGetCIDInfo](#piecesgetcidinfo) * [PiecesGetMaxOffset](#piecesgetmaxoffset) @@ -1705,6 +1707,23 @@ Response: } ``` +## Online + + +### OnlineBackup +There are not yet any comments for this method. + +Perms: admin + +Inputs: +```json +[ + "string value" +] +``` + +Response: `{}` + ## Pieces diff --git a/itests/framework/framework.go b/itests/framework/framework.go index c55d8f3be..63fb91c35 100644 --- a/itests/framework/framework.go +++ b/itests/framework/framework.go @@ -17,6 +17,7 @@ import ( "github.com/filecoin-project/boost/node" "github.com/filecoin-project/boost/node/config" "github.com/filecoin-project/boost/node/modules/dtypes" + "github.com/filecoin-project/boost/node/repo" "github.com/filecoin-project/boost/storagemarket" "github.com/filecoin-project/boost/storagemarket/types" "github.com/filecoin-project/boost/storagemarket/types/dealcheckpoints" @@ -232,7 +233,7 @@ func (f *TestFramework) Start() error { // Create an in-memory repo r := lotus_repo.NewMemory(nil) - lr, err := r.Lock(node.Boost) + lr, err := r.Lock(repo.Boost) if err != nil { return err } diff --git a/node/builder.go b/node/builder.go index a260e54e6..ef5c9df3e 100644 --- a/node/builder.go +++ b/node/builder.go @@ -19,9 +19,11 @@ import ( lotus_storageadapter "github.com/filecoin-project/boost/markets/storageadapter" "github.com/filecoin-project/boost/node/config" "github.com/filecoin-project/boost/node/impl" + "github.com/filecoin-project/boost/node/impl/backupmgr" "github.com/filecoin-project/boost/node/impl/common" "github.com/filecoin-project/boost/node/modules" "github.com/filecoin-project/boost/node/modules/dtypes" + "github.com/filecoin-project/boost/node/repo" "github.com/filecoin-project/boost/protocolproxy" "github.com/filecoin-project/boost/retrievalmarket/lp2pimpl" "github.com/filecoin-project/boost/retrievalmarket/rtvllog" @@ -156,6 +158,7 @@ const ( HandleBoostDealsKey HandleContractDealsKey HandleProposalLogCleanerKey + HandleOnlineBackupMgrKey // daemon ExtractApiKey @@ -601,6 +604,7 @@ func ConfigBoost(cfg *config.Boost) Option { Override(new(lotus_modules.MinerSealingService), lotus_modules.ConnectSealingService(cfg.SealerApiInfo)), Override(new(sealer.StorageAuth), lotus_modules.StorageAuthWithURL(cfg.SectorIndexApiInfo)), + Override(new(*backupmgr.BackupMgr), modules.NewOnlineBackupMgr(cfg)), // Dynamic Lotus configs Override(new(lotus_dtypes.ConsiderOnlineStorageDealsConfigFunc), lotus_modules.NewConsiderOnlineStorageDealsConfigFunc), @@ -651,7 +655,7 @@ func BoostAPI(out *api.Boost) Option { ), func(s *Settings) error { - s.nodeType = Boost + s.nodeType = repo.Boost return nil }, @@ -663,29 +667,3 @@ func BoostAPI(out *api.Boost) Option { }, ) } - -var Boost boost - -type boost struct{} - -func (f boost) Type() string { - return "Boost" -} - -func (f boost) Config() interface{} { - return config.DefaultBoost() -} - -func (boost) SupportsStagingDeals() {} - -func (boost) APIFlags() []string { - return []string{"boost-api-url"} -} - -func (boost) RepoFlags() []string { - return []string{"boost-repo"} -} - -func (boost) APIInfoEnvVars() (primary string, fallbacks []string, deprecated []string) { - return "BOOST_API_INFO", nil, nil -} diff --git a/node/impl/backupmgr/backup.go b/node/impl/backupmgr/backup.go new file mode 100644 index 000000000..f4915f0a5 --- /dev/null +++ b/node/impl/backupmgr/backup.go @@ -0,0 +1,258 @@ +package backupmgr + +import ( + "context" + "database/sql" + "fmt" + "io/ioutil" + "os" + "path" + "sync" + "time" + + boostdb "github.com/filecoin-project/boost/db" + "github.com/filecoin-project/boost/node/repo" + "github.com/filecoin-project/lotus/lib/backupds" + "github.com/filecoin-project/lotus/node/modules/dtypes" + lotus_repo "github.com/filecoin-project/lotus/node/repo" + "github.com/ipfs/go-datastore" + "github.com/mitchellh/go-homedir" +) + +const MetadataFileName = "metadata" +const ConfigDirName = "config" + +var BkpFileList = []string{"config.toml", + "storage.json", + "token"} + +var RestoreFileChk = []string{ + "metadata", + "boost.db", +} + +type BackupMgr struct { + src lotus_repo.LockedRepo + ds dtypes.MetadataDS + dbName string + lck sync.Mutex + db *sql.DB +} + +func NewBackupMgr(src lotus_repo.LockedRepo, ds datastore.Batching, name string, db *sql.DB) *BackupMgr { + return &BackupMgr{ + src: src, + ds: ds, + dbName: name, + db: db, + } +} + +func (b *BackupMgr) Backup(ctx context.Context, dstDir string) error { + _, err := os.Stat(dstDir) + if os.IsNotExist(err) { + return fmt.Errorf("could not open backup location: %w", err) + } + + return b.initBackup(ctx, dstDir) +} + +func (b *BackupMgr) initBackup(ctx context.Context, dstDir string) error { + s := b.lck.TryLock() + if !s { + return fmt.Errorf("unable to take lock for backup, please check if there is already another backup running") + } + defer b.lck.Unlock() + + // Create tmp backup directory and open it as Boost repo + bkpDir, err := ioutil.TempDir("", "boost_backup_") + if err != nil { + return err + } + + defer os.RemoveAll(bkpDir) + + err = b.takeBackup(ctx, bkpDir) + if err != nil { + return err + } + + // Move directory to the backup location + backupDirName := "boost_backup_" + time.Now().Format("20060102150405") + if err := os.Rename(bkpDir, path.Join(dstDir, backupDirName)); err != nil { + // Try to rename in place if move fails. This would preserve the backup directory and allow user to use the backup + if merr := os.Rename(bkpDir, path.Join(os.TempDir(), backupDirName)); merr != nil { + return fmt.Errorf("failed to move backup directory and error renaming backup directory %s: %w", bkpDir, merr) + } + return fmt.Errorf("error moving backup directory %s to %s: %w\nbackup directory saved as %v", bkpDir, dstDir, err, path.Join(os.TempDir(), backupDirName)) + } + + return nil +} + +func (b *BackupMgr) takeBackup(ctx context.Context, bkpDir string) error { + dstRepo, err := lotus_repo.NewFS(bkpDir) + if err != nil { + return err + } + + dst, err := dstRepo.Lock(repo.Boost) + if err != nil { + return err + } + + defer dst.Close() + + // Create keystore at backup location + if err := os.Mkdir(path.Join(dst.Path(), "keystore"), 0700); err != nil { + return fmt.Errorf("error creating keystore directory %s: %w", path.Join(dst.Path(), "keystore"), err) + } + + // Copy keys from Boost to the backup repo + err = CopyKeysBetweenRepos(b.src, dst) + if err != nil { + return err + } + + // Copy metadata from boost repo to backup repo + fpath, err := homedir.Expand(path.Join(dst.Path(), MetadataFileName)) + if err != nil { + return fmt.Errorf("expanding metadata file path: %w", err) + } + + err = BackupMetadata(ctx, b.ds, fpath) + if err != nil { + return err + } + + // Backup the SQL DBs + if err := boostdb.SqlBackup(b.db, bkpDir, b.dbName); err != nil { + return err + } + + // Generate the list of files to be copied from boost repo to the backup repo + fl, err := GenerateBkpFileList(b.src.Path(), false) + if err != nil { + return fmt.Errorf("unable to generate list of files: %w", err) + } + + // Copy files + if err := os.Mkdir(path.Join(bkpDir, ConfigDirName), 0755); err != nil { + return fmt.Errorf("error creating config directory %s: %w", path.Join(bkpDir, ConfigDirName), err) + } + if err := CopyFiles(b.src.Path(), dst.Path(), fl); err != nil { + return fmt.Errorf("error copying file: %w", err) + } + + return nil +} + +func CopyKeysBetweenRepos(srcRepo lotus_repo.LockedRepo, dstRepo lotus_repo.LockedRepo) error { + srcKS, err := srcRepo.KeyStore() + if err != nil { + return err + } + + dstKS, err := dstRepo.KeyStore() + if err != nil { + return err + } + + keys, err := srcKS.List() + if err != nil { + return err + } + + for _, k := range keys { + ki, err := srcKS.Get(k) + if err != nil { + return err + } + err = dstKS.Put(k, ki) + if err != nil { + return err + } + } + return nil +} + +func BackupMetadata(ctx context.Context, srcDS datastore.Batching, fpath string) error { + bds, err := backupds.Wrap(srcDS, backupds.NoLogdir) + if err != nil { + return err + } + + out, err := os.OpenFile(fpath, os.O_CREATE|os.O_WRONLY, 0644) + if err != nil { + return fmt.Errorf("opening backup file %s: %w", fpath, err) + } + + if err := bds.Backup(ctx, out); err != nil { + return fmt.Errorf("backup error: %w", err) + } + + if err := out.Close(); err != nil { + return fmt.Errorf("closing backup file: %w", err) + } + + return nil +} + +func CopyFiles(srcDir, destDir string, flist []string) error { + + for _, fName := range flist { + + f, err := os.Lstat(path.Join(srcDir, fName)) + + if os.IsNotExist(err) { + fmt.Printf("Not copying %s as file does not exists\n", path.Join(srcDir, fName)) + return nil + } + + if err != nil && !os.IsNotExist(err) { + return err + } + + // Handle if symlinks + if f.Mode()&os.ModeSymlink == os.ModeSymlink { + linkDest, err := os.Readlink(path.Join(srcDir, fName)) + if err != nil { + return err + } + if err = os.Symlink(linkDest, path.Join(destDir, fName)); err != nil { + return err + } + } else { + + input, err := ioutil.ReadFile(path.Join(srcDir, fName)) + if err != nil { + return err + } + + err = ioutil.WriteFile(path.Join(destDir, fName), input, f.Mode()) + if err != nil { + return err + } + } + } + + return nil +} + +func GenerateBkpFileList(repoPath string, offline bool) ([]string, error) { + cfgFiles, err := ioutil.ReadDir(path.Join(repoPath, ConfigDirName)) + if err != nil { + return nil, fmt.Errorf("failed to read files from config directory: %w", err) + } + var fl []string + for _, cfgFile := range cfgFiles { + f := path.Join(ConfigDirName, cfgFile.Name()) + fl = append(fl, f) + } + fl = append(fl, BkpFileList...) + if offline { + fl = append(fl, boostdb.DealsDBName) + } + + return fl, nil +} diff --git a/node/impl/boost.go b/node/impl/boost.go index fb4336968..9b1ecd9bd 100644 --- a/node/impl/boost.go +++ b/node/impl/boost.go @@ -8,6 +8,7 @@ import ( "net/http" "sort" + "github.com/filecoin-project/boost/node/impl/backupmgr" "github.com/multiformats/go-multihash" "go.opentelemetry.io/otel/attribute" @@ -86,6 +87,8 @@ type BoostAPI struct { DS lotus_dtypes.MetadataDS + Bkp *backupmgr.BackupMgr + ConsiderOnlineStorageDealsConfigFunc lotus_dtypes.ConsiderOnlineStorageDealsConfigFunc `optional:"true"` SetConsiderOnlineStorageDealsConfigFunc lotus_dtypes.SetConsiderOnlineStorageDealsConfigFunc `optional:"true"` ConsiderOnlineRetrievalDealsConfigFunc lotus_dtypes.ConsiderOnlineRetrievalDealsConfigFunc `optional:"true"` @@ -510,3 +513,7 @@ func (sm *BoostAPI) BlockstoreHas(ctx context.Context, c cid.Cid) (bool, error) func (sm *BoostAPI) BlockstoreGetSize(ctx context.Context, c cid.Cid) (int, error) { return sm.IndexBackedBlockstore.GetSize(ctx, c) } + +func (sm *BoostAPI) OnlineBackup(ctx context.Context, dstDir string) error { + return sm.Bkp.Backup(ctx, dstDir) +} diff --git a/node/modules/storageminer.go b/node/modules/storageminer.go index 0d49e49c2..ff17b4b26 100644 --- a/node/modules/storageminer.go +++ b/node/modules/storageminer.go @@ -17,6 +17,7 @@ import ( "github.com/filecoin-project/boost/markets/sectoraccessor" "github.com/filecoin-project/boost/markets/storageadapter" "github.com/filecoin-project/boost/node/config" + "github.com/filecoin-project/boost/node/impl/backupmgr" "github.com/filecoin-project/boost/node/modules/dtypes" brm "github.com/filecoin-project/boost/retrievalmarket/lib" "github.com/filecoin-project/boost/retrievalmarket/rtvllog" @@ -306,7 +307,7 @@ func StorageNetworkName(ctx helpers.MetricsCtx, a v1api.FullNode) (dtypes.Networ func NewBoostDB(r lotus_repo.LockedRepo) (*sql.DB, error) { // fixes error "database is locked", caused by concurrent access from deal goroutines to a single sqlite3 db connection // see: https://github.com/mattn/go-sqlite3#:~:text=Error%3A%20database%20is%20locked - dbPath := path.Join(r.Path(), "boost.db?cache=shared") + dbPath := path.Join(r.Path(), db.DealsDBName+"?cache=shared") return db.SqlDB(dbPath) } @@ -317,7 +318,7 @@ type LogSqlDB struct { func NewLogsSqlDB(r repo.LockedRepo) (*LogSqlDB, error) { // fixes error "database is locked", caused by concurrent access from deal goroutines to a single sqlite3 db connection // see: https://github.com/mattn/go-sqlite3#:~:text=Error%3A%20database%20is%20locked - dbPath := path.Join(r.Path(), "boost.logs.db?cache=shared") + dbPath := path.Join(r.Path(), db.LogsDBName+"?cache=shared") d, err := db.SqlDB(dbPath) if err != nil { return nil, err @@ -677,6 +678,12 @@ func NewTracing(cfg *config.Boost) func(lc fx.Lifecycle) (*tracing.Tracing, erro } } +func NewOnlineBackupMgr(cfg *config.Boost) func(lc fx.Lifecycle, r lotus_repo.LockedRepo, ds lotus_dtypes.MetadataDS, dealsDB *sql.DB) *backupmgr.BackupMgr { + return func(lc fx.Lifecycle, r lotus_repo.LockedRepo, ds lotus_dtypes.MetadataDS, dealsDB *sql.DB) *backupmgr.BackupMgr { + return backupmgr.NewBackupMgr(r, ds, db.DealsDBName, dealsDB) + } +} + // NewProviderTransferNetwork sets up the libp2p protocol networking for data transfer func NewProviderTransferNetwork(h host.Host) lotus_dtypes.ProviderTransferNetwork { // Leave it up to the client to reconnect diff --git a/node/repo/boostrepo.go b/node/repo/boostrepo.go new file mode 100644 index 000000000..b20c697c4 --- /dev/null +++ b/node/repo/boostrepo.go @@ -0,0 +1,31 @@ +package repo + +import ( + "github.com/filecoin-project/boost/node/config" +) + +var Boost boost + +type boost struct{} + +func (f boost) Type() string { + return "Boost" +} + +func (f boost) Config() interface{} { + return config.DefaultBoost() +} + +func (boost) SupportsStagingDeals() {} + +func (boost) APIFlags() []string { + return []string{"boost-api-url"} +} + +func (boost) RepoFlags() []string { + return []string{"boost-repo"} +} + +func (boost) APIInfoEnvVars() (primary string, fallbacks []string, deprecated []string) { + return "BOOST_API_INFO", nil, nil +}