update

cmd/geth/main.go

@@ -160,8 +160,6 @@ var (
 		utils.MinerNotifyFullFlag,
 		configFileFlag,
 		utils.CatalystFlag,
-		utils.AncientBackUpFlag,
-		utils.GenesisFlag,
 	}
 
 	rpcFlags = []cli.Flag{

cmd/geth/snapshot.go

@@ -18,15 +18,12 @@ package main
 
 import (
 	"bytes"
-	"encoding/json"
 	"errors"
-	"os"
 	"path/filepath"
 	"time"
 
 	"github.com/ethereum/go-ethereum/cmd/utils"
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/rawdb"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/state/pruner"
@@ -93,8 +90,6 @@ the trie clean cache with default directory will be deleted.
 				Flags: []cli.Flag{
 					utils.DataDirFlag,
 					utils.AncientFlag,
-					utils.AncientBackUpFlag,
-					utils.GenesisFlag,
 				},
 				Description: `
 Offline prune for block data.
@@ -171,56 +166,62 @@ It's also usable without snapshot enabled.
 
 func pruneBlock(ctx *cli.Context) error {
 	stack, config := makeConfigNode(ctx)
-	//defer stack.Close()
-	chaindb := utils.MakeChainDatabase(ctx, stack, false)
-	// Make sure we have a valid genesis JSON
-	genesisPath := ctx.GlobalString(utils.GenesisFlag.Name)
-	if len(genesisPath) == 0 {
-		utils.Fatalf("Must supply path to genesis JSON file")
-	}
-	file, err := os.Open(genesisPath)
-	if err != nil {
-		utils.Fatalf("Failed to read genesis file: %v", err)
-	}
-	defer file.Close()
+	defer stack.Close()
 
-	genesis := new(core.Genesis)
-	if err := json.NewDecoder(file).Decode(genesis); err != nil {
-		utils.Fatalf("invalid genesis file: %v", err)
-	}
-	if err != nil {
-		utils.Fatalf("Failed to decode genesis: %v", err)
+	chaindb := utils.MakeChainDatabaseForBlockPrune(ctx, stack, false)
+	chaindb.Close()
+
+	var oldAncientPath, newAncientPath string
+	if path := getAncientPath(ctx); path != "" {
+		oldAncientPath = path + "/ancient"
+		newAncientPath = path + "/ancient_back"
+	} else {
+		utils.Fatalf("Prune failed, did not specify the AncientPath %v")
 	}
-	freezer := config.Eth.DatabaseFreezer
 
+	//lock, _, err := fileutil.Flock(filepath.Join(oldAncientPath, "FLOCK"))
+	// lock, _, err := fileutil.Flock(oldAncientPath)
+	// if err != nil {
+	// 	return err
+	// }
+	// _, err = os.Open(oldAncientPath)
+	// if err != nil {
+	// 	utils.Fatalf("Failed to read genesis file: %v", err)
+	// }
 	for _, name := range []string{"chaindata"} {
-		root := stack.ResolvePath(name) // /Users/user/storage/Private_BSC_Storage/build/bin/node/geth/chaindata
+		root := stack.ResolvePath(name)
 		switch {
-		case freezer == "":
-			freezer = filepath.Join(root, "ancient")
-		case !filepath.IsAbs(freezer):
-			freezer = stack.ResolvePath(freezer)
+		case oldAncientPath == "":
+			oldAncientPath = filepath.Join(root, "ancient")
+		case !filepath.IsAbs(oldAncientPath):
+			oldAncientPath = stack.ResolvePath(oldAncientPath)
 		}
-		pruner, err := pruner.NewBlockPruner(chaindb, stack, stack.ResolvePath(""), freezer, genesis)
+		pruner, err := pruner.NewBlockPruner(chaindb, stack, stack.ResolvePath(""), oldAncientPath)
 		if err != nil {
 			utils.Fatalf("Failed to create block pruner", err)
 		}
-		backfreezer := filepath.Join(root, "ancient_back_up")
-		if err := pruner.BlockPruneBackUp(name, config.Eth.DatabaseCache, utils.MakeDatabaseHandles(), backfreezer, "", false); err != nil {
+
+		if err := pruner.BlockPruneBackUp(name, config.Eth.DatabaseCache, utils.MakeDatabaseHandles(), newAncientPath, oldAncientPath, "", false); err != nil {
 			log.Error("Failed to back up block", "err", err)
 			return err
 		}
 	}
+
 	log.Info("geth block offline pruning backup successfully")
-	oldAncientPath := ctx.GlobalString(utils.AncientFlag.Name)
-	newAncientPath := ctx.GlobalString(utils.AncientBackUpFlag.Name)
+
 	if err := pruner.BlockPrune(oldAncientPath, newAncientPath); err != nil {
 		utils.Fatalf("Failed to prune block", err)
 		return err
 	}
+	//lock.Release()
+	log.Info("Block prune successfully")
 	return nil
 }
 
+func getAncientPath(ctx *cli.Context) string {
+	return ctx.GlobalString(utils.AncientFlag.Name)
+}
+
 func pruneState(ctx *cli.Context) error {
 	stack, config := makeConfigNode(ctx)
 	defer stack.Close()

cmd/utils/flags.go

@@ -134,14 +134,6 @@ var (
 		Name:  "datadir.ancient",
 		Usage: "Data directory for ancient chain segments (default = inside chaindata)",
 	}
-	AncientBackUpFlag = DirectoryFlag{
-		Name:  "datadir.backup",
-		Usage: "Data directory for ancient directory backup",
-	}
-	GenesisFlag = DirectoryFlag{
-		Name:  "datadir.genesis",
-		Usage: "Data directory for genesis file",
-	}
 	DiffFlag = DirectoryFlag{
 		Name:  "datadir.diff",
 		Usage: "Data directory for difflayer segments (default = inside chaindata)",
@@ -1912,6 +1904,28 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly bool) ethdb.
 	return chainDb
 }
 
+// MakeChainDatabase open an LevelDB using the flags passed to the client and will hard crash if it fails.
+func MakeChainDatabaseForBlockPrune(ctx *cli.Context, stack *node.Node, readonly bool) ethdb.Database {
+	var (
+		cache   = ctx.GlobalInt(CacheFlag.Name) * ctx.GlobalInt(CacheDatabaseFlag.Name) / 100
+		handles = MakeDatabaseHandles()
+
+		err     error
+		chainDb ethdb.Database
+	)
+	if ctx.GlobalString(SyncModeFlag.Name) == "light" {
+		name := "lightchaindata"
+		chainDb, err = stack.OpenDatabase(name, cache, handles, "", readonly)
+	} else {
+		name := "chaindata"
+		chainDb, err = stack.OpenDatabaseWithFreezerForPruneBlock(name, cache, handles, ctx.GlobalString(AncientFlag.Name), "", readonly)
+	}
+	if err != nil {
+		Fatalf("Could not open database: %v", err)
+	}
+	return chainDb
+}
+
 func MakeGenesis(ctx *cli.Context) *core.Genesis {
 	var genesis *core.Genesis
 	switch {

core/rawdb/accessors_chain.go

@@ -294,6 +294,20 @@ func ReadHeaderRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValu
 	return nil // Can't find the data anywhere.
 }
 
+func ReadOffSetOfAncientFreezer(db ethdb.KeyValueStore) uint64 {
+	offset, _ := db.Get(offSetOfAncientFreezer)
+	if offset == nil {
+		return 0
+	}
+	return new(big.Int).SetBytes(offset).Uint64()
+}
+
+func WriteOffSetOfAncientFreezer(db ethdb.KeyValueStore, offset uint64) {
+	if err := db.Put(offSetOfAncientFreezer, new(big.Int).SetUint64(offset).Bytes()); err != nil {
+		log.Crit("Failed to store offSetOfAncientFreezer", "err", err)
+	}
+}
+
 // HasHeader verifies the existence of a block header corresponding to the hash.
 func HasHeader(db ethdb.Reader, hash common.Hash, number uint64) bool {
 	if has, err := db.Ancient(freezerHashTable, number); err == nil && common.BytesToHash(has) == hash {
@@ -724,7 +738,6 @@ func WriteAncientBlock(db ethdb.AncientWriter, block *types.Block, receipts type
 	return len(headerBlob) + len(bodyBlob) + len(receiptBlob) + len(tdBlob) + common.HashLength
 }
 
-
 // DeleteBlock removes all block data associated with a hash.
 func DeleteBlock(db ethdb.KeyValueWriter, hash common.Hash, number uint64) {
 	DeleteReceipts(db, hash, number)

core/rawdb/database.go

@@ -122,11 +122,6 @@ func (db *nofreezedb) AppendAncient(number uint64, hash, header, body, receipts,
 	return errNotSupported
 }
 
-// AppendAncientNoBody returns an error as we don't have a backing chain freezer.
-func (db *nofreezedb) AppendAncientNoBody(number uint64, hash, header, receipts, td []byte) error {
-	return errNotSupported
-}
-
 // TruncateAncients returns an error as we don't have a backing chain freezer.
 func (db *nofreezedb) TruncateAncients(items uint64) error {
 	return errNotSupported
@@ -162,6 +157,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st
 	if err != nil {
 		return nil, err
 	}
+	offset := ReadOffSetOfAncientFreezer(db)
 	// Since the freezer can be stored separately from the user's key-value database,
 	// there's a fairly high probability that the user requests invalid combinations
 	// of the freezer and database. Ensure that we don't shoot ourselves in the foot
@@ -184,46 +180,48 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st
 	// If the genesis hash is empty, we have a new key-value store, so nothing to
 	// validate in this method. If, however, the genesis hash is not nil, compare
 	// it to the freezer content.
-	if kvgenesis, _ := db.Get(headerHashKey(0)); len(kvgenesis) > 0 {
-		if frozen, _ := frdb.Ancients(); frozen > 0 {
-			// If the freezer already contains something, ensure that the genesis blocks
-			// match, otherwise we might mix up freezers across chains and destroy both
-			// the freezer and the key-value store.
-			frgenesis, err := frdb.Ancient(freezerHashTable, 0)
-			if err != nil {
-				return nil, fmt.Errorf("failed to retrieve genesis from ancient %v", err)
-			} else if !bytes.Equal(kvgenesis, frgenesis) {
-				return nil, fmt.Errorf("genesis mismatch: %#x (leveldb) != %#x (ancients)", kvgenesis, frgenesis)
-			}
-			// Key-value store and freezer belong to the same network. Ensure that they
-			// are contiguous, otherwise we might end up with a non-functional freezer.
-			if kvhash, _ := db.Get(headerHashKey(frozen)); len(kvhash) == 0 {
-				// Subsequent header after the freezer limit is missing from the database.
-				// Reject startup is the database has a more recent head.
-				if *ReadHeaderNumber(db, ReadHeadHeaderHash(db)) > frozen-1 {
-					return nil, fmt.Errorf("gap (#%d) in the chain between ancients and leveldb", frozen)
+	if offset == 0 {
+		if kvgenesis, _ := db.Get(headerHashKey(0)); len(kvgenesis) > 0 {
+			if frozen, _ := frdb.Ancients(); frozen > 0 {
+				// If the freezer already contains something, ensure that the genesis blocks
+				// match, otherwise we might mix up freezers across chains and destroy both
+				// the freezer and the key-value store.
+				frgenesis, err := frdb.Ancient(freezerHashTable, 0)
+				if err != nil {
+					return nil, fmt.Errorf("failed to retrieve genesis from ancient %v", err)
+				} else if !bytes.Equal(kvgenesis, frgenesis) {
+					return nil, fmt.Errorf("genesis mismatch: %#x (leveldb) != %#x (ancients)", kvgenesis, frgenesis)
 				}
-				// Database contains only older data than the freezer, this happens if the
-				// state was wiped and reinited from an existing freezer.
-			}
-			// Otherwise, key-value store continues where the freezer left off, all is fine.
-			// We might have duplicate blocks (crash after freezer write but before key-value
-			// store deletion, but that's fine).
-		} else {
-			// If the freezer is empty, ensure nothing was moved yet from the key-value
-			// store, otherwise we'll end up missing data. We check block #1 to decide
-			// if we froze anything previously or not, but do take care of databases with
-			// only the genesis block.
-			if ReadHeadHeaderHash(db) != common.BytesToHash(kvgenesis) {
-				// Key-value store contains more data than the genesis block, make sure we
-				// didn't freeze anything yet.
-				if kvblob, _ := db.Get(headerHashKey(1)); len(kvblob) == 0 {
-					return nil, errors.New("ancient chain segments already extracted, please set --datadir.ancient to the correct path")
+				// Key-value store and freezer belong to the same network. Ensure that they
+				// are contiguous, otherwise we might end up with a non-functional freezer.
+				if kvhash, _ := db.Get(headerHashKey(frozen)); len(kvhash) == 0 {
+					// Subsequent header after the freezer limit is missing from the database.
+					// Reject startup is the database has a more recent head.
+					if *ReadHeaderNumber(db, ReadHeadHeaderHash(db)) > frozen-1 {
+						return nil, fmt.Errorf("gap (#%d) in the chain between ancients and leveldb", frozen)
+					}
+					// Database contains only older data than the freezer, this happens if the
+					// state was wiped and reinited from an existing freezer.
+				}
+				// Otherwise, key-value store continues where the freezer left off, all is fine.
+				// We might have duplicate blocks (crash after freezer write but before key-value
+				// store deletion, but that's fine).
+			} else {
+				// If the freezer is empty, ensure nothing was moved yet from the key-value
+				// store, otherwise we'll end up missing data. We check block #1 to decide
+				// if we froze anything previously or not, but do take care of databases with
+				// only the genesis block.
+				if ReadHeadHeaderHash(db) != common.BytesToHash(kvgenesis) {
+					// Key-value store contains more data than the genesis block, make sure we
+					// didn't freeze anything yet.
+					if kvblob, _ := db.Get(headerHashKey(1)); len(kvblob) == 0 {
+						return nil, errors.New("ancient chain segments already extracted, please set --datadir.ancient to the correct path")
+					}
+					// Block #1 is still in the database, we're allowed to init a new feezer
 				}
-				// Block #1 is still in the database, we're allowed to init a new feezer
+				// Otherwise, the head header is still the genesis, we're allowed to init a new
+				// feezer.
 			}
-			// Otherwise, the head header is still the genesis, we're allowed to init a new
-			// feezer.
 		}
 	}
 	// Freezer is consistent with the key-value database, permit combining the two
@@ -236,6 +234,42 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st
 	}, nil
 }
 
+// NewDatabaseWithFreezer creates a high level database on top of a given key-
+// value data store with a freezer moving immutable chain segments into cold
+// storage.
+//Without goroutine of freeze running
+func NewDatabaseWithFreezerForPruneBlock(db ethdb.KeyValueStore, freezer string, namespace string, readonly bool) (ethdb.Database, error) {
+	// Create the idle freezer instance
+	frdb, err := newFreezer(freezer, namespace, readonly)
+	if err != nil {
+		return nil, err
+	}
+
+	return &freezerdb{
+		KeyValueStore: db,
+		AncientStore:  frdb,
+	}, nil
+}
+
+// NewDatabaseWithFreezer creates a high level database on top of a given key-
+// value data store with a freezer moving immutable chain segments into cold
+// storage.
+func NewDatabaseWithFreezerBackup(offset uint64, db ethdb.KeyValueStore, freezer string, namespace string, readonly bool) (ethdb.Database, error) {
+	// Create the idle freezer instance
+	frdb, err := newFreezer(freezer, namespace, readonly)
+	if err != nil {
+		return nil, err
+	}
+
+	//Assign the new offset to the new backup freezer while creating freezer
+	frdb.offset = offset
+
+	return &freezerdb{
+		KeyValueStore: db,
+		AncientStore:  frdb,
+	}, nil
+}
+
 // NewMemoryDatabase creates an ephemeral in-memory key-value database without a
 // freezer moving immutable chain segments into cold storage.
 func NewMemoryDatabase() ethdb.Database {
@@ -274,6 +308,37 @@ func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, freezer
 	return frdb, nil
 }
 
+// NewLevelDBDatabaseWithFreezer creates a persistent key-value database with a
+// freezer moving immutable chain segments into cold storage.
+func NewLevelDBDatabaseWithFreezerForPruneBlock(file string, cache int, handles int, freezer string, namespace string, readonly bool) (ethdb.Database, error) {
+	kvdb, err := leveldb.New(file, cache, handles, namespace, readonly)
+	if err != nil {
+		return nil, err
+	}
+	frdb, err := NewDatabaseWithFreezerForPruneBlock(kvdb, freezer, namespace, readonly)
+	if err != nil {
+		kvdb.Close()
+		return nil, err
+	}
+	return frdb, nil
+}
+
+// NewLevelDBDatabaseWithFreezer creates a persistent key-value database with a
+// freezer moving immutable chain segments into cold storage.
+func NewLevelDBDatabaseWithFreezerBackup(offset uint64, file string, cache int, handles int, freezer string, namespace string, readonly bool) (ethdb.Database, error) {
+	kvdb, err := leveldb.New(file, cache, handles, namespace, readonly)
+	if err != nil {
+		return nil, err
+	}
+	db, err := NewDatabaseWithFreezerBackup(offset, kvdb, freezer, namespace, readonly)
+	if err != nil {
+		kvdb.Close()
+		return nil, err
+	}
+
+	return db, nil
+}
+
 type counter uint64
 
 func (c counter) String() string {