From eea88af0b5fcc2693760d4ce5a6faaef430de26d Mon Sep 17 00:00:00 2001 From: fearlessfe <505380967@qq.com> Date: Thu, 5 Dec 2024 22:23:17 +0800 Subject: [PATCH] feat: use pebble db and make storage config unified --- beacon/storage.go | 6 +++--- beacon/storage_test.go | 3 +-- cmd/shisui/main.go | 19 ++++++++----------- state/storage.go | 16 ++++++++-------- storage/config.go | 3 --- storage/pebble/storage.go | 15 ++++----------- storage/pebble/storage_test.go | 11 +++++------ storage/sqlite/storage.go | 6 +++--- storage/sqlite/storage_test.go | 5 ++--- storage/tests/storage_bench_test.go | 12 +++++------- 10 files changed, 39 insertions(+), 57 deletions(-) diff --git a/beacon/storage.go b/beacon/storage.go index 0c61441..13205d8 100644 --- a/beacon/storage.go +++ b/beacon/storage.go @@ -34,10 +34,10 @@ type beaconStorageCache struct { var _ storage2.ContentStorage = &Storage{} -func NewBeaconStorage(config storage2.PortalStorageConfig) (storage2.ContentStorage, error) { +func NewBeaconStorage(config storage2.PortalStorageConfig, db *sql.DB) (storage2.ContentStorage, error) { bs := &Storage{ storageCapacityInBytes: config.StorageCapacityMB * BytesInMB, - db: config.DB, + db: db, log: log.New("beacon_storage"), spec: config.Spec, cache: &beaconStorageCache{}, @@ -47,7 +47,7 @@ func NewBeaconStorage(config storage2.PortalStorageConfig) (storage2.ContentStor } var err error - portalStorageMetrics, err = portalwire.NewPortalStorageMetrics(config.NetworkName, config.DB) + portalStorageMetrics, err = portalwire.NewPortalStorageMetrics(config.NetworkName, db) if err != nil { return nil, err } diff --git a/beacon/storage_test.go b/beacon/storage_test.go index eefd123..dcf4174 100644 --- a/beacon/storage_test.go +++ b/beacon/storage_test.go @@ -64,11 +64,10 @@ func genStorage(testDir string) (storage.ContentStorage, error) { } config := &storage.PortalStorageConfig{ StorageCapacityMB: 1000, - DB: db, NodeId: enode.ID(zeroNodeId), Spec: configs.Mainnet, } - return NewBeaconStorage(*config) + return NewBeaconStorage(*config, db) } type entry struct { diff --git a/cmd/shisui/main.go b/cmd/shisui/main.go index 90bc010..4f6ac69 100644 --- a/cmd/shisui/main.go +++ b/cmd/shisui/main.go @@ -39,7 +39,7 @@ import ( "github.com/optimism-java/shisui2/portalwire" "github.com/optimism-java/shisui2/state" "github.com/optimism-java/shisui2/storage" - "github.com/optimism-java/shisui2/storage/sqlite" + "github.com/optimism-java/shisui2/storage/pebble" "github.com/optimism-java/shisui2/web3" "github.com/protolambda/zrnt/eth2/configs" "github.com/urfave/cli/v2" @@ -379,16 +379,15 @@ func doPortMapping(natm nat.Interface, ln *enode.LocalNode, addr *net.UDPAddr) { func initHistory(config Config, server *rpc.Server, conn discover.UDPConn, localNode *enode.LocalNode, discV5 *discover.UDPv5, utp *portalwire.PortalUtp) (*history.Network, error) { networkName := portalwire.History.Name() - db, err := sqlite.NewDB(config.DataDir, networkName) + db, err := pebble.NewDB(config.DataDir, 16, 400, networkName) if err != nil { return nil, err } - contentStorage, err := sqlite.NewHistoryStorage(storage.PortalStorageConfig{ + contentStorage, err := pebble.NewStorage(storage.PortalStorageConfig{ StorageCapacityMB: config.DataCapacity, - DB: db, NodeId: localNode.ID(), NetworkName: networkName, - }) + }, db) if err != nil { return nil, err } @@ -435,11 +434,10 @@ func initBeacon(config Config, server *rpc.Server, conn discover.UDPConn, localN contentStorage, err := beacon.NewBeaconStorage(storage.PortalStorageConfig{ StorageCapacityMB: config.DataCapacity, - DB: sqlDb, NodeId: localNode.ID(), Spec: configs.Mainnet, NetworkName: portalwire.Beacon.Name(), - }) + }, sqlDb) if err != nil { return nil, err } @@ -473,16 +471,15 @@ func initBeacon(config Config, server *rpc.Server, conn discover.UDPConn, localN func initState(config Config, server *rpc.Server, conn discover.UDPConn, localNode *enode.LocalNode, discV5 *discover.UDPv5, utp *portalwire.PortalUtp) (*state.Network, error) { networkName := portalwire.State.Name() - db, err := sqlite.NewDB(config.DataDir, networkName) + db, err := pebble.NewDB(config.DataDir, 16, 400, networkName) if err != nil { return nil, err } - contentStorage, err := sqlite.NewHistoryStorage(storage.PortalStorageConfig{ + contentStorage, err := pebble.NewStorage(storage.PortalStorageConfig{ StorageCapacityMB: config.DataCapacity, - DB: db, NodeId: localNode.ID(), NetworkName: networkName, - }) + }, db) if err != nil { return nil, err } diff --git a/state/storage.go b/state/storage.go index 3585bdf..77399a4 100644 --- a/state/storage.go +++ b/state/storage.go @@ -3,9 +3,9 @@ package state import ( "bytes" "crypto/sha256" - "database/sql" "errors" + "github.com/cockroachdb/pebble" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" @@ -24,24 +24,24 @@ var _ storage.ContentStorage = &Storage{} type Storage struct { store storage.ContentStorage - db *sql.DB + db *pebble.DB log log.Logger } var portalStorageMetrics *portalwire.PortalStorageMetrics -func NewStateStorage(store storage.ContentStorage, db *sql.DB) *Storage { +func NewStateStorage(store storage.ContentStorage, db *pebble.DB) *Storage { stateStorage := &Storage{ store: store, db: db, log: log.New("storage", "state"), } - var err error - portalStorageMetrics, err = portalwire.NewPortalStorageMetrics("state", db) - if err != nil { - return nil - } + // var err error + // portalStorageMetrics, err = portalwire.NewPortalStorageMetrics("state", db) + // if err != nil { + // return nil + // } return stateStorage } diff --git a/storage/config.go b/storage/config.go index 13a5e53..59ef27d 100644 --- a/storage/config.go +++ b/storage/config.go @@ -1,15 +1,12 @@ package storage import ( - "database/sql" - "github.com/ethereum/go-ethereum/p2p/enode" "github.com/protolambda/zrnt/eth2/beacon/common" ) type PortalStorageConfig struct { StorageCapacityMB uint64 - DB *sql.DB NodeId enode.ID Spec *common.Spec NetworkName string diff --git a/storage/pebble/storage.go b/storage/pebble/storage.go index 177b6ab..3dca136 100644 --- a/storage/pebble/storage.go +++ b/storage/pebble/storage.go @@ -1,4 +1,4 @@ -package pepple +package pebble import ( "bytes" @@ -31,14 +31,7 @@ const ( var _ storage.ContentStorage = &ContentStorage{} -type PeppleStorageConfig struct { - StorageCapacityMB uint64 - DB *pebble.DB - NodeId enode.ID - NetworkName string -} - -func NewPeppleDB(dataDir string, cache, handles int, namespace string) (*pebble.DB, error) { +func NewDB(dataDir string, cache, handles int, namespace string) (*pebble.DB, error) { // Ensure we have some minimal caching and file guarantees if cache < minCache { cache = minCache @@ -128,10 +121,10 @@ type ContentStorage struct { bytePool sync.Pool } -func NewPeppleStorage(config PeppleStorageConfig) (storage.ContentStorage, error) { +func NewStorage(config storage.PortalStorageConfig, db *pebble.DB) (storage.ContentStorage, error) { cs := &ContentStorage{ nodeId: config.NodeId, - db: config.DB, + db: db, storageCapacityInBytes: config.StorageCapacityMB * 1000_000, log: log.New("storage", config.NetworkName), writeOptions: &pebble.WriteOptions{Sync: false}, diff --git a/storage/pebble/storage_test.go b/storage/pebble/storage_test.go index 99799d0..0172b90 100644 --- a/storage/pebble/storage_test.go +++ b/storage/pebble/storage_test.go @@ -1,4 +1,4 @@ -package pepple +package pebble import ( "testing" @@ -18,7 +18,7 @@ func genBytes(length int) []byte { } func TestNewPeppleDB(t *testing.T) { - db, err := NewPeppleDB(t.TempDir(), 16, 16, "test") + db, err := NewDB(t.TempDir(), 16, 16, "test") assert.NoError(t, err) defer db.Close() @@ -26,18 +26,17 @@ func TestNewPeppleDB(t *testing.T) { } func setupTestStorage(t *testing.T) storage.ContentStorage { - db, err := NewPeppleDB(t.TempDir(), 16, 16, "test") + db, err := NewDB(t.TempDir(), 16, 16, "test") assert.NoError(t, err) t.Cleanup(func() { db.Close() }) - config := PeppleStorageConfig{ + config := storage.PortalStorageConfig{ StorageCapacityMB: 1, - DB: db, NodeId: uint256.NewInt(0).Bytes32(), NetworkName: "test", } - storage, err := NewPeppleStorage(config) + storage, err := NewStorage(config, db) assert.NoError(t, err) return storage } diff --git a/storage/sqlite/storage.go b/storage/sqlite/storage.go index f5a7192..b63791c 100644 --- a/storage/sqlite/storage.go +++ b/storage/sqlite/storage.go @@ -104,10 +104,10 @@ func NewDB(dataDir string, network string) (*sql.DB, error) { return sqlDb, err } -func NewHistoryStorage(config storage.PortalStorageConfig) (storage.ContentStorage, error) { +func NewStorage(config storage.PortalStorageConfig, db *sql.DB) (storage.ContentStorage, error) { hs := &ContentStorage{ nodeId: config.NodeId, - sqliteDB: config.DB, + sqliteDB: db, storageCapacityInBytes: config.StorageCapacityMB * 1000000, log: log.New("storage", config.NetworkName), } @@ -124,7 +124,7 @@ func NewHistoryStorage(config storage.PortalStorageConfig) (storage.ContentStora // necessary to test NetworkName==history because state also initialize HistoryStorage if strings.ToLower(config.NetworkName) == "history" { - portalStorageMetrics, err = portalwire.NewPortalStorageMetrics(config.NetworkName, config.DB) + portalStorageMetrics, err = portalwire.NewPortalStorageMetrics(config.NetworkName, db) if err != nil { return nil, err } diff --git a/storage/sqlite/storage_test.go b/storage/sqlite/storage_test.go index fe0324e..0fce15b 100644 --- a/storage/sqlite/storage_test.go +++ b/storage/sqlite/storage_test.go @@ -31,11 +31,10 @@ func newContentStorage(storageCapacityInMB uint64, nodeId enode.ID) (*ContentSto if err != nil { return nil, err } - hs, err := NewHistoryStorage(contentStorage.PortalStorageConfig{ - DB: db, + hs, err := NewStorage(contentStorage.PortalStorageConfig{ StorageCapacityMB: storageCapacityInMB, NodeId: nodeId, - }) + }, db) if err != nil { return nil, err } diff --git a/storage/tests/storage_bench_test.go b/storage/tests/storage_bench_test.go index fa6fa3c..c2fe791 100644 --- a/storage/tests/storage_bench_test.go +++ b/storage/tests/storage_bench_test.go @@ -48,13 +48,12 @@ func BenchmarkStorageComparison(b *testing.B) { b.Run("Pebble_"+tc.name, func(b *testing.B) { dir, _ := os.MkdirTemp("", "pebble-bench-*") - db, _ := ethpepple.NewPeppleDB(dir, 16, 16, "bench") - storage, _ := ethpepple.NewPeppleStorage(ethpepple.PeppleStorageConfig{ + db, _ := ethpepple.NewDB(dir, 16, 16, "bench") + storage, _ := ethpepple.NewStorage(storage.PortalStorageConfig{ StorageCapacityMB: 1000, - DB: db, NodeId: enode.ID{}, NetworkName: "bench", - }) + }, db) defer func() { db.Close() os.RemoveAll(dir) @@ -77,12 +76,11 @@ func BenchmarkStorageComparison(b *testing.B) { b.Run("SQLite_"+tc.name, func(b *testing.B) { dir, _ := os.MkdirTemp("", "sqlite-bench-*") db, _ := sqlite.NewDB(dir, "bench") - storage, _ := sqlite.NewHistoryStorage(storage.PortalStorageConfig{ + storage, _ := sqlite.NewStorage(storage.PortalStorageConfig{ StorageCapacityMB: 1000, - DB: db, NodeId: enode.ID{}, NetworkName: "bench", - }) + }, db) defer func() { storage.Close() os.RemoveAll(dir)