Merge pull request #334 from ava-labs/db-upgrade

Db upgrade

.ci/lint.sh

@@ -3,5 +3,4 @@ curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/insta
 
 golangci-lint --version
 
-
-golangci-lint run --max-same-issues 0
+golangci-lint run --max-same-issues=0 --timeout=2m

.ci/run_e2e_tests.sh

@@ -18,7 +18,7 @@ echo "Using Avalanche Image: $AVALANCHE_IMAGE"
 
 DOCKER_REPO="avaplatform"
 BYZANTINE_IMAGE="$DOCKER_REPO/avalanche-byzantine:apricot-phase2"
-TEST_SUITE_IMAGE="$DOCKER_REPO/avalanche-testing:apricot-phase2-local-genesis"
+TEST_SUITE_IMAGE="$DOCKER_REPO/avalanche-testing:apricot-phase2-db-upgrade"
 
 # Kurtosis Environment Parameters
 KURTOSIS_CORE_CHANNEL="1.0.3"

.github/workflows/build-deb-pkg.sh

@@ -14,10 +14,15 @@ OK=`cp ./build/avalanchego $AVALANCHE_BUILD_BIN_DIR`
 if [[ $OK -ne 0 ]]; then
   exit $OK;
 fi
-OK=`cp ./build/plugins/evm $AVALANCHE_LIB_DIR`
+OK=`cp -r ./build/avalanchego-latest $AVALANCHE_LIB_DIR`
 if [[ $OK -ne 0 ]]; then
   exit $OK;
 fi
+OK=`cp -r ./build/avalanchego-preupgrade $AVALANCHE_LIB_DIR`
+if [[ $OK -ne 0 ]]; then
+  exit $OK;
+fi
+
 OK=`cp $TEMPLATE/control $DEBIAN_CONF/control`
 if [[ $OK -ne 0 ]]; then
   exit $OK;

.github/workflows/build-macos-release.yml

@@ -39,7 +39,7 @@ jobs:
         sudo installer -pkg AWSCLIV2.pkg -target /
 
     - name: Create zip file
-      run: 7z a avalanchego-macos-${VERSION}.zip build/avalanchego build/plugins
+      run: 7z a avalanchego-macos-${VERSION}.zip build/avalanchego build/avalanchego-latest build/avalanchego-preupgrade
       env:
         VERSION: ${{ steps.get_version.outputs.VERSION }}
 

.github/workflows/build-tgz-pkg.sh

@@ -8,10 +8,15 @@ OK=`cp ./build/avalanchego $AVALANCHE_ROOT`
 if [[ $OK -ne 0 ]]; then
   exit $OK;
 fi
-OK=`cp -r ./build/plugins $AVALANCHE_ROOT`
+OK=`cp -r ./build/avalanchego-preupgrade $AVALANCHE_ROOT`
 if [[ $OK -ne 0 ]]; then
   exit $OK;
 fi
+OK=`cp -r ./build/avalanchego-latest $AVALANCHE_ROOT`
+if [[ $OK -ne 0 ]]; then
+  exit $OK;
+fi
+
 
 echo "Build tgz package..."
 cd $PKG_ROOT

.github/workflows/build-win-release.yml

@@ -37,8 +37,11 @@ jobs:
     - name: Create zip
       run: |
         mv .\build\avalanchego .\build\avalanchego.exe
-        mv .\build\plugins\evm .\build\plugins\evm.exe
-        Compress-Archive -Path .\build\avalanchego.exe,.\build\plugins -DestinationPath .\build\avalanchego-win-${{ steps.get_version.outputs.VERSION }}-experimental.zip
+        mv .\build\avalanchego-latest\avalanchego-process .\build\avalanchego-latest\avalanchego-process.exe
+        mv .\build\avalanchego-latest\plugins\evm .\build\avalanchego-latest\plugins\evm.exe
+        mv .\build\avalanchego-preupgrade\avalanchego-process .\build\avalanchego-preupgrade\avalanchego-process.exe
+        mv .\build\avalanchego-preupgrade\plugins\evm .\build\avalanchego-preupgrade\plugins\evm.exe
+        Compress-Archive -Path .\build\avalanchego.exe,.\build\avalanchego-latest,.\build\avalanchego-preupgrade -DestinationPath .\build\avalanchego-win-${{ steps.get_version.outputs.VERSION }}-experimental.zip
 
     - name: Configure AWS credentials
       uses: aws-actions/configure-aws-credentials@v1

.gitignore

@@ -14,6 +14,7 @@ awscpu
 
 # Test binary, build with `go test -c`
 *.test
+tmp/
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out

.golangci.yml

@@ -1,15 +1,23 @@
+run:
+  # skip auto-generated files.
+  skip-files:
+    - ".*\\.pb\\.go$"
+
 linters:
   # please, do not use `enable-all`: it's deprecated and will be removed soon.
   # inverted configuration with `enable-all` and `disable` is not scalable during updates of golangci-lint
   disable-all: true
 
   enable:
+    - asciicheck
     - bodyclose
     - deadcode
     - errcheck
+    - exportloopref
     - goconst
     - gocritic
     - gofmt
+    - gofumpt
     - goimports
     - golint
     - gosec
@@ -19,9 +27,20 @@ linters:
     - interfacer
     - misspell
     - nakedret
+    - nolintlint
+    - prealloc
+    - stylecheck
+    - typecheck
     - unconvert
+    - unparam
+    - unused
     - varcheck
+    - unconvert
+    - whitespace
     # - lll
-    # - gocyclo
     # - gomnd
     # - goprintffuncname
+    # - staticcheck
+    # - structcheck
+    # - goerr113
+    # - noctx

Dockerfile

@@ -20,6 +20,7 @@ COPY . avalanchego
 
 WORKDIR $GOPATH/src/github.com/ava-labs/avalanchego
 RUN export AVALANCHEGO_COMMIT=$AVALANCHEGO_COMMIT
+RUN mkdir -p build/avalanchego-preupgrade
 RUN ./scripts/build.sh
 
 RUN ln -sv $GOPATH/src/github.com/ava-labs/avalanchego/ /avalanchego

README.md

@@ -12,7 +12,7 @@ a blockchains platform with high throughput, and blazing fast transactions.
 Avalanche is an incredibly lightweight protocol, so the minimum computer requirements are quite modest.
 Note that as network usage increases, hardware requirements may change.
 
-- Hardware: 2 GHz or faster CPU, 4 GB RAM, >= 200 GB storage.
+- Hardware: 2 GHz or faster CPU, 6 GB RAM, >= 200 GB storage.
 - OS: Ubuntu >= 18.04 or Mac OS X >= Catalina.
 - Software: [Go](https://golang.org/doc/install) version >= 1.15.5 and set up [`$GOPATH`](https://github.com/golang/go/wiki/SettingGOPATH).
 - Network: IPv4 or IPv6 network connection, with an open public port.

api/admin/service.go

@@ -27,9 +27,7 @@ const (
 	stacktraceFile = "stacktrace.txt"
 )
 
-var (
-	errAliasTooLong = errors.New("alias length is too long")
-)
+var errAliasTooLong = errors.New("alias length is too long")
 
 // Admin is the API service for node admin management
 type Admin struct {

api/auth/auth_test.go

@@ -31,10 +31,8 @@ func init() {
 	}
 }
 
-var (
-	// Always returns 200 (http.StatusOK)
-	dummyHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})
-)
+// Always returns 200 (http.StatusOK)
+var dummyHandler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})
 
 func TestNewTokenWrongPassword(t *testing.T) {
 	auth := NewFromHash(logging.NoLog{}, "auth", hashedPassword)

api/info/service.go

@@ -21,7 +21,7 @@ import (
 
 // Info is the API service for unprivileged info on a node
 type Info struct {
-	version       version.Version
+	version       version.Application
 	nodeID        ids.ShortID
 	networkID     uint32
 	log           logging.Logger
@@ -34,7 +34,7 @@ type Info struct {
 // NewService returns a new admin API service
 func NewService(
 	log logging.Logger,
-	version version.Version,
+	version version.Application,
 	nodeID ids.ShortID,
 	networkID uint32,
 	chainManager chains.Manager,

api/keystore/blockchain_keystore.go

@@ -9,9 +9,7 @@ import (
 	"github.com/ava-labs/avalanchego/ids"
 )
 
-var (
-	_ BlockchainKeystore = &blockchainKeystore{}
-)
+var _ BlockchainKeystore = &blockchainKeystore{}
 
 type BlockchainKeystore interface {
 	// Get a database that is able to read and write unencrypted values from the

api/keystore/codec.go

@@ -16,9 +16,7 @@ const (
 	codecVersion = 0
 )
 
-var (
-	c codec.Manager
-)
+var c codec.Manager
 
 func init() {
 	lc := linearcodec.New(reflectcodec.DefaultTagName, maxSliceLength)

api/keystore/gkeystore/keystore_client.go

@@ -16,9 +16,7 @@ import (
 	"github.com/ava-labs/avalanchego/database/rpcdb/rpcdbproto"
 )
 
-var (
-	_ keystore.BlockchainKeystore = &Client{}
-)
+var _ keystore.BlockchainKeystore = &Client{}
 
 // Client is a snow.Keystore that talks over RPC.
 type Client struct {

api/keystore/gkeystore/keystore_server.go

@@ -18,9 +18,7 @@ import (
 	"github.com/ava-labs/avalanchego/vms/rpcchainvm/grpcutils"
 )
 
-var (
-	_ gkeystoreproto.KeystoreServer = &Server{}
-)
+var _ gkeystoreproto.KeystoreServer = &Server{}
 
 // Server is a snow.Keystore that is managed over RPC.
 type Server struct {

api/keystore/keystore.go

@@ -14,6 +14,7 @@ import (
 	"github.com/ava-labs/avalanchego/chains/atomic"
 	"github.com/ava-labs/avalanchego/database"
 	"github.com/ava-labs/avalanchego/database/encdb"
+	"github.com/ava-labs/avalanchego/database/manager"
 	"github.com/ava-labs/avalanchego/database/prefixdb"
 	"github.com/ava-labs/avalanchego/ids"
 	"github.com/ava-labs/avalanchego/utils/logging"
@@ -31,6 +32,10 @@ var (
 	errEmptyUsername = errors.New("empty username")
 	errUserMaxLength = fmt.Errorf("username exceeds maximum length of %d chars", maxUserLen)
 
+	usersPrefix = []byte("users")
+	bcsPrefix   = []byte("bcs")
+	migratedKey = []byte("migrated")
+
 	_ Keystore = &keystore{}
 )
 
@@ -107,13 +112,15 @@ type keystore struct {
 	//          BID  BID  BID
 }
 
-func New(log logging.Logger, db database.Database) Keystore {
-	return &keystore{
+func New(log logging.Logger, dbManager manager.Manager) (Keystore, error) {
+	currentDB := dbManager.Current()
+	keystore := &keystore{
 		log:                log,
 		usernameToPassword: make(map[string]*password.Hash),
-		userDB:             prefixdb.New([]byte("users"), db),
-		bcDB:               prefixdb.New([]byte("bcs"), db),
+		userDB:             prefixdb.New(usersPrefix, currentDB.Database),
+		bcDB:               prefixdb.New(bcsPrefix, currentDB.Database),
 	}
+	return keystore, keystore.migrate(dbManager)
 }
 
 func (ks *keystore) CreateHandler() (http.Handler, error) {

api/keystore/migration.go

@@ -0,0 +1,91 @@
+// (c) 2021-2022, Ava Labs, Inc. All rights reserved.
+// See the file LICENSE for licensing terms.
+package keystore
+
+import (
+	"github.com/ava-labs/avalanchego/chains/atomic"
+	"github.com/ava-labs/avalanchego/database"
+	"github.com/ava-labs/avalanchego/database/manager"
+	"github.com/ava-labs/avalanchego/database/prefixdb"
+	"github.com/ava-labs/avalanchego/version"
+)
+
+// Perform a database migration if required
+func (ks *keystore) migrate(dbManager manager.Manager) error {
+	prevDB, prevDBExists := dbManager.Previous()
+	if !prevDBExists {
+		// There is nothing to migrate
+		return nil
+	}
+	prevDBVersion := prevDB.Version
+	currentDB := dbManager.Current()
+	currentDBVersion := currentDB.Version
+	// Right now the only valid migration is from database version 1.0.0 to 1.4.5
+	if prevDBVersion.Compare(version.DatabaseVersion1_0_0) == 0 &&
+		currentDBVersion.Compare(version.DatabaseVersion1_4_5) == 0 {
+		return ks.migrate1_0_0To1_4_5(prevDB, currentDB)
+	}
+	return nil
+}
+
+func (ks *keystore) migrate1_0_0To1_4_5(prevDB, currentDB *manager.VersionedDatabase) error {
+	migrated, err := currentDB.Database.Has(migratedKey)
+	if err != nil {
+		return err
+	} else if migrated {
+		// Skip migration if previously done.
+		return nil
+	}
+	ks.log.Info("migrating keystore from database version %s to %s", prevDB.Version, currentDB.Version)
+
+	previousUserDB := prefixdb.New(usersPrefix, prevDB.Database)
+	previousBCDB := prefixdb.New(bcsPrefix, prevDB.Database)
+	userIterator := previousUserDB.NewIterator()
+	defer userIterator.Release()
+	for userIterator.Next() {
+		username := userIterator.Key()
+		exists, err := ks.userDB.Has(username)
+		if err != nil {
+			return err
+		} else if exists { // already have this username in the keystore
+			continue
+		}
+
+		userBatch := ks.userDB.NewBatch()
+		if err := userBatch.Put(username, userIterator.Value()); err != nil {
+			return err
+		}
+
+		currentUserBCDB := prefixdb.New(username, ks.bcDB)
+		previousUserBCDB := prefixdb.New(username, previousBCDB)
+		bcsBatch := currentUserBCDB.NewBatch()
+		if err := migrateUserBCDB(previousUserBCDB, bcsBatch, userBatch); err != nil {
+			return err
+		}
+	}
+
+	if err := userIterator.Error(); err != nil {
+		return err
+	}
+	if err := currentDB.Database.Put(migratedKey, []byte(prevDB.Version.String())); err != nil {
+		return err
+	}
+	ks.log.Info("finished migrating keystore from database version %s to %s", prevDB.Version, currentDB.Version)
+	return nil
+}
+
+func migrateUserBCDB(previousUserBCDB database.Database, bcsBatch database.Batch, userBatch database.Batch) error {
+	iterator := previousUserBCDB.NewIterator()
+	defer iterator.Release()
+
+	for iterator.Next() {
+		if err := bcsBatch.Put(iterator.Key(), iterator.Value()); err != nil {
+			return err
+		}
+	}
+
+	if err := iterator.Error(); err != nil {
+		return err
+	}
+	return atomic.WriteAll(userBatch, bcsBatch)
+}