Implement EIP-3076 minimal slashing protection, using a filesystem database

beacon-chain/db/restore.go

@@ -22,7 +22,14 @@ func Restore(cliCtx *cli.Context) error {
 	targetDir := cliCtx.String(cmd.RestoreTargetDirFlag.Name)
 
 	restoreDir := path.Join(targetDir, kv.BeaconNodeDbDirName)
-	if file.Exists(path.Join(restoreDir, kv.DatabaseFileName)) {
+	restoreFile := path.Join(restoreDir, kv.DatabaseFileName)
+
+	dbExists, err := file.Exists(restoreFile, file.Regular)
+	if err != nil {
+		return errors.Wrapf(err, "could not check if database exists in %s", restoreFile)
+	}
+
+	if dbExists {
 		resp, err := prompt.ValidatePrompt(
 			os.Stdin, dbExistsYesNoPrompt, prompt.ValidateYesOrNo,
 		)

cmd/validator/accounts/backup_test.go

@@ -98,7 +98,9 @@ func TestBackupAccounts_Noninteractive_Derived(t *testing.T) {
 
 	// We check a backup.zip file was created at the output path.
 	zipFilePath := filepath.Join(backupDir, accounts.ArchiveFilename)
-	assert.DeepEqual(t, true, file.Exists(zipFilePath))
+	fileExists, err := file.Exists(zipFilePath, file.Regular)
+	require.NoError(t, err, "could not check if backup file exists")
+	assert.Equal(t, true, fileExists, "backup file does not exist")
 
 	// We attempt to unzip the file and verify the keystores do match our accounts.
 	f, err := os.Open(zipFilePath)
@@ -189,7 +191,9 @@ func TestBackupAccounts_Noninteractive_Imported(t *testing.T) {
 
 	// We check a backup.zip file was created at the output path.
 	zipFilePath := filepath.Join(backupDir, accounts.ArchiveFilename)
-	assert.DeepEqual(t, true, file.Exists(zipFilePath))
+	exists, err := file.Exists(zipFilePath, file.Regular)
+	require.NoError(t, err, "could not check if backup file exists")
+	assert.Equal(t, true, exists, "backup file does not exist")
 
 	// We attempt to unzip the file and verify the keystores do match our accounts.
 	f, err := os.Open(zipFilePath)

cmd/validator/accounts/exit_test.go

@@ -395,5 +395,7 @@ func TestExitAccountsCli_WriteJSON_NoBroadcast(t *testing.T) {
 	require.Equal(t, 1, len(formattedExitedKeys))
 	assert.Equal(t, "0x"+keystore.Pubkey[:12], formattedExitedKeys[0])
 
-	require.Equal(t, true, file.Exists(path.Join(out, "validator-exit-1.json")), "Expected file to exist")
+	exists, err := file.Exists(path.Join(out, "validator-exit-1.json"), file.Regular)
+	require.NoError(t, err, "could not check if exit file exists")
+	require.Equal(t, true, exists, "Expected file to exist")
 }

cmd/validator/db/db.go

@@ -10,6 +10,22 @@ import (
 
 var log = logrus.WithField("prefix", "db")
 
+var (
+	// SourceDataDirFlag defines a path on disk where source Prysm databases are stored. Used for conversion.
+	SourceDataDirFlag = &cli.StringFlag{
+		Name:     "source-data-dir",
+		Usage:    "Source data directory",
+		Required: true,
+	}
+
+	// SourceDataDirFlag defines a path on disk where source Prysm databases are stored. Used for conversion.
+	TargetDataDirFlag = &cli.StringFlag{
+		Name:     "target-data-dir",
+		Usage:    "Target data directory",
+		Required: true,
+	}
+)
+
 // Commands for interacting with the Prysm validator database.
 var Commands = &cli.Command{
 	Name:     "db",
@@ -66,5 +82,29 @@ var Commands = &cli.Command{
 				},
 			},
 		},
+		{
+			Name:     "convert-complete-to-minimal",
+			Category: "db",
+			Usage:    "Convert a complete EIP-3076 slashing protection to a minimal one",
+			Flags: []cli.Flag{
+				SourceDataDirFlag,
+				TargetDataDirFlag,
+			},
+			Before: func(cliCtx *cli.Context) error {
+				return cmd.LoadFlagsFromConfig(cliCtx, cliCtx.Command.Flags)
+			},
+			Action: func(cliCtx *cli.Context) error {
+				sourcedDatabasePath := cliCtx.String(SourceDataDirFlag.Name)
+				targetDatabasePath := cliCtx.String(TargetDataDirFlag.Name)
+
+				// Convert the database
+				err := validatordb.ConvertDatabase(cliCtx.Context, sourcedDatabasePath, targetDatabasePath, false)
+				if err != nil {
+					log.WithError(err).Fatal("Could not convert database")
+				}
+
+				return nil
+			},
+		},
 	},
 }

cmd/validator/slashing-protection/BUILD.bazel

@@ -17,8 +17,11 @@ go_library(
         "//io/file:go_default_library",
         "//runtime/tos:go_default_library",
         "//validator/accounts/userprompt:go_default_library",
+        "//validator/db/filesystem:go_default_library",
+        "//validator/db/iface:go_default_library",
         "//validator/db/kv:go_default_library",
         "//validator/slashing-protection-history:go_default_library",
+        "//validator/slashing-protection-history/format:go_default_library",
         "@com_github_pkg_errors//:go_default_library",
         "@com_github_sirupsen_logrus//:go_default_library",
         "@com_github_urfave_cli_v2//:go_default_library",
@@ -35,7 +38,7 @@ go_test(
         "//io/file:go_default_library",
         "//testing/assert:go_default_library",
         "//testing/require:go_default_library",
-        "//validator/db/kv:go_default_library",
+        "//validator/db/common:go_default_library",
         "//validator/db/testing:go_default_library",
         "//validator/slashing-protection-history/format:go_default_library",
         "//validator/testing:go_default_library",

cmd/validator/slashing-protection/export.go

@@ -8,10 +8,14 @@ import (
 	"github.com/pkg/errors"
 	"github.com/prysmaticlabs/prysm/v5/cmd"
 	"github.com/prysmaticlabs/prysm/v5/cmd/validator/flags"
+	"github.com/prysmaticlabs/prysm/v5/config/features"
 	"github.com/prysmaticlabs/prysm/v5/io/file"
 	"github.com/prysmaticlabs/prysm/v5/validator/accounts/userprompt"
+	"github.com/prysmaticlabs/prysm/v5/validator/db/filesystem"
+	"github.com/prysmaticlabs/prysm/v5/validator/db/iface"
 	"github.com/prysmaticlabs/prysm/v5/validator/db/kv"
 	slashingprotection "github.com/prysmaticlabs/prysm/v5/validator/slashing-protection-history"
+	"github.com/prysmaticlabs/prysm/v5/validator/slashing-protection-history/format"
 	"github.com/urfave/cli/v2"
 )
 
@@ -30,39 +34,67 @@ const (
 // the validator's db into an EIP standard slashing protection format
 // 4. Format and save the JSON file to a user's specified output directory.
 func exportSlashingProtectionJSON(cliCtx *cli.Context) error {
+	var (
+		validatorDB iface.ValidatorDB
+		found       bool
+		err         error
+	)
+
 	log.Info(
 		"This command exports your validator's attestation and proposal history into " +
 			"a file that can then be imported into any other Prysm setup across computers",
 	)
-	var err error
+
+	// Check if a minimal database is requested
+	isDatabaseMinimal := cliCtx.Bool(features.EnableMinimalSlashingProtection.Name)
+
+	// Read the data directory from the CLI context.
 	dataDir := cliCtx.String(cmd.DataDirFlag.Name)
 	if !cliCtx.IsSet(cmd.DataDirFlag.Name) {
 		dataDir, err = userprompt.InputDirectory(cliCtx, userprompt.DataDirDirPromptText, cmd.DataDirFlag)
 		if err != nil {
 			return errors.Wrapf(err, "could not read directory value from input")
 		}
 	}
-	// ensure that the validator.db is found under the specified dir or its subdirectories
-	found, _, err := file.RecursiveFileFind(kv.ProtectionDbFileName, dataDir)
+
+	// Ensure that the database is found under the specified dir or its subdirectories
+	if isDatabaseMinimal {
+		found, _, err = file.RecursiveDirFind(filesystem.DatabaseDirName, dataDir)
+	} else {
+		found, _, err = file.RecursiveFileFind(kv.ProtectionDbFileName, dataDir)
+	}
+
 	if err != nil {
 		return errors.Wrapf(err, "error finding validator database at path %s", dataDir)
 	}
+
 	if !found {
-		return fmt.Errorf(
-			"validator.db file (validator database) was not found at path %s, so nothing to export",
-			dataDir,
-		)
+		databaseFileDir := kv.ProtectionDbFileName
+		if isDatabaseMinimal {
+			databaseFileDir = filesystem.DatabaseDirName
+		}
+		return fmt.Errorf("%s (validator database) was not found at path %s, so nothing to export", databaseFileDir, dataDir)
+	}
+
+	// Open the validator database.
+	if isDatabaseMinimal {
+		validatorDB, err = filesystem.NewStore(dataDir, nil)
+	} else {
+		validatorDB, err = kv.NewKVStore(cliCtx.Context, dataDir, nil)
 	}
 
-	validatorDB, err := kv.NewKVStore(cliCtx.Context, dataDir, &kv.Config{})
 	if err != nil {
 		return errors.Wrapf(err, "could not access validator database at path %s", dataDir)
 	}
+
+	// Close the database when we're done.
 	defer func() {
 		if err := validatorDB.Close(); err != nil {
 			log.WithError(err).Errorf("Could not close validator DB")
 		}
 	}()
+
+	// Export the slashing protection history from the validator's database.
 	eipJSON, err := slashingprotection.ExportStandardProtectionJSON(cliCtx.Context, validatorDB)
 	if err != nil {
 		return errors.Wrap(err, "could not export slashing protection history")
@@ -79,39 +111,60 @@ func exportSlashingProtectionJSON(cliCtx *cli.Context) error {
 		)
 	}
 
+	// Write the result to the output file
+	if err := writeToOutput(cliCtx, eipJSON); err != nil {
+		return errors.Wrap(err, "could not write slashing protection history to output file")
+	}
+
+	return nil
+}
+
+func writeToOutput(cliCtx *cli.Context, eipJSON *format.EIPSlashingProtectionFormat) error {
+	// Get the output directory where the slashing protection history file will be stored
 	outputDir, err := userprompt.InputDirectory(
 		cliCtx,
 		"Enter your desired output directory for your slashing protection history file",
 		flags.SlashingProtectionExportDirFlag,
 	)
+
 	if err != nil {
 		return errors.Wrap(err, "could not get slashing protection json file")
 	}
+
 	if outputDir == "" {
 		return errors.New("output directory not specified")
 	}
+
+	// Check is the output directory already exists, if not, create it
 	exists, err := file.HasDir(outputDir)
 	if err != nil {
 		return errors.Wrapf(err, "could not check if output directory %s already exists", outputDir)
 	}
+
 	if !exists {
 		if err := file.MkdirAll(outputDir); err != nil {
 			return errors.Wrapf(err, "could not create output directory %s", outputDir)
 		}
 	}
+
+	// Write into the output file
 	outputFilePath := filepath.Join(outputDir, jsonExportFileName)
 	log.Infof("Writing slashing protection export JSON file to %s", outputFilePath)
+
 	encoded, err := json.MarshalIndent(eipJSON, "", "\t")
 	if err != nil {
 		return errors.Wrap(err, "could not JSON marshal slashing protection history")
 	}
+
 	if err := file.WriteFile(outputFilePath, encoded); err != nil {
 		return errors.Wrapf(err, "could not write file to path %s", outputFilePath)
 	}
+
 	log.Infof(
 		"Successfully wrote %s. You can import this file using Prysm's "+
 			"validator slashing-protection-history import command in another machine",
 		outputFilePath,
 	)
+
 	return nil
 }

cmd/validator/slashing-protection/import.go

@@ -7,10 +7,12 @@ import (
 	"github.com/pkg/errors"
 	"github.com/prysmaticlabs/prysm/v5/cmd"
 	"github.com/prysmaticlabs/prysm/v5/cmd/validator/flags"
+	"github.com/prysmaticlabs/prysm/v5/config/features"
 	"github.com/prysmaticlabs/prysm/v5/io/file"
 	"github.com/prysmaticlabs/prysm/v5/validator/accounts/userprompt"
+	"github.com/prysmaticlabs/prysm/v5/validator/db/filesystem"
+	"github.com/prysmaticlabs/prysm/v5/validator/db/iface"
 	"github.com/prysmaticlabs/prysm/v5/validator/db/kv"
-	slashingprotection "github.com/prysmaticlabs/prysm/v5/validator/slashing-protection-history"
 	"github.com/urfave/cli/v2"
 )
 
@@ -24,36 +26,61 @@ import (
 // 4. Call the function which actually imports the data from
 // the standard slashing protection JSON file into our database.
 func importSlashingProtectionJSON(cliCtx *cli.Context) error {
-	var err error
+	var (
+		valDB iface.ValidatorDB
+		found bool
+		err   error
+	)
+
+	// Check if a minimal database is requested
+	isDatabaseMimimal := cliCtx.Bool(features.EnableMinimalSlashingProtection.Name)
+
+	// Get the data directory from the CLI context.
 	dataDir := cliCtx.String(cmd.DataDirFlag.Name)
 	if !cliCtx.IsSet(cmd.DataDirFlag.Name) {
 		dataDir, err = userprompt.InputDirectory(cliCtx, userprompt.DataDirDirPromptText, cmd.DataDirFlag)
 		if err != nil {
 			return errors.Wrapf(err, "could not read directory value from input")
 		}
 	}
-	// ensure that the validator.db is found under the specified dir or its subdirectories
-	found, _, err := file.RecursiveFileFind(kv.ProtectionDbFileName, dataDir)
+
+	// Ensure that the database is found under the specified directory or its subdirectories
+	if isDatabaseMimimal {
+		found, _, err = file.RecursiveDirFind(filesystem.DatabaseDirName, dataDir)
+	} else {
+		found, _, err = file.RecursiveFileFind(kv.ProtectionDbFileName, dataDir)
+	}
+
 	if err != nil {
 		return errors.Wrapf(err, "error finding validator database at path %s", dataDir)
 	}
+
+	message := "Found existing database inside of %s"
 	if !found {
-		log.Infof(
-			"Did not find existing validator.db inside of %s, creating a new one",
-			dataDir,
-		)
+		message = "Did not find existing database inside of %s, creating a new one"
+	}
+
+	log.Infof(message, dataDir)
+
+	// Open the validator database.
+	if isDatabaseMimimal {
+		valDB, err = filesystem.NewStore(dataDir, nil)
 	} else {
-		log.Infof("Found existing validator.db inside of %s", dataDir)
+		valDB, err = kv.NewKVStore(cliCtx.Context, dataDir, nil)
 	}
-	valDB, err := kv.NewKVStore(cliCtx.Context, dataDir, &kv.Config{})
+
 	if err != nil {
 		return errors.Wrapf(err, "could not access validator database at path: %s", dataDir)
 	}
+
+	// Close the database when we're done.
 	defer func() {
 		if err := valDB.Close(); err != nil {
 			log.WithError(err).Errorf("Could not close validator DB")
 		}
 	}()
+
+	// Get the path to the slashing protection JSON file from the CLI context.
 	protectionFilePath, err := userprompt.InputDirectory(cliCtx, userprompt.SlashingProtectionJSONPromptText, flags.SlashingProtectionJSONFileFlag)
 	if err != nil {
 		return errors.Wrap(err, "could not get slashing protection json file")
@@ -65,17 +92,22 @@ func importSlashingProtectionJSON(cliCtx *cli.Context) error {
 			flags.SlashingProtectionJSONFileFlag.Name,
 		)
 	}
+
+	// Read the JSON file from user input.
 	enc, err := file.ReadFileAsBytes(protectionFilePath)
 	if err != nil {
 		return err
 	}
+
+	// Import the data from the standard slashing protection JSON file into our database.
 	log.Infof("Starting import of slashing protection file %s", protectionFilePath)
 	buf := bytes.NewBuffer(enc)
-	if err := slashingprotection.ImportStandardProtectionJSON(
-		cliCtx.Context, valDB, buf,
-	); err != nil {
-		return err
+
+	if err := valDB.ImportStandardProtectionJSON(cliCtx.Context, buf); err != nil {
+		return errors.Wrapf(err, "could not import slashing protection JSON file %s", protectionFilePath)
 	}
+
 	log.Infof("Slashing protection JSON successfully imported into %s", dataDir)
+
 	return nil
 }