Skip to content

Commit

Permalink
HIP-30: Set up pre-image generation, recording, export and import (ha…
Browse files Browse the repository at this point in the history 
…rmony-one#4494)

* flags: set up preimage flags

* hip30: set up preimage import, export, api

* save pre-images by default

* add pre images api

* goimports

* commit rpc preimages file

* preimages: re-generate them using CLI

* add metrics and numbers for pre-images

* automate generation after import

* move from rpc to core

* goimports

* add back core/preimages.go file

* goimports

* goimports

* export prometheus metric when no error importing preimage

* add preimages flags to rootflags

---------

Co-authored-by: Nita Neou (Soph) <soph@harmony.one>
  • Loading branch information
@MaxMustermann2 @sophoah @Frozen
2 people authored and Frozen committed Sep 19, 2023
1 parent 6d65d11 commit c85e485
Show file tree
Hide file tree
Showing 18 changed files with 682 additions and 4 deletions.
13 changes: 13 additions & 0 deletions cmd/harmony/default.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,7 @@ var defaultConfig = harmonyconfig.HarmonyConfig{
RateLimterEnabled: true,
RequestsPerSecond: nodeconfig.DefaultRPCRateLimit,
EvmCallTimeout: nodeconfig.DefaultEvmCallTimeout,
PreimagesEnabled: false,
},
BLSKeys: harmonyconfig.BlsConfig{
KeyDir: "./.hmy/blskeys",
Expand Down Expand Up @@ -149,6 +150,13 @@ var defaultRevertConfig = harmonyconfig.RevertConfig{
RevertTo: 0,
}

var defaultPreimageConfig = harmonyconfig.PreimageConfig{
ImportFrom: "",
ExportTo: "",
GenerateStart: 0,
GenerateEnd: 0,
}

var defaultLogContext = harmonyconfig.LogContext{
IP: "127.0.0.1",
Port: 9000,
Expand Down Expand Up @@ -291,6 +299,11 @@ func getDefaultRevertConfigCopy() harmonyconfig.RevertConfig {
return config
}

func getDefaultPreimageConfigCopy() harmonyconfig.PreimageConfig {
config := defaultPreimageConfig
return config
}

func getDefaultLogContextCopy() harmonyconfig.LogContext {
config := defaultLogContext
return config
Expand Down
64 changes: 64 additions & 0 deletions cmd/harmony/flags.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@ var (

rpcOptFlags = []cli.Flag{
rpcDebugEnabledFlag,
rpcPreimagesEnabledFlag,
rpcEthRPCsEnabledFlag,
rpcStakingRPCsEnabledFlag,
rpcLegacyRPCsEnabledFlag,
Expand Down Expand Up @@ -205,6 +206,13 @@ var (
revertBeforeFlag,
}

preimageFlags = []cli.Flag{
preimageImportFlag,
preimageExportFlag,
preimageGenerateStartFlag,
preimageGenerateEndFlag,
}

legacyRevertFlags = []cli.Flag{
legacyRevertBeaconFlag,
legacyRevertBeforeFlag,
Expand Down Expand Up @@ -370,6 +378,7 @@ func getRootFlags() []cli.Flag {
flags = append(flags, sysFlags...)
flags = append(flags, devnetFlags...)
flags = append(flags, revertFlags...)
flags = append(flags, preimageFlags...)
flags = append(flags, legacyMiscFlags...)
flags = append(flags, prometheusFlags...)
flags = append(flags, syncFlags...)
Expand Down Expand Up @@ -827,6 +836,12 @@ var (
DefValue: defaultConfig.RPCOpt.DebugEnabled,
Hidden: true,
}
rpcPreimagesEnabledFlag = cli.BoolFlag{
Name: "rpc.preimages",
Usage: "enable preimages export api",
DefValue: defaultConfig.RPCOpt.PreimagesEnabled,
Hidden: true, // not for end users
}

rpcEthRPCsEnabledFlag = cli.BoolFlag{
Name: "rpc.eth",
Expand Down Expand Up @@ -879,6 +894,9 @@ func applyRPCOptFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) {
if cli.IsFlagChanged(cmd, rpcDebugEnabledFlag) {
config.RPCOpt.DebugEnabled = cli.GetBoolFlagValue(cmd, rpcDebugEnabledFlag)
}
if cli.IsFlagChanged(cmd, rpcPreimagesEnabledFlag) {
config.RPCOpt.PreimagesEnabled = cli.GetBoolFlagValue(cmd, rpcPreimagesEnabledFlag)
}
if cli.IsFlagChanged(cmd, rpcEthRPCsEnabledFlag) {
config.RPCOpt.EthRPCsEnabled = cli.GetBoolFlagValue(cmd, rpcEthRPCsEnabledFlag)
}
Expand Down Expand Up @@ -1656,6 +1674,52 @@ func applyRevertFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) {
}
}

var (
preimageImportFlag = cli.StringFlag{
Name: "preimage.import",
Usage: "Import pre-images from CSV file",
Hidden: true,
DefValue: defaultPreimageConfig.ImportFrom,
}
preimageExportFlag = cli.StringFlag{
Name: "preimage.export",
Usage: "Export pre-images to CSV file",
Hidden: true,
DefValue: defaultPreimageConfig.ExportTo,
}
preimageGenerateStartFlag = cli.Uint64Flag{
Name: "preimage.start",
Usage: "The block number from which pre-images are to be generated",
Hidden: true,
DefValue: defaultPreimageConfig.GenerateStart,
}
preimageGenerateEndFlag = cli.Uint64Flag{
Name: "preimage.end",
Usage: "The block number upto and including which pre-images are to be generated",
Hidden: true,
DefValue: defaultPreimageConfig.GenerateEnd,
}
)

func applyPreimageFlags(cmd *cobra.Command, config *harmonyconfig.HarmonyConfig) {
if cli.HasFlagsChanged(cmd, preimageFlags) {
cfg := getDefaultPreimageConfigCopy()
config.Preimage = &cfg
}
if cli.IsFlagChanged(cmd, preimageImportFlag) {
config.Preimage.ImportFrom = cli.GetStringFlagValue(cmd, preimageImportFlag)
}
if cli.IsFlagChanged(cmd, preimageExportFlag) {
config.Preimage.ExportTo = cli.GetStringFlagValue(cmd, preimageExportFlag)
}
if cli.IsFlagChanged(cmd, preimageGenerateStartFlag) {
config.Preimage.GenerateStart = cli.GetUint64FlagValue(cmd, preimageGenerateStartFlag)
}
if cli.IsFlagChanged(cmd, preimageGenerateEndFlag) {
config.Preimage.GenerateEnd = cli.GetUint64FlagValue(cmd, preimageGenerateEndFlag)
}
}

var (
legacyPortFlag = cli.IntFlag{
Name: "port",
Expand Down
89 changes: 89 additions & 0 deletions cmd/harmony/flags_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ func TestHarmonyFlags(t *testing.T) {
RateLimterEnabled: true,
RequestsPerSecond: 1000,
EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout,
PreimagesEnabled: defaultConfig.RPCOpt.PreimagesEnabled,
},
WS: harmonyconfig.WsConfig{
Enabled: true,
Expand Down Expand Up @@ -752,6 +753,7 @@ func TestRPCOptFlags(t *testing.T) {
RateLimterEnabled: true,
RequestsPerSecond: 1000,
EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout,
PreimagesEnabled: defaultConfig.RPCOpt.PreimagesEnabled,
},
},

Expand All @@ -766,6 +768,7 @@ func TestRPCOptFlags(t *testing.T) {
RateLimterEnabled: true,
RequestsPerSecond: 1000,
EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout,
PreimagesEnabled: defaultConfig.RPCOpt.PreimagesEnabled,
},
},

Expand All @@ -780,6 +783,7 @@ func TestRPCOptFlags(t *testing.T) {
RateLimterEnabled: true,
RequestsPerSecond: 1000,
EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout,
PreimagesEnabled: defaultConfig.RPCOpt.PreimagesEnabled,
},
},

Expand All @@ -794,6 +798,7 @@ func TestRPCOptFlags(t *testing.T) {
RateLimterEnabled: true,
RequestsPerSecond: 1000,
EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout,
PreimagesEnabled: defaultConfig.RPCOpt.PreimagesEnabled,
},
},

Expand All @@ -808,6 +813,7 @@ func TestRPCOptFlags(t *testing.T) {
RateLimterEnabled: true,
RequestsPerSecond: 1000,
EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout,
PreimagesEnabled: defaultConfig.RPCOpt.PreimagesEnabled,
},
},

Expand All @@ -822,6 +828,7 @@ func TestRPCOptFlags(t *testing.T) {
RateLimterEnabled: true,
RequestsPerSecond: 1000,
EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout,
PreimagesEnabled: defaultConfig.RPCOpt.PreimagesEnabled,
},
},

Expand All @@ -836,6 +843,7 @@ func TestRPCOptFlags(t *testing.T) {
RateLimterEnabled: true,
RequestsPerSecond: 2000,
EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout,
PreimagesEnabled: defaultConfig.RPCOpt.PreimagesEnabled,
},
},

Expand All @@ -850,6 +858,7 @@ func TestRPCOptFlags(t *testing.T) {
RateLimterEnabled: false,
RequestsPerSecond: 2000,
EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout,
PreimagesEnabled: defaultConfig.RPCOpt.PreimagesEnabled,
},
},

Expand All @@ -864,6 +873,22 @@ func TestRPCOptFlags(t *testing.T) {
RateLimterEnabled: true,
RequestsPerSecond: 1000,
EvmCallTimeout: "10s",
PreimagesEnabled: defaultConfig.RPCOpt.PreimagesEnabled,
},
},

{
args: []string{"--rpc.preimages"},
expConfig: harmonyconfig.RpcOptConfig{
DebugEnabled: false,
EthRPCsEnabled: true,
StakingRPCsEnabled: true,
LegacyRPCsEnabled: true,
RpcFilterFile: "./.hmy/rpc_filter.txt",
RateLimterEnabled: true,
RequestsPerSecond: 1000,
EvmCallTimeout: defaultConfig.RPCOpt.EvmCallTimeout,
PreimagesEnabled: true,
},
},
}
Expand Down Expand Up @@ -1509,6 +1534,70 @@ func TestRevertFlags(t *testing.T) {
}
}

func TestPreimageFlags(t *testing.T) {
tests := []struct {
args []string
expConfig *harmonyconfig.PreimageConfig
expErr error
}{
{
args: []string{},
expConfig: nil,
},
{
args: []string{"--preimage.import", "/path/to/source.csv"},
expConfig: &harmonyconfig.PreimageConfig{
ImportFrom: "/path/to/source.csv",
ExportTo: defaultPreimageConfig.ExportTo,
GenerateStart: defaultPreimageConfig.GenerateStart,
GenerateEnd: defaultPreimageConfig.GenerateEnd,
},
},
{
args: []string{"--preimage.export", "/path/to/destination.csv"},
expConfig: &harmonyconfig.PreimageConfig{
ImportFrom: defaultPreimageConfig.ImportFrom,
ExportTo: "/path/to/destination.csv",
GenerateStart: defaultPreimageConfig.GenerateStart,
GenerateEnd: defaultPreimageConfig.GenerateEnd,
},
},
{
args: []string{"--preimage.start", "1"},
expConfig: &harmonyconfig.PreimageConfig{
ImportFrom: defaultPreimageConfig.ImportFrom,
ExportTo: defaultPreimageConfig.ExportTo,
GenerateStart: 1,
GenerateEnd: defaultPreimageConfig.GenerateEnd,
},
},
{
args: []string{"--preimage.end", "2"},
expConfig: &harmonyconfig.PreimageConfig{
ImportFrom: defaultPreimageConfig.ImportFrom,
ExportTo: defaultPreimageConfig.ExportTo,
GenerateStart: defaultPreimageConfig.GenerateStart,
GenerateEnd: 2,
},
},
}
for i, test := range tests {
ts := newFlagTestSuite(t, preimageFlags, applyPreimageFlags)
hc, err := ts.run(test.args)

if assErr := assertError(err, test.expErr); assErr != nil {
t.Fatalf("Test %v: %v", i, assErr)
}
if err != nil || test.expErr != nil {
continue
}
if !reflect.DeepEqual(hc.Preimage, test.expConfig) {
t.Errorf("Test %v:\n\t%+v\n\t%+v", i, hc.Preimage, test.expConfig)
}
ts.tearDown()
}
}

func TestDNSSyncFlags(t *testing.T) {
tests := []struct {
args []string
Expand Down
Loading

0 comments on commit c85e485

Please sign in to comment.