Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

tests: fix state test error checking #25702

Merged
merged 3 commits into from
Sep 26, 2022
Merged

tests: fix state test error checking #25702

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
2d5d663
tests: add error check for false negatives
meowsbits Sep 6, 2022
a3ed8fa
tests: raise error returned by core.ApplyMessage
meowsbits Sep 6, 2022
f8cbab5
tests: method checks error expectations for state tests
meowsbits Sep 6, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 6 additions & 14 deletions tests/state_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,12 +57,12 @@ func TestState(t *testing.T) {

// Broken tests:
// Expected failures:
//st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/Byzantium/0`, "bug in test")
//st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/Byzantium/3`, "bug in test")
//st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/Constantinople/0`, "bug in test")
//st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/Constantinople/3`, "bug in test")
//st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/ConstantinopleFix/0`, "bug in test")
//st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/ConstantinopleFix/3`, "bug in test")
// st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/Byzantium/0`, "bug in test")
// st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/Byzantium/3`, "bug in test")
// st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/Constantinople/0`, "bug in test")
// st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/Constantinople/3`, "bug in test")
// st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/ConstantinopleFix/0`, "bug in test")
// st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/ConstantinopleFix/3`, "bug in test")

// For Istanbul, older tests were moved into LegacyTests
for _, dir := range []string{
Expand All @@ -78,10 +78,6 @@ func TestState(t *testing.T) {
t.Run(key+"/trie", func(t *testing.T) {
withTrace(t, test.gasLimit(subtest), func(vmconfig vm.Config) error {
_, _, err := test.Run(subtest, vmconfig, false)
if err != nil && len(test.json.Post[subtest.Fork][subtest.Index].ExpectException) > 0 {
// Ignore expected errors (TODO MariusVanDerWijden check error string)
return nil
}
return st.checkFailure(t, err)
})
})
Expand All @@ -93,10 +89,6 @@ func TestState(t *testing.T) {
return err
}
}
if err != nil && len(test.json.Post[subtest.Fork][subtest.Index].ExpectException) > 0 {
// Ignore expected errors (TODO MariusVanDerWijden check error string)
return nil
}
return st.checkFailure(t, err)
})
})
Expand Down
35 changes: 32 additions & 3 deletions tests/state_test_util.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,11 +159,39 @@ func (t *StateTest) Subtests() []StateSubtest {
return sub
}

// checkError checks if the error returned by the state transition matches any expected error.
// A failing expectation returns a wrapped version of the original error, if any,
// or a new error detailing the failing expectation.
// This function does not return or modify the original error, it only evaluates and returns expectations for the error.
func (t *StateTest) checkError(subtest StateSubtest, err error) error {
expectedError := t.json.Post[subtest.Fork][subtest.Index].ExpectException
if err == nil && expectedError == "" {
return nil
}
if err == nil && expectedError != "" {
return fmt.Errorf("expected error %q, got no error", expectedError)
}
if err != nil && expectedError == "" {
return fmt.Errorf("unexpected error: %w", err)
}
if err != nil && expectedError != "" {
// Ignore expected errors (TODO MariusVanDerWijden check error string)
return nil
}
return nil
}

// Run executes a specific subtest and verifies the post-state and logs
func (t *StateTest) Run(subtest StateSubtest, vmconfig vm.Config, snapshotter bool) (*snapshot.Tree, *state.StateDB, error) {
snaps, statedb, root, err := t.RunNoVerify(subtest, vmconfig, snapshotter)
if checkedErr := t.checkError(subtest, err); checkedErr != nil {
return snaps, statedb, checkedErr
}
// The error has been checked; if it was unexpected, it's already returned.
if err != nil {
return snaps, statedb, err
// Here, an error exists but it was expected.
// We do not check the post state or logs.
return snaps, statedb, nil
}
post := t.json.Post[subtest.Fork][subtest.Index]
// N.B: We need to do this in a two-step process, because the first Commit takes care
Expand Down Expand Up @@ -231,7 +259,8 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh
snapshot := statedb.Snapshot()
gaspool := new(core.GasPool)
gaspool.AddGas(block.GasLimit())
if _, err := core.ApplyMessage(evm, msg, gaspool); err != nil {
_, err = core.ApplyMessage(evm, msg, gaspool)
if err != nil {
statedb.RevertToSnapshot(snapshot)
}
// Add 0-value mining reward. This only makes a difference in the cases
Expand All @@ -244,7 +273,7 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh
statedb.Commit(config.IsEIP158(block.Number()))
// And _now_ get the state root
root := statedb.IntermediateRoot(config.IsEIP158(block.Number()))
return snaps, statedb, root, nil
return snaps, statedb, root, err
}

func (t *StateTest) gasLimit(subtest StateSubtest) uint64 {
Expand Down