Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Consolidate EVM benchmark framework #150

Merged
merged 4 commits into from
Mar 13, 2024
Merged

Consolidate EVM benchmark framework #150

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
42b10a9
Change the way EVM benchmark subtests are ran
Stefan-Ethernal Mar 10, 2024
6840b27
Remove redundant listFolders call
Stefan-Ethernal Mar 11, 2024
23cac69
Do not execute EVM state tests multiple times
Stefan-Ethernal Mar 11, 2024
f91d80b
Remove listFolders function
Stefan-Ethernal Mar 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 29 additions & 28 deletions tests/evm_benchmark_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@ import (
"fmt"
"math/big"
"os"
"path/filepath"
"strings"
"testing"
"time"

Expand All @@ -18,48 +20,47 @@ import (
)

const (
benchmarksDir = "evm-benchmarks/benchmarks"
chainID = 10
benchmarksDir = "evm-benchmarks"
testsExtension = ".json"
chainID = 10
)

func BenchmarkEVM(b *testing.B) {
folders, err := listFolders([]string{benchmarksDir})
files, err := listFiles(benchmarksDir, testsExtension)
require.NoError(b, err)

for _, folder := range folders {
files, err := listFiles(folder, ".json")
require.NoError(b, err)
for _, file := range files {
name := filepath.ToSlash(strings.TrimPrefix(strings.TrimSuffix(file, testsExtension), benchmarksDir+string(filepath.Separator)))

for _, file := range files {
name := getTestName(file)
b.Run(name, func(b *testing.B) {
data, err := os.ReadFile(file)
require.NoError(b, err)

b.Run(name, func(b *testing.B) {
data, err := os.ReadFile(file)
require.NoError(b, err)
var testCases map[string]testCase
if err = json.Unmarshal(data, &testCases); err != nil {
b.Fatalf("failed to unmarshal %s: %v", file, err)
}

var testCases map[string]testCase
if err = json.Unmarshal(data, &testCases); err != nil {
b.Fatalf("failed to unmarshal %s: %v", file, err)
}

for _, tc := range testCases {
for fork, postState := range tc.Post {
forks, exists := Forks[fork]
if !exists {
b.Logf("%s fork is not supported, skipping test case.", fork)
continue
}
for _, tc := range testCases {
for fork, postState := range tc.Post {
forks, exists := Forks[fork]
if !exists {
b.Logf("%s fork is not supported, skipping test case.", fork)
continue
}

fc := &forkConfig{name: fork, forks: forks}
fc := &forkConfig{name: fork, forks: forks}

for idx, postStateEntry := range postState {
for idx, postStateEntry := range postState {
key := fmt.Sprintf("%s/%d", fork, idx)
b.Run(key, func(b *testing.B) {
err := runBenchmarkTest(b, tc, fc, postStateEntry)
require.NoError(b, err, fmt.Sprintf("test %s (case#%d) execution failed", name, idx))
}
})
}
}
})
}
}
})
}
}

Expand Down
73 changes: 34 additions & 39 deletions tests/state_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,60 +52,55 @@ func TestState(t *testing.T) {
"CALLBlake2f_MaxRounds",
}

folders, err := listFolders([]string{stateTestsDir})
files, err := listFiles(stateTestsDir, ".json")
require.NoError(t, err)

for _, folder := range folders {
files, err := listFiles(folder, ".json")
require.NoError(t, err)
for _, file := range files {
if contains(long, file) && testing.Short() {
t.Logf("Long test '%s' is skipped in short mode\n", file)

for _, file := range files {
if contains(long, file) && testing.Short() {
t.Logf("Long test '%s' is skipped in short mode\n", file)

continue
}
continue
}

if contains(skip, file) {
t.Logf("Test '%s' is skipped\n", file)
if contains(skip, file) {
t.Logf("Test '%s' is skipped\n", file)

continue
}
continue
}

file := file
t.Run(file, func(t *testing.T) {
t.Parallel()
file := file
t.Run(file, func(t *testing.T) {
t.Parallel()

data, err := os.ReadFile(file)
require.NoError(t, err)
data, err := os.ReadFile(file)
require.NoError(t, err)

var testCases map[string]testCase
if err = json.Unmarshal(data, &testCases); err != nil {
t.Fatalf("failed to unmarshal %s: %v", file, err)
}
var testCases map[string]testCase
if err = json.Unmarshal(data, &testCases); err != nil {
t.Fatalf("failed to unmarshal %s: %v", file, err)
}

for _, tc := range testCases {
for fork, postState := range tc.Post {
forks, exists := Forks[fork]
if !exists {
t.Logf("%s fork is not supported, skipping test case.", fork)
continue
}
for _, tc := range testCases {
for fork, postState := range tc.Post {
forks, exists := Forks[fork]
if !exists {
t.Logf("%s fork is not supported, skipping test case.", fork)
continue
}

fc := &forkConfig{name: fork, forks: forks}
fc := &forkConfig{name: fork, forks: forks}

for idx, postStateEntry := range postState {
start := time.Now()
err := runSpecificTestCase(t, file, tc, fc, idx, postStateEntry)
for idx, postStateEntry := range postState {
start := time.Now()
err := runSpecificTestCase(t, file, tc, fc, idx, postStateEntry)

t.Logf("'%s' executed. Fork: %s. Case: %d, Duration=%v\n", file, fork, idx, time.Since(start))
t.Logf("'%s' executed. Fork: %s. Case: %d, Duration=%v\n", file, fork, idx, time.Since(start))

require.NoError(t, tc.checkError(fork, idx, err))
}
require.NoError(t, tc.checkError(fork, idx, err))
}
}
})
}
}
})
}
}

Expand Down
56 changes: 13 additions & 43 deletions tests/state_test_util.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (
"fmt"
"io/fs"
"math/big"
"os"
"path/filepath"
"strings"
"testing"
Expand Down Expand Up @@ -615,57 +614,28 @@ func contains(l []string, name string) bool {
return false
}

func listFolders(paths []string) ([]string, error) {
var folders []string

for _, rootPath := range paths {
err := filepath.WalkDir(rootPath, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}

if d.IsDir() {
files, err := os.ReadDir(path)
if err != nil {
return err
}

if len(files) > 0 {
folders = append(folders, path)
}
}

return nil
})

if err != nil {
return nil, err
}
}

return folders, nil
}

func listFiles(folder string, extensions ...string) ([]string, error) {
var files []string

err := filepath.WalkDir(folder, func(path string, d fs.DirEntry, err error) error {
err := filepath.Walk(folder, func(path string, info fs.FileInfo, err error) error {
if err != nil {
return err
}

if !d.IsDir() {
if len(extensions) > 0 {
// filter files by extensions
for _, ext := range extensions {
if strings.HasSuffix(path, ext) {
files = append(files, path)
}
if info.IsDir() {
return nil
}

if len(extensions) > 0 {
// filter files by extensions
for _, ext := range extensions {
if fileExt := filepath.Ext(path); fileExt == ext {
files = append(files, path)
}
} else {
// if no extensions filter is provided, add all files
files = append(files, path)
}
} else {
// if no extensions filter is provided, add all files
files = append(files, path)
}

return nil
Expand Down
Loading