Merge pull request #287 from ethereum/bench_helpers

bench: Extract and improve benchmarking helpers

test/bench/CMakeLists.txt

@@ -4,10 +4,14 @@
 
 include(CheckIncludeFileCXX)
 
-add_executable(evmone-bench bench.cpp)
-
+add_executable(evmone-bench)
 target_include_directories(evmone-bench PRIVATE ${evmone_private_include_dir})
 target_link_libraries(evmone-bench PRIVATE evmone testutils evmc::loader benchmark::benchmark)
+target_sources(
+    evmone-bench PRIVATE
+    bench.cpp
+    helpers.hpp
+)
 
 set(HAVE_STD_FILESYSTEM 0)
 
@@ -16,7 +20,7 @@ if(NOT APPLE)
     # For macos assume not available. XCode 11 provides the header, but it can only be used
     # when building for macos 10.15.
     check_include_file_cxx(filesystem have_filesystem_include)
-    if (have_filesystem_include)
+    if(have_filesystem_include)
         set(HAVE_STD_FILESYSTEM 1)
     endif()
 endif()

test/bench/bench.cpp

@@ -2,14 +2,11 @@
 // Copyright 2019 The evmone Authors.
 // SPDX-License-Identifier: Apache-2.0
 
+#include "helpers.hpp"
 #include <benchmark/benchmark.h>
 #include <evmc/evmc.hpp>
 #include <evmc/loader.h>
-#include <evmone/analysis.hpp>
 #include <evmone/evmone.h>
-#include <test/utils/utils.hpp>
-
-#include <cctype>
 #include <fstream>
 #include <iostream>
 
@@ -24,6 +21,10 @@ namespace fs = ghc::filesystem;
 
 using namespace benchmark;
 
+namespace evmone::test
+{
+std::map<std::string_view, evmc::VM> registered_vms;
+
 namespace
 {
 struct BenchmarkCase
@@ -51,68 +52,8 @@ struct BenchmarkCase
     {}
 };
 
-constexpr auto gas_limit = std::numeric_limits<int64_t>::max();
-constexpr auto inputs_extension = ".inputs";
-
-std::map<std::string_view, evmc::VM> registered_vms;
-
-
-void analyse(State& state, bytes_view code) noexcept
-{
-    auto bytes_analysed = uint64_t{0};
-    for (auto _ : state)
-    {
-        auto r = evmone::analyze(EVMC_ISTANBUL, code.data(), code.size());
-        DoNotOptimize(r);
-        bytes_analysed += code.size();
-    }
-    state.counters["size"] = Counter(static_cast<double>(code.size()));
-    state.counters["rate"] = Counter(static_cast<double>(bytes_analysed), Counter::kIsRate);
-}
-
-inline evmc::result execute(evmc::VM& vm, bytes_view code, bytes_view input) noexcept
-{
-    auto msg = evmc_message{};
-    msg.gas = gas_limit;
-    msg.input_data = input.data();
-    msg.input_size = input.size();
-    return vm.execute(EVMC_ISTANBUL, msg, code.data(), code.size());
-}
-
-void execute(State& state, evmc::VM& vm, bytes_view code, bytes_view input,
-    bytes_view expected_output) noexcept
-{
-    {  // Test run.
-        const auto r = execute(vm, code, input);
-        if (r.status_code != EVMC_SUCCESS)
-        {
-            state.SkipWithError(("failure: " + std::to_string(r.status_code)).c_str());
-            return;
-        }
 
-        if (!expected_output.empty())
-        {
-            const auto output = bytes_view{r.output_data, r.output_size};
-            if (output != expected_output)
-            {
-                auto error = "got: " + hex(output) + "  expected: " + hex(expected_output);
-                state.SkipWithError(error.c_str());
-                return;
-            }
-        }
-    }
-
-    auto total_gas_used = int64_t{0};
-    auto iteration_gas_used = int64_t{0};
-    for (auto _ : state)
-    {
-        auto r = execute(vm, code, input);
-        iteration_gas_used = gas_limit - r.gas_left;
-        total_gas_used += iteration_gas_used;
-    }
-    state.counters["gas_used"] = Counter(static_cast<double>(iteration_gas_used));
-    state.counters["gas_rate"] = Counter(static_cast<double>(total_gas_used), Counter::kIsRate);
-}
+constexpr auto inputs_extension = ".inputs";
 
 /// Loads the benchmark case's inputs from the inputs file at the given path.
 std::vector<BenchmarkCase::Input> load_inputs(const fs::path& path)
@@ -221,7 +162,7 @@ void register_benchmarks(const std::vector<BenchmarkCase>& benchmark_cases)
         if (registered_vms.count("advanced"))
         {
             RegisterBenchmark(("advanced/analyse/" + b.name).c_str(), [&b](State& state) {
-                analyse(state, b.code);
+                analyse(state, default_revision, b.code);
             })->Unit(kMicrosecond);
         }
 
@@ -322,9 +263,11 @@ std::tuple<int, std::vector<BenchmarkCase>> parseargs(int argc, char** argv)
     }
 }
 }  // namespace
+}  // namespace evmone::test
 
 int main(int argc, char** argv)
 {
+    using namespace evmone::test;
     try
     {
         Initialize(&argc, argv);

test/bench/helpers.hpp

@@ -0,0 +1,81 @@
+// evmone: Fast Ethereum Virtual Machine implementation
+// Copyright 2019 The evmone Authors.
+// SPDX-License-Identifier: Apache-2.0
+#pragma once
+
+#include "test/utils/utils.hpp"
+#include <benchmark/benchmark.h>
+#include <evmc/evmc.hpp>
+#include <evmone/analysis.hpp>
+
+namespace evmone::test
+{
+extern std::map<std::string_view, evmc::VM> registered_vms;
+
+constexpr auto default_revision = EVMC_ISTANBUL;
+constexpr auto default_gas_limit = std::numeric_limits<int64_t>::max();
+
+inline void analyse(benchmark::State& state, evmc_revision rev, bytes_view code) noexcept
+{
+    auto bytes_analysed = uint64_t{0};
+    for (auto _ : state)
+    {
+        auto r = evmone::analyze(rev, code.data(), code.size());
+        benchmark::DoNotOptimize(r);
+        bytes_analysed += code.size();
+    }
+
+    using benchmark::Counter;
+    state.counters["size"] = Counter(static_cast<double>(code.size()));
+    state.counters["rate"] = Counter(static_cast<double>(bytes_analysed), Counter::kIsRate);
+}
+
+inline evmc::result execute(
+    evmc::VM& vm, evmc_revision rev, int64_t gas_limit, bytes_view code, bytes_view input) noexcept
+{
+    auto msg = evmc_message{};
+    msg.gas = gas_limit;
+    msg.input_data = input.data();
+    msg.input_size = input.size();
+    return vm.execute(rev, msg, code.data(), code.size());
+}
+
+inline void execute(benchmark::State& state, evmc::VM& vm, bytes_view code, bytes_view input = {},
+    bytes_view expected_output = {}) noexcept
+{
+    constexpr auto rev = default_revision;
+    constexpr auto gas_limit = default_gas_limit;
+    {  // Test run.
+        const auto r = execute(vm, rev, gas_limit, code, input);
+        if (r.status_code != EVMC_SUCCESS)
+        {
+            state.SkipWithError(("failure: " + std::to_string(r.status_code)).c_str());
+            return;
+        }
+
+        if (!expected_output.empty())
+        {
+            const auto output = bytes_view{r.output_data, r.output_size};
+            if (output != expected_output)
+            {
+                state.SkipWithError(
+                    ("got: " + hex(output) + "  expected: " + hex(expected_output)).c_str());
+                return;
+            }
+        }
+    }
+
+    auto total_gas_used = int64_t{0};
+    auto iteration_gas_used = int64_t{0};
+    for (auto _ : state)
+    {
+        auto r = execute(vm, rev, gas_limit, code, input);
+        iteration_gas_used = gas_limit - r.gas_left;
+        total_gas_used += iteration_gas_used;
+    }
+
+    using benchmark::Counter;
+    state.counters["gas_used"] = Counter(static_cast<double>(iteration_gas_used));
+    state.counters["gas_rate"] = Counter(static_cast<double>(total_gas_used), Counter::kIsRate);
+}
+}  // namespace evmone::test