From aaa53b142ef2d34acb2f0b0c0dc294920504a0d5 Mon Sep 17 00:00:00 2001
From: Devon Estes <devon.c.estes@gmail.com>
Date: Mon, 29 Jan 2018 14:34:23 +0100
Subject: [PATCH 1/5] Add `dry_run` functionality

This adds a new feature for a dry run, which will execute each scenario
(including before and after hooks) to make sure all your scenarios will
run without exception before doing the actual benchmark. This isn't
counted towards the run times of any benchmarks.

I also ran the formatter on files I touched while adding this feature.
---
 .tool-versions                         |   2 +-
 lib/benchee/benchmark/runner.ex        | 224 +++++---
 lib/benchee/configuration.ex           | 144 +++---
 test/benchee/benchmark/runner_test.exs | 690 +++++++++++++++----------
 4 files changed, 639 insertions(+), 421 deletions(-)

diff --git a/.tool-versions b/.tool-versions
index b6a4778d..48f07ad7 100644
--- a/.tool-versions
+++ b/.tool-versions
@@ -1,2 +1,2 @@
-elixir 1.6.0
+elixir master
 erlang 20.2
diff --git a/lib/benchee/benchmark/runner.ex b/lib/benchee/benchmark/runner.ex
index a2cc3b37..8d32e0c6 100644
--- a/lib/benchee/benchmark/runner.ex
+++ b/lib/benchee/benchmark/runner.ex
@@ -23,9 +23,9 @@ defmodule Benchee.Benchmark.Runner do
   There will be `parallel` processes spawned executing the benchmark job in
   parallel.
   """
-  @spec run_scenarios([Scenario.t], ScenarioContext.t) :: [Scenario.t]
+  @spec run_scenarios([Scenario.t()], ScenarioContext.t()) :: [Scenario.t()]
   def run_scenarios(scenarios, scenario_context) do
-    Enum.map(scenarios, fn(scenario) ->
+    Enum.map(scenarios, fn scenario ->
       parallel_benchmark(scenario, scenario_context)
     end)
   end
@@ -35,32 +35,49 @@ defmodule Benchee.Benchmark.Runner do
          scenario_context = %ScenarioContext{
            printer: printer,
            config: config
-         }) do
+         }
+       ) do
     printer.benchmarking(job_name, input_name, config)
+    dry_run(scenario, scenario_context)
+
     measurements =
       1..config.parallel
-      |> Parallel.map(fn(_) -> measure_scenario(scenario, scenario_context) end)
-      |> List.flatten
+      |> Parallel.map(fn _ -> measure_scenario(scenario, scenario_context) end)
+      |> List.flatten()
+
     %Scenario{scenario | run_times: measurements}
   end
 
+  # This will run the given scenario exactly once, including the before and
+  # after hooks, to ensure the function can execute without raising an error.
+  defp dry_run(scenario, scenario_context = %ScenarioContext{config: %{dry_run: true}}) do
+    scenario_input = run_before_scenario(scenario, scenario_context)
+    scenario_context = %ScenarioContext{scenario_context | scenario_input: scenario_input}
+    measure_iteration(scenario, scenario_context)
+    run_after_scenario(scenario, scenario_context)
+    nil
+  end
+
+  defp dry_run(_, _), do: nil
+
   defp measure_scenario(scenario, scenario_context) do
     scenario_input = run_before_scenario(scenario, scenario_context)
-    scenario_context =
-      %ScenarioContext{scenario_context | scenario_input: scenario_input}
+    scenario_context = %ScenarioContext{scenario_context | scenario_input: scenario_input}
     _ = run_warmup(scenario, scenario_context)
     measurements = run_benchmark(scenario, scenario_context)
     run_after_scenario(scenario, scenario_context)
     measurements
   end
 
-  defp run_before_scenario(%Scenario{
-                             before_scenario: local_before_scenario,
-                             input: input
-                           },
-                           %ScenarioContext{
-                             config: %{before_scenario: global_before_scenario}
-                           }) do
+  defp run_before_scenario(
+         %Scenario{
+           before_scenario: local_before_scenario,
+           input: input
+         },
+         %ScenarioContext{
+           config: %{before_scenario: global_before_scenario}
+         }
+       ) do
     input
     |> run_before_function(global_before_scenario)
     |> run_before_function(local_before_scenario)
@@ -74,51 +91,62 @@ defmodule Benchee.Benchmark.Runner do
     end
   end
 
-  defp run_warmup(scenario, scenario_context = %ScenarioContext{
-                   config: %Configuration{warmup: warmup}
-                 }) do
+  defp run_warmup(
+         scenario,
+         scenario_context = %ScenarioContext{
+           config: %Configuration{warmup: warmup}
+         }
+       ) do
     measure_runtimes(scenario, scenario_context, warmup, false)
   end
 
-  defp run_benchmark(scenario, scenario_context = %ScenarioContext{
-                      config: %Configuration{
-                        time: run_time,
-                        print: %{fast_warning: fast_warning}
-                      }
-                    }) do
+  defp run_benchmark(
+         scenario,
+         scenario_context = %ScenarioContext{
+           config: %Configuration{
+             time: run_time,
+             print: %{fast_warning: fast_warning}
+           }
+         }
+       ) do
     measure_runtimes(scenario, scenario_context, run_time, fast_warning)
   end
 
-  defp run_after_scenario(%{
-                            after_scenario: local_after_scenario
-                          },
-                          %{
-                            config: %{after_scenario: global_after_scenario},
-                            scenario_input: input
-                          }) do
-    if local_after_scenario,  do: local_after_scenario.(input)
+  defp run_after_scenario(
+         %{
+           after_scenario: local_after_scenario
+         },
+         %{
+           config: %{after_scenario: global_after_scenario},
+           scenario_input: input
+         }
+       ) do
+    if local_after_scenario, do: local_after_scenario.(input)
     if global_after_scenario, do: global_after_scenario.(input)
   end
 
   defp measure_runtimes(scenario, context, run_time, fast_warning)
   defp measure_runtimes(_, _, 0, _), do: []
+
   defp measure_runtimes(scenario, scenario_context, run_time, fast_warning) do
     end_time = current_time() + run_time
-    :erlang.garbage_collect
+    :erlang.garbage_collect()
+
     {num_iterations, initial_run_time} =
       determine_n_times(scenario, scenario_context, fast_warning)
-    new_context =
-      %ScenarioContext{scenario_context |
-        current_time: current_time(),
+
+    new_context = %ScenarioContext{
+      scenario_context
+      | current_time: current_time(),
         end_time: end_time,
         num_iterations: num_iterations
-      }
+    }
 
     do_benchmark(scenario, new_context, [initial_run_time])
   end
 
   defp current_time do
-    :erlang.system_time :micro_seconds
+    :erlang.system_time(:micro_seconds)
   end
 
   # If a function executes way too fast measurements are too unreliable and
@@ -126,18 +154,26 @@ defmodule Benchee.Benchmark.Runner do
   # executed in the measurement cycle.
   @minimum_execution_time 10
   @times_multiplier 10
-  defp determine_n_times(scenario, scenario_context = %ScenarioContext{
-                           num_iterations: num_iterations,
-                           printer: printer
-                         }, fast_warning) do
+  defp determine_n_times(
+         scenario,
+         scenario_context = %ScenarioContext{
+           num_iterations: num_iterations,
+           printer: printer
+         },
+         fast_warning
+       ) do
     run_time = measure_iteration(scenario, scenario_context)
+
     if run_time >= @minimum_execution_time do
       {num_iterations, run_time / num_iterations}
     else
       if fast_warning, do: printer.fast_warning()
-      new_context = %ScenarioContext{scenario_context |
-        num_iterations: num_iterations * @times_multiplier
+
+      new_context = %ScenarioContext{
+        scenario_context
+        | num_iterations: num_iterations * @times_multiplier
       }
+
       determine_n_times(scenario, new_context, false)
     end
   end
@@ -147,49 +183,64 @@ defmodule Benchee.Benchmark.Runner do
   # of all processes. That's why we add them to the scenario once after
   # measuring has finished. `scenario` is still needed in general for the
   # benchmarking function, hooks etc.
-  defp do_benchmark(_scenario,
-                    %ScenarioContext{
-                      current_time: current_time, end_time: end_time
-                    }, run_times) when current_time > end_time do
+  defp do_benchmark(
+         _scenario,
+         %ScenarioContext{
+           current_time: current_time,
+           end_time: end_time
+         },
+         run_times
+       )
+       when current_time > end_time do
     # restore correct order - important for graphing
     Enum.reverse(run_times)
   end
+
   defp do_benchmark(scenario, scenario_context, run_times) do
     run_time = iteration_time(scenario, scenario_context)
-    updated_context =
-      %ScenarioContext{scenario_context | current_time: current_time()}
+    updated_context = %ScenarioContext{scenario_context | current_time: current_time()}
     do_benchmark(scenario, updated_context, [run_time | run_times])
   end
 
-  defp iteration_time(scenario, scenario_context = %ScenarioContext{
-                                  num_iterations: num_iterations
-                                }) do
+  defp iteration_time(
+         scenario,
+         scenario_context = %ScenarioContext{
+           num_iterations: num_iterations
+         }
+       ) do
     measure_iteration(scenario, scenario_context) / num_iterations
   end
 
-  defp measure_iteration(scenario = %Scenario{function: function},
-                         scenario_context = %ScenarioContext{
-                           num_iterations: 1
-                         }) do
+  defp measure_iteration(
+         scenario = %Scenario{function: function},
+         scenario_context = %ScenarioContext{
+           num_iterations: 1
+         }
+       ) do
     new_input = run_before_each(scenario, scenario_context)
-    {microseconds, return_value} = :timer.tc main_function(function, new_input)
+    {microseconds, return_value} = :timer.tc(main_function(function, new_input))
     run_after_each(return_value, scenario, scenario_context)
     microseconds
   end
-  defp measure_iteration(scenario, scenario_context = %ScenarioContext{
-                          num_iterations: iterations,
-                        }) when iterations > 1 do
+
+  defp measure_iteration(
+         scenario,
+         scenario_context = %ScenarioContext{
+           num_iterations: iterations
+         }
+       )
+       when iterations > 1 do
     # When we have more than one iteration, then the repetition and calling
     # of hooks is already included in the function, for reference/reasoning see
     # `build_benchmarking_function/2`
     function = build_benchmarking_function(scenario, scenario_context)
-    {microseconds, _return_value} = :timer.tc function
+    {microseconds, _return_value} = :timer.tc(function)
     microseconds
   end
 
   @no_input Benchmark.no_input()
   defp main_function(function, @no_input), do: function
-  defp main_function(function, input),     do: fn -> function.(input) end
+  defp main_function(function, input), do: fn -> function.(input) end
 
   # Builds the appropriate function to benchmark. Takes into account the
   # combinations of the following cases:
@@ -204,23 +255,28 @@ defmodule Benchee.Benchmark.Runner do
   #   hooks.
   defp build_benchmarking_function(
          %Scenario{
-           function: function, before_each: nil, after_each: nil
+           function: function,
+           before_each: nil,
+           after_each: nil
          },
          %ScenarioContext{
            num_iterations: iterations,
            scenario_input: input,
            config: %{after_each: nil, before_each: nil}
-         })
-         when iterations > 1 do
+         }
+       )
+       when iterations > 1 do
     main = main_function(function, input)
     # with no before/after each we can safely omit them and don't get the hit
     # on run time measurements (See PR discussions for this for more info #127)
     fn -> RepeatN.repeat_n(main, iterations) end
   end
+
   defp build_benchmarking_function(
          scenario = %Scenario{function: function},
-         scenario_context = %ScenarioContext{num_iterations: iterations})
-         when iterations > 1 do
+         scenario_context = %ScenarioContext{num_iterations: iterations}
+       )
+       when iterations > 1 do
     fn ->
       RepeatN.repeat_n(
         fn ->
@@ -234,26 +290,30 @@ defmodule Benchee.Benchmark.Runner do
     end
   end
 
-  defp run_before_each(%{
-                         before_each: local_before_each
-                       },
-                       %{
-                         config: %{before_each: global_before_each},
-                         scenario_input: input
-                       }) do
+  defp run_before_each(
+         %{
+           before_each: local_before_each
+         },
+         %{
+           config: %{before_each: global_before_each},
+           scenario_input: input
+         }
+       ) do
     input
     |> run_before_function(global_before_each)
     |> run_before_function(local_before_each)
   end
 
-  defp run_after_each(return_value,
-                      %{
-                        after_each: local_after_each
-                      },
-                      %{
-                        config: %{after_each: global_after_each}
-                      }) do
-    if local_after_each,  do: local_after_each.(return_value)
+  defp run_after_each(
+         return_value,
+         %{
+           after_each: local_after_each
+         },
+         %{
+           config: %{after_each: global_after_each}
+         }
+       ) do
+    if local_after_each, do: local_after_each.(return_value)
     if global_after_each, do: global_after_each.(return_value)
   end
 end
diff --git a/lib/benchee/configuration.ex b/lib/benchee/configuration.ex
index 8c4cceb5..307c5f6d 100644
--- a/lib/benchee/configuration.ex
+++ b/lib/benchee/configuration.ex
@@ -12,53 +12,52 @@ defmodule Benchee.Configuration do
     Formatters.Console
   }
 
-  defstruct [
-    parallel:             1,
-    time:                 5,
-    warmup:               2,
-    formatters:           [Console],
-    print: %{
-      benchmarking:       true,
-      configuration:      true,
-      fast_warning:       true
-    },
-    inputs:               nil,
-    save:                 false,
-    load:                 false,
-    # formatters should end up here but known once are still picked up at
-    # the top level for now
-    formatter_options: %{
-      console: %{
-        comparison:          true,
-        extended_statistics: false
-      }
-    },
-    unit_scaling:         :best,
-    # If you/your plugin/whatever needs it your data can go here
-    assigns:              %{},
-    before_each:          nil,
-    after_each:           nil,
-    before_scenario:      nil,
-    after_scenario:       nil
-  ]
+  defstruct parallel: 1,
+            time: 5,
+            warmup: 2,
+            dry_run: false,
+            formatters: [Console],
+            print: %{
+              benchmarking: true,
+              configuration: true,
+              fast_warning: true
+            },
+            inputs: nil,
+            save: false,
+            load: false,
+            # formatters should end up here but known once are still picked up at
+            # the top level for now
+            formatter_options: %{
+              console: %{
+                comparison: true,
+                extended_statistics: false
+              }
+            },
+            unit_scaling: :best,
+            # If you/your plugin/whatever needs it your data can go here
+            assigns: %{},
+            before_each: nil,
+            after_each: nil,
+            before_scenario: nil,
+            after_scenario: nil
 
   @type t :: %__MODULE__{
-    parallel:          integer,
-    time:              number,
-    warmup:            number,
-    formatters:        [((Suite.t) -> Suite.t)],
-    print:             map,
-    inputs:            %{Suite.key => any} | nil,
-    save:              map | false,
-    load:              String.t | [String.t] | false,
-    formatter_options: map,
-    unit_scaling:      Scale.scaling_strategy,
-    assigns:           map,
-    before_each:       fun | nil,
-    after_each:        fun | nil,
-    before_scenario:   fun | nil,
-    after_scenario:    fun | nil
-  }
+          parallel: integer,
+          time: number,
+          warmup: number,
+          formatters: [(Suite.t() -> Suite.t())],
+          print: map,
+          inputs: %{Suite.key() => any} | nil,
+          save: map | false,
+          load: String.t() | [String.t()] | false,
+          formatter_options: map,
+          unit_scaling: Scale.scaling_strategy(),
+          assigns: map,
+          before_each: fun | nil,
+          after_each: fun | nil,
+          before_scenario: fun | nil,
+          after_scenario: fun | nil
+        }
 
   @type user_configuration :: map | keyword
   @time_keys [:time, :warmup]
@@ -265,22 +264,23 @@ defmodule Benchee.Configuration do
         scenarios: []
       }
   """
-  @spec init(user_configuration) :: Suite.t
+  @spec init(user_configuration) :: Suite.t()
   def init(config \\ %{}) do
-    :ok = :timer.start
+    :ok = :timer.start()
 
-    config = config
-             |> standardized_user_configuration
-             |> merge_with_defaults
-             |> convert_time_to_micro_s
-             |> save_option_conversion
+    config =
+      config
+      |> standardized_user_configuration
+      |> merge_with_defaults
+      |> convert_time_to_micro_s
+      |> save_option_conversion
 
     %Suite{configuration: config}
   end
 
   defp standardized_user_configuration(config) do
     config
-    |> DeepConvert.to_map
+    |> DeepConvert.to_map()
     |> translate_formatter_keys
     |> force_string_input_keys
   end
@@ -289,16 +289,19 @@ defmodule Benchee.Configuration do
   # formatter_options now
   @formatter_keys [:console, :csv, :json, :html]
   defp translate_formatter_keys(config) do
-      {formatter_options, config} = Map.split(config, @formatter_keys)
+    {formatter_options, config} = Map.split(config, @formatter_keys)
     DeepMerge.deep_merge(%{formatter_options: formatter_options}, config)
   end
 
   defp force_string_input_keys(config = %{inputs: inputs}) do
-    standardized_inputs = for {name, value} <- inputs, into: %{} do
-                            {to_string(name), value}
-                          end
+    standardized_inputs =
+      for {name, value} <- inputs, into: %{} do
+        {to_string(name), value}
+      end
+
     %{config | inputs: standardized_inputs}
   end
+
   defp force_string_input_keys(config), do: config
 
   defp merge_with_defaults(user_config) do
@@ -306,17 +309,20 @@ defmodule Benchee.Configuration do
   end
 
   defp convert_time_to_micro_s(config) do
-    Enum.reduce @time_keys, config, fn(key, new_config) ->
-      {_, new_config} = Map.get_and_update! new_config, key, fn(seconds) ->
-        {seconds, Duration.microseconds({seconds, :second})}
-      end
+    Enum.reduce(@time_keys, config, fn key, new_config ->
+      {_, new_config} =
+        Map.get_and_update!(new_config, key, fn seconds ->
+          {seconds, Duration.microseconds({seconds, :second})}
+        end)
+
       new_config
-    end
+    end)
   end
 
   defp save_option_conversion(config = %{save: false}) do
     config
   end
+
   defp save_option_conversion(config = %{save: save_values}) do
     save_options = Map.merge(save_defaults(), save_values)
 
@@ -325,15 +331,16 @@ defmodule Benchee.Configuration do
       path: save_options.path
     }
 
-    %__MODULE__{config |
-      formatters: config.formatters ++ [Benchee.Formatters.TaggedSave],
-      formatter_options:
-        Map.put(config.formatter_options, :tagged_save, tagged_save_options)
+    %__MODULE__{
+      config
+      | formatters: config.formatters ++ [Benchee.Formatters.TaggedSave],
+        formatter_options: Map.put(config.formatter_options, :tagged_save, tagged_save_options)
     }
   end
 
   defp save_defaults do
-    now = DateTime.utc_now
+    now = DateTime.utc_now()
+
     %{
       tag: "#{now.year}-#{now.month}-#{now.day}--#{now.hour}-#{now.minute}-#{now.second}-utc",
       path: "benchmark.benchee"
@@ -345,8 +352,9 @@ defimpl DeepMerge.Resolver, for: Benchee.Configuration do
   def resolve(_original, override = %{__struct__: Benchee.Configuration}, _) do
     override
   end
+
   def resolve(original, override, resolver) when is_map(override) do
     merged = Map.merge(original, override, resolver)
-    struct! Benchee.Configuration, Map.to_list(merged)
+    struct!(Benchee.Configuration, Map.to_list(merged))
   end
 end
diff --git a/test/benchee/benchmark/runner_test.exs b/test/benchee/benchmark/runner_test.exs
index ff9d8bcc..163b5ee0 100644
--- a/test/benchee/benchmark/runner_test.exs
+++ b/test/benchee/benchmark/runner_test.exs
@@ -5,18 +5,21 @@ defmodule Benchee.Benchmark.RunnerTest do
   alias Benchee.Benchmark.Scenario
   alias Benchee.Test.FakeBenchmarkPrinter, as: TestPrinter
 
-  @config %Configuration{parallel: 1,
-                         time:     40_000,
-                         warmup:   20_000,
-                         inputs:   nil,
-                         print:    %{fast_warning: false, configuration: true}}
+  @config %Configuration{
+    parallel: 1,
+    time: 40_000,
+    warmup: 20_000,
+    inputs: nil,
+    dry_run: false,
+    print: %{fast_warning: false, configuration: true}
+  }
   @system %{
-    elixir:           "1.4.0",
-    erlang:           "19.1",
-    num_cores:        "4",
-    os:               "Super Duper",
+    elixir: "1.4.0",
+    erlang: "19.1",
+    num_cores: "4",
+    os: "Super Duper",
     available_memory: "8 Trillion",
-    cpu_speed:        "light speed"
+    cpu_speed: "light speed"
   }
   @default_suite %Suite{configuration: @config, system: @system}
 
@@ -25,10 +28,11 @@ defmodule Benchee.Benchmark.RunnerTest do
   end
 
   defp run_times_for(suite, job_name, input_name \\ Benchmark.no_input()) do
-    filter_fun = fn(scenario) ->
+    filter_fun = fn scenario ->
       scenario.job_name == job_name && scenario.input_name == input_name
     end
-    map_fun = fn(scenario) -> scenario.run_times end
+
+    map_fun = fn scenario -> scenario.run_times end
 
     suite.scenarios
     |> Enum.filter(filter_fun)
@@ -37,8 +41,9 @@ defmodule Benchee.Benchmark.RunnerTest do
 
   describe ".run_scenarios" do
     test "runs a benchmark suite and enriches it with measurements" do
-      retrying fn ->
+      retrying(fn ->
         suite = test_suite(%Suite{configuration: %{time: 60_000, warmup: 10_000}})
+
         new_suite =
           suite
           |> Benchmark.benchmark("Name", fn -> :timer.sleep(10) end)
@@ -49,12 +54,13 @@ defmodule Benchee.Benchmark.RunnerTest do
 
         # should be 6 but gotta give it a bit leeway
         assert length(run_times) >= 5
-      end
+      end)
     end
 
     test "runs a suite with multiple jobs and gathers results" do
-      retrying fn ->
+      retrying(fn ->
         suite = test_suite(%Suite{configuration: %{time: 100_000, warmup: 10_000}})
+
         new_suite =
           suite
           |> Benchmark.benchmark("Name", fn -> :timer.sleep(19) end)
@@ -65,14 +71,16 @@ defmodule Benchee.Benchmark.RunnerTest do
         assert length(run_times_for(new_suite, "Name")) >= 4
         # should be ~11, but gotta give it some leeway
         assert length(run_times_for(new_suite, "Name 2")) >= 8
-      end
+      end)
     end
 
     test "can run multiple benchmarks in parallel" do
       suite = test_suite(%Suite{configuration: %{parallel: 4, time: 60_000}})
-      new_suite = suite
-                  |> Benchmark.benchmark("", fn -> :timer.sleep 10 end)
-                  |> Benchmark.measure(TestPrinter)
+
+      new_suite =
+        suite
+        |> Benchmark.benchmark("", fn -> :timer.sleep(10) end)
+        |> Benchmark.measure(TestPrinter)
 
       # it does more work when working in parallel than it does alone
       assert length(run_times_for(new_suite, "")) >= 12
@@ -90,12 +98,13 @@ defmodule Benchee.Benchmark.RunnerTest do
     end
 
     test "very fast functions print a warning" do
-      output = ExUnit.CaptureIO.capture_io fn ->
-        %Suite{configuration: %{print: %{fast_warning: true}}}
-        |> test_suite()
-        |> Benchmark.benchmark("", fn -> 1 end)
-        |> Benchmark.measure()
-      end
+      output =
+        ExUnit.CaptureIO.capture_io(fn ->
+          %Suite{configuration: %{print: %{fast_warning: true}}}
+          |> test_suite()
+          |> Benchmark.benchmark("", fn -> 1 end)
+          |> Benchmark.measure()
+        end)
 
       # need to asser on IO here as our message sending trick doesn't work
       # as we spawn new processes to do our benchmarking work therfore the
@@ -104,10 +113,11 @@ defmodule Benchee.Benchmark.RunnerTest do
     end
 
     test "very fast function times are reported correctly" do
-      suite = test_suite()
-              |> Benchmark.benchmark("", fn -> 1 end)
-              |> Benchmark.measure(TestPrinter)
-              |> Benchee.statistics()
+      suite =
+        test_suite()
+        |> Benchmark.benchmark("", fn -> 1 end)
+        |> Benchmark.measure(TestPrinter)
+        |> Benchee.statistics()
 
       [%{run_time_statistics: %{average: average}}] = suite.scenarios
 
@@ -116,42 +126,48 @@ defmodule Benchee.Benchmark.RunnerTest do
     end
 
     test "doesn't take longer than advertised for very fast funs" do
-      retrying fn ->
+      retrying(fn ->
         time = 20_000
         warmup = 10_000
         projected = time + warmup
 
-        suite = %Suite{configuration: %{time: time, warmup: warmup}}
-                |> test_suite()
-                |> Benchmark.benchmark("", fn -> :timer.sleep(1) end)
+        suite =
+          %Suite{configuration: %{time: time, warmup: warmup}}
+          |> test_suite()
+          |> Benchmark.benchmark("", fn -> :timer.sleep(1) end)
 
-        {time, _} = :timer.tc fn -> Benchmark.measure(suite, TestPrinter) end
+        {time, _} = :timer.tc(fn -> Benchmark.measure(suite, TestPrinter) end)
 
         # if the system is too busy there are too many false positives
         leeway = projected * 0.4
-        assert_in_delta projected, time, leeway,
+
+        assert_in_delta projected,
+                        time,
+                        leeway,
                         "excution took too long #{time} vs. #{projected} +- #{leeway}"
-      end
+      end)
     end
 
     test "variance does not skyrocket on very fast functions" do
-      retrying fn ->
+      retrying(fn ->
         range = 0..10
-        suite = %Suite{configuration: %{time: 150_000, warmup: 20_000}}
-                |> test_suite
-                |> Benchmark.benchmark("noop", fn -> 1 + 1 end)
-                |> Benchmark.benchmark("map", fn ->
-                     Enum.map(range, fn(i) -> i end)
-                   end)
-                |> Benchmark.measure(TestPrinter)
-                |> Statistics.statistics
-
-        stats = Enum.map(suite.scenarios, fn(scenario) -> scenario.run_time_statistics end)
-
-        Enum.each(stats, fn(%Statistics{std_dev_ratio: std_dev_ratio}) ->
+
+        suite =
+          %Suite{configuration: %{time: 150_000, warmup: 20_000}}
+          |> test_suite
+          |> Benchmark.benchmark("noop", fn -> 1 + 1 end)
+          |> Benchmark.benchmark("map", fn ->
+            Enum.map(range, fn i -> i end)
+          end)
+          |> Benchmark.measure(TestPrinter)
+          |> Statistics.statistics()
+
+        stats = Enum.map(suite.scenarios, fn scenario -> scenario.run_time_statistics end)
+
+        Enum.each(stats, fn %Statistics{std_dev_ratio: std_dev_ratio} ->
           assert std_dev_ratio <= 2.5
         end)
-      end
+      end)
     end
 
     test "never calls the function if warmup and time are 0" do
@@ -166,7 +182,7 @@ defmodule Benchee.Benchmark.RunnerTest do
     end
 
     test "run times of the scenarios are empty when nothing runs" do
-      %{scenarios: [scenario]} = 
+      %{scenarios: [scenario]} =
         %Suite{configuration: %{time: 0, warmup: 0}}
         |> test_suite
         |> Benchmark.benchmark("don't care", fn -> 0 end)
@@ -178,7 +194,7 @@ defmodule Benchee.Benchmark.RunnerTest do
     @no_input Benchmark.no_input()
     test "asks to print what is currently benchmarking" do
       test_suite()
-      |> Benchmark.benchmark("Something", fn -> :timer.sleep 10 end)
+      |> Benchmark.benchmark("Something", fn -> :timer.sleep(10) end)
       |> Benchmark.measure(TestPrinter)
 
       assert_receive {:benchmarking, "Something", @no_input}
@@ -191,47 +207,47 @@ defmodule Benchee.Benchmark.RunnerTest do
 
       %Suite{configuration: %{inputs: @inputs}}
       |> test_suite
-      |> Benchmark.benchmark("one", fn(input) -> send ref, {:one, input} end)
-      |> Benchmark.benchmark("two", fn(input) -> send ref, {:two, input} end)
+      |> Benchmark.benchmark("one", fn input -> send(ref, {:one, input}) end)
+      |> Benchmark.benchmark("two", fn input -> send(ref, {:two, input}) end)
       |> Benchmark.measure(TestPrinter)
 
-      Enum.each @inputs, fn({_name, value}) ->
+      Enum.each(@inputs, fn {_name, value} ->
         assert_receive {:one, ^value}
         assert_receive {:two, ^value}
-      end
+      end)
     end
 
     test "notifies which input is being benchmarked now" do
       %Suite{configuration: %{inputs: @inputs}}
       |> test_suite
-      |> Benchmark.benchmark("one", fn(_) -> nil end)
+      |> Benchmark.benchmark("one", fn _ -> nil end)
       |> Benchmark.measure(TestPrinter)
 
-      Enum.each @inputs, fn({name, _value}) ->
+      Enum.each(@inputs, fn {name, _value} ->
         assert_received {:benchmarking, "one", ^name}
-      end
+      end)
     end
 
     test "populates results for all inputs" do
-      retrying fn ->
+      retrying(fn ->
         inputs = %{
-          "Short wait"  => 9,
+          "Short wait" => 9,
           "Longer wait" => 19
         }
-        config = %{time: 100_000,
-                   warmup: 10_000,
-                   inputs: inputs}
+
+        config = %{time: 100_000, warmup: 10_000, inputs: inputs}
+
         new_suite =
           %Suite{configuration: config}
           |> test_suite
-          |> Benchmark.benchmark("sleep", fn(input) -> :timer.sleep(input) end)
+          |> Benchmark.benchmark("sleep", fn input -> :timer.sleep(input) end)
           |> Benchmark.measure(TestPrinter)
 
         # should be ~11 but the good old leeway
         assert length(run_times_for(new_suite, "sleep", "Short wait")) >= 8
         # should be 5 but the good old leeway
         assert length(run_times_for(new_suite, "sleep", "Longer wait")) >= 4
-      end
+      end)
     end
 
     test "runs the job exactly once if its time exceeds given time" do
@@ -245,14 +261,16 @@ defmodule Benchee.Benchmark.RunnerTest do
     end
 
     test "stores run times in the right order" do
-      retrying fn ->
-        {:ok, agent} = Agent.start fn -> 10 end
+      retrying(fn ->
+        {:ok, agent} = Agent.start(fn -> 10 end)
+
         increasing_function = fn ->
-          Agent.update agent, fn(state) ->
-            :timer.sleep state
+          Agent.update(agent, fn state ->
+            :timer.sleep(state)
             state + 30
-          end
+          end)
         end
+
         run_times =
           %Suite{configuration: %{time: 70_000, warmup: 0}}
           |> test_suite
@@ -260,11 +278,12 @@ defmodule Benchee.Benchmark.RunnerTest do
           |> Benchmark.measure(TestPrinter)
           |> run_times_for("Sleep more")
 
-        assert length(run_times) >= 2 # should be 3 but good old leeway
+        # should be 3 but good old leeway
+        assert length(run_times) >= 2
         # as the function takes more time each time called run times should be
         # as if sorted ascending
         assert run_times == Enum.sort(run_times)
-      end
+      end)
     end
 
     # important for when we load scenarios but want them to run again and not
@@ -282,9 +301,10 @@ defmodule Benchee.Benchmark.RunnerTest do
         ]
       }
 
-      %Suite{scenarios: [scenario]} = suite
-                                 |> test_suite
-                                 |> Benchmark.measure(TestPrinter)
+      %Suite{scenarios: [scenario]} =
+        suite
+        |> test_suite
+        |> Benchmark.measure(TestPrinter)
 
       # our previous run time isn't there anymore
       refute Enum.member?(scenario.run_times, 1_000_000)
@@ -292,23 +312,28 @@ defmodule Benchee.Benchmark.RunnerTest do
 
     test "global hooks triggers" do
       me = self()
+
       %Suite{
         configuration: %{
           warmup: 0,
           time: 100,
-          before_each: fn(input) -> send(me, :before); input end,
-          after_each: fn(_) -> send(me, :after) end
+          before_each: fn input ->
+            send(me, :before)
+            input
+          end,
+          after_each: fn _ -> send(me, :after) end
         }
       }
       |> test_suite
-      |> Benchmark.benchmark("job", fn -> :timer.sleep 1 end)
+      |> Benchmark.benchmark("job", fn -> :timer.sleep(1) end)
       |> Benchmark.measure(TestPrinter)
 
-      assert_received_exactly [:before, :after]
+      assert_received_exactly([:before, :after])
     end
 
     test "scenario hooks triggers" do
       me = self()
+
       %Suite{
         configuration: %{
           warmup: 0,
@@ -316,21 +341,33 @@ defmodule Benchee.Benchmark.RunnerTest do
         }
       }
       |> test_suite
-      |> Benchmark.benchmark("job", {
-           fn -> :timer.sleep 1 end,
-           before_each: fn(input) -> send(me, :before); input end,
-           after_each: fn(_) -> send(me, :after) end,
-           before_scenario: fn(input) -> send(me, :before_scenario); input end,
-           after_scenario: fn(_) -> send(me, :after_scenario) end})
+      |> Benchmark.benchmark(
+        "job",
+        {fn -> :timer.sleep(1) end,
+         before_each: fn input ->
+           send(me, :before)
+           input
+         end,
+         after_each: fn _ -> send(me, :after) end,
+         before_scenario: fn input ->
+           send(me, :before_scenario)
+           input
+         end,
+         after_scenario: fn _ -> send(me, :after_scenario) end}
+      )
       |> Benchmark.measure(TestPrinter)
 
-      assert_received_exactly [
-        :before_scenario, :before, :after, :after_scenario
-      ]
+      assert_received_exactly([
+        :before_scenario,
+        :before,
+        :after,
+        :after_scenario
+      ])
     end
 
     test "hooks trigger during warmup and runtime but scenarios once" do
       me = self()
+
       %Suite{
         configuration: %{
           warmup: 100,
@@ -338,172 +375,248 @@ defmodule Benchee.Benchmark.RunnerTest do
         }
       }
       |> test_suite
-      |> Benchmark.benchmark("job", {
-           fn -> :timer.sleep 1 end,
-           before_each: fn(input) -> send(me, :before); input end,
-           after_each: fn(_) -> send(me, :after) end,
-           before_scenario: fn(input) -> send(me, :before_scenario); input end,
-           after_scenario: fn(_) -> send(me, :after_scenario) end})
+      |> Benchmark.benchmark(
+        "job",
+        {fn -> :timer.sleep(1) end,
+         before_each: fn input ->
+           send(me, :before)
+           input
+         end,
+         after_each: fn _ -> send(me, :after) end,
+         before_scenario: fn input ->
+           send(me, :before_scenario)
+           input
+         end,
+         after_scenario: fn _ -> send(me, :after_scenario) end}
+      )
       |> Benchmark.measure(TestPrinter)
 
-      assert_received_exactly [
-        :before_scenario, :before, :after, :before, :after, :after_scenario
-      ]
+      assert_received_exactly([
+        :before_scenario,
+        :before,
+        :after,
+        :before,
+        :after,
+        :after_scenario
+      ])
     end
 
     test "hooks trigger for each input" do
       me = self()
+
       %Suite{
         configuration: %{
           warmup: 0,
           time: 100,
-          before_each: fn(input) -> send(me, :global_before); input end,
-          after_each:  fn(_) -> send me, :global_after end,
-          before_scenario: fn(input) ->
+          before_each: fn input ->
+            send(me, :global_before)
+            input
+          end,
+          after_each: fn _ -> send(me, :global_after) end,
+          before_scenario: fn input ->
             send(me, :global_before_scenario)
             input
           end,
-          after_scenario:  fn(_) -> send me, :global_after_scenario end,
+          after_scenario: fn _ -> send(me, :global_after_scenario) end,
           inputs: %{"one" => 1, "two" => 2}
         }
       }
       |> test_suite
-      |> Benchmark.benchmark("job", {
-           fn(_) -> :timer.sleep 1 end,
-           before_each: fn(input) -> send(me, :local_before); input end,
-           after_each: fn(_) -> send(me, :local_after) end,
-           before_scenario: fn(input) ->
-             send(me, :local_before_scenario)
-             input
-           end,
-           after_scenario: fn(_) -> send(me, :local_after_scenario) end})
+      |> Benchmark.benchmark(
+        "job",
+        {fn _ -> :timer.sleep(1) end,
+         before_each: fn input ->
+           send(me, :local_before)
+           input
+         end,
+         after_each: fn _ -> send(me, :local_after) end,
+         before_scenario: fn input ->
+           send(me, :local_before_scenario)
+           input
+         end,
+         after_scenario: fn _ -> send(me, :local_after_scenario) end}
+      )
       |> Benchmark.measure(TestPrinter)
 
-      assert_received_exactly [
-        :global_before_scenario, :local_before_scenario, :global_before, :local_before, :local_after, :global_after, :local_after_scenario,
-        :global_after_scenario,
-        :global_before_scenario, :local_before_scenario, :global_before, :local_before, :local_after, :global_after, :local_after_scenario,
+      assert_received_exactly([
+        :global_before_scenario,
+        :local_before_scenario,
+        :global_before,
+        :local_before,
+        :local_after,
+        :global_after,
+        :local_after_scenario,
         :global_after_scenario,
-      ]
+        :global_before_scenario,
+        :local_before_scenario,
+        :global_before,
+        :local_before,
+        :local_after,
+        :global_after,
+        :local_after_scenario,
+        :global_after_scenario
+      ])
     end
 
     test "scenario hooks trigger only for that scenario" do
       me = self()
+
       %Suite{
         configuration: %{
           warmup: 0,
           time: 100,
-          before_each: fn(input) -> send(me, :global_before); input end,
-          after_each:  fn(_) -> send me, :global_after end,
-          after_scenario: fn(_) -> send me, :global_after_scenario end
+          before_each: fn input ->
+            send(me, :global_before)
+            input
+          end,
+          after_each: fn _ -> send(me, :global_after) end,
+          after_scenario: fn _ -> send(me, :global_after_scenario) end
         }
       }
       |> test_suite
-      |> Benchmark.benchmark("job", {
-           fn -> :timer.sleep 1 end,
-           before_each: fn(input) -> send(me, :local_1_before); input end,
-           before_scenario: fn(input) ->
-             send me, :local_scenario_before
-             input
-           end})
-      |> Benchmark.benchmark("job 2", fn -> :timer.sleep 1 end)
+      |> Benchmark.benchmark(
+        "job",
+        {fn -> :timer.sleep(1) end,
+         before_each: fn input ->
+           send(me, :local_1_before)
+           input
+         end,
+         before_scenario: fn input ->
+           send(me, :local_scenario_before)
+           input
+         end}
+      )
+      |> Benchmark.benchmark("job 2", fn -> :timer.sleep(1) end)
       |> Benchmark.measure(TestPrinter)
 
-      assert_received_exactly [
-        :global_before, :local_scenario_before, :local_1_before, :global_after,
+      assert_received_exactly([
+        :global_before,
+        :local_scenario_before,
+        :local_1_before,
+        :global_after,
         :global_after_scenario,
-        :global_before, :global_after, :global_after_scenario
-      ]
+        :global_before,
+        :global_after,
+        :global_after_scenario
+      ])
     end
 
     test "different hooks trigger only for that scenario" do
       me = self()
+
       %Suite{
         configuration: %{
           warmup: 0,
           time: 100,
-          before_each: fn(input) -> send(me, :global_before); input end,
-          after_each:  fn(_) -> send me, :global_after end
+          before_each: fn input ->
+            send(me, :global_before)
+            input
+          end,
+          after_each: fn _ -> send(me, :global_after) end
         }
       }
       |> test_suite
-      |> Benchmark.benchmark("job", {
-           fn -> :timer.sleep 1 end,
-           before_each: fn(input) -> send(me, :local_before); input end,
-           after_each:  fn(_) -> send me, :local_after end,
-           before_scenario: fn(input) ->
-             send me, :local_before_scenario
-             input
-           end})
-      |> Benchmark.benchmark("job 2", {
-           fn -> :timer.sleep 1 end,
-           before_each: fn(input) -> send(me, :local_2_before); input end,
-           after_each:  fn(_) -> send me, :local_2_after end,
-           after_scenario: fn(_) -> send me, :local_2_after_scenario end})
+      |> Benchmark.benchmark(
+        "job",
+        {fn -> :timer.sleep(1) end,
+         before_each: fn input ->
+           send(me, :local_before)
+           input
+         end,
+         after_each: fn _ -> send(me, :local_after) end,
+         before_scenario: fn input ->
+           send(me, :local_before_scenario)
+           input
+         end}
+      )
+      |> Benchmark.benchmark(
+        "job 2",
+        {fn -> :timer.sleep(1) end,
+         before_each: fn input ->
+           send(me, :local_2_before)
+           input
+         end,
+         after_each: fn _ -> send(me, :local_2_after) end,
+         after_scenario: fn _ -> send(me, :local_2_after_scenario) end}
+      )
       |> Benchmark.measure(TestPrinter)
 
-      assert_received_exactly [
-        :local_before_scenario, :global_before, :local_before, :local_after,
+      assert_received_exactly([
+        :local_before_scenario,
+        :global_before,
+        :local_before,
+        :local_after,
+        :global_after,
+        :global_before,
+        :local_2_before,
+        :local_2_after,
         :global_after,
-        :global_before, :local_2_before, :local_2_after, :global_after,
         :local_2_after_scenario
-      ]
+      ])
     end
 
     test "each triggers for every invocation, scenario once" do
       me = self()
+
       suite = %Suite{
-                configuration: %{
-                  warmup: 0,
-                  time: 10_000,
-                  before_each: fn(input) -> send(me, :global_before); input end,
-                  after_each:  fn(_) -> send me, :global_after end,
-                  before_scenario: fn(input) ->
-                    send me, :global_before_scenario
-                    input
-                  end,
-                  after_scenario: fn(_) -> send me, :global_after_scenario end,
-                }
-              }
+        configuration: %{
+          warmup: 0,
+          time: 10_000,
+          before_each: fn input ->
+            send(me, :global_before)
+            input
+          end,
+          after_each: fn _ -> send(me, :global_after) end,
+          before_scenario: fn input ->
+            send(me, :global_before_scenario)
+            input
+          end,
+          after_scenario: fn _ -> send(me, :global_after_scenario) end
+        }
+      }
+
       result =
         suite
         |> test_suite
-        |> Benchmark.benchmark("job", {
-             fn -> :timer.sleep 1 end,
-             before_each: fn(input) -> send(me, :local_before); input end,
-             after_each:  fn(_) -> send me, :local_after end,
-             before_scenario: fn(input) ->
-               send(me, :local_before_scenario)
-               input
-             end,
-             after_scenario: fn(_) -> send(me, :local_after_scenario) end})
+        |> Benchmark.benchmark(
+          "job",
+          {fn -> :timer.sleep(1) end,
+           before_each: fn input ->
+             send(me, :local_before)
+             input
+           end,
+           after_each: fn _ -> send(me, :local_after) end,
+           before_scenario: fn input ->
+             send(me, :local_before_scenario)
+             input
+           end,
+           after_scenario: fn _ -> send(me, :local_after_scenario) end}
+        )
         |> Benchmark.measure(TestPrinter)
 
-      {:messages, messages} = Process.info self(), :messages
+      {:messages, messages} = Process.info(self(), :messages)
 
       global_before_sceneario_count =
-        Enum.count messages, fn(msg) -> msg == :global_before_scenario end
+        Enum.count(messages, fn msg -> msg == :global_before_scenario end)
+
       local_before_sceneario_count =
-        Enum.count messages, fn(msg) -> msg == :local_before_scenario end
+        Enum.count(messages, fn msg -> msg == :local_before_scenario end)
+
       local_after_sceneario_count =
-        Enum.count messages, fn(msg) -> msg == :local_after_scenario end
+        Enum.count(messages, fn msg -> msg == :local_after_scenario end)
+
       global_after_sceneario_count =
-        Enum.count messages, fn(msg) -> msg == :global_after_scenario end
+        Enum.count(messages, fn msg -> msg == :global_after_scenario end)
 
       assert global_before_sceneario_count == 1
-      assert local_before_sceneario_count  == 1
-      assert local_after_sceneario_count   == 1
-      assert global_after_sceneario_count  == 1
-
-      global_before_count =
-        Enum.count messages, fn(message) -> message == :global_before end
-      local_before_count =
-        Enum.count messages, fn(message) -> message == :local_before end
-      local_after_count =
-        Enum.count messages, fn(message) -> message == :local_after end
-      global_after_count =
-        Enum.count messages, fn(message) -> message == :global_after end
+      assert local_before_sceneario_count == 1
+      assert local_after_sceneario_count == 1
+      assert global_after_sceneario_count == 1
 
+      global_before_count = Enum.count(messages, fn message -> message == :global_before end)
+      local_before_count = Enum.count(messages, fn message -> message == :local_before end)
+      local_after_count = Enum.count(messages, fn message -> message == :local_after end)
+      global_after_count = Enum.count(messages, fn message -> message == :global_after end)
 
       assert local_before_count == global_before_count
       assert local_after_count == global_after_count
@@ -520,32 +633,38 @@ defmodule Benchee.Benchmark.RunnerTest do
 
     test "hooks also trigger for very fast invocations" do
       me = self()
+
       suite = %Suite{
-                configuration: %{
-                  warmup: 1,
-                  time: 1_000,
-                  before_each: fn(input) -> send(me, :global_before); input end,
-                  after_each:  fn(_) -> send me, :global_after end
-                }
-              }
+        configuration: %{
+          warmup: 1,
+          time: 1_000,
+          before_each: fn input ->
+            send(me, :global_before)
+            input
+          end,
+          after_each: fn _ -> send(me, :global_after) end
+        }
+      }
+
       result =
         suite
         |> test_suite
-        |> Benchmark.benchmark("job", {fn -> 0 end,
-             before_each: fn(input) -> send(me, :local_before); input end,
-             after_each:  fn(_) -> send me, :local_after end})
+        |> Benchmark.benchmark(
+          "job",
+          {fn -> 0 end,
+           before_each: fn input ->
+             send(me, :local_before)
+             input
+           end,
+           after_each: fn _ -> send(me, :local_after) end}
+        )
         |> Benchmark.measure(TestPrinter)
 
-      {:messages, messages} = Process.info self(), :messages
-      global_before_count =
-        Enum.count messages, fn(message) -> message == :global_before end
-      local_before_count =
-        Enum.count messages, fn(message) -> message == :local_before end
-      local_after_count =
-        Enum.count messages, fn(message) -> message == :local_after end
-      global_after_count =
-        Enum.count messages, fn(message) -> message == :global_after end
-
+      {:messages, messages} = Process.info(self(), :messages)
+      global_before_count = Enum.count(messages, fn message -> message == :global_before end)
+      local_before_count = Enum.count(messages, fn message -> message == :local_before end)
+      local_after_count = Enum.count(messages, fn message -> message == :local_after end)
+      global_after_count = Enum.count(messages, fn message -> message == :global_after end)
 
       assert local_before_count == global_before_count
       assert local_after_count == global_after_count
@@ -565,42 +684,46 @@ defmodule Benchee.Benchmark.RunnerTest do
 
     test "after_each hooks have access to the return value of the invocation" do
       me = self()
+
       %Suite{
         configuration: %{
           warmup: 100,
           time: 100,
-          after_each: fn(out) -> send(me, {:global, out}) end
+          after_each: fn out -> send(me, {:global, out}) end
         }
       }
       |> test_suite
       |> Benchmark.benchmark("job", {fn ->
-           # still keep to make sure we only get one iteration and not too fast
-           :timer.sleep 1
-           :value
-         end, after_each: fn(out) -> send(me, {:local, out}) end})
+         # still keep to make sure we only get one iteration and not too fast
+         :timer.sleep(1)
+         :value
+       end, after_each: fn out -> send(me, {:local, out}) end})
       |> Benchmark.measure(TestPrinter)
 
-      assert_received_exactly [
-        {:global, :value}, {:local, :value},
-        {:global, :value}, {:local, :value}
-      ]
+      assert_received_exactly([
+        {:global, :value},
+        {:local, :value},
+        {:global, :value},
+        {:local, :value}
+      ])
     end
 
     test "after_each hooks with super fast functions" do
       me = self()
+
       %Suite{
         configuration: %{
           warmup: 100,
           time: 100,
-          after_each: fn(out) -> send(me, {:global, out}) end
+          after_each: fn out -> send(me, {:global, out}) end
         }
       }
       |> test_suite
       |> Benchmark.benchmark("job", {fn ->
-           # still keep to make sure we only get one iteration and not too fast
-           :timer.sleep 1
-           :value
-         end, after_each: fn(out) -> send(me, {:local, out}) end})
+         # still keep to make sure we only get one iteration and not too fast
+         :timer.sleep(1)
+         :value
+       end, after_each: fn out -> send(me, {:local, out}) end})
       |> Benchmark.measure(TestPrinter)
 
       assert_received {:global, :value}
@@ -611,97 +734,124 @@ defmodule Benchee.Benchmark.RunnerTest do
 
     test "hooks dealing with inputs can adapt it and pass it on" do
       me = self()
+
       %Suite{
         configuration: %{
           warmup: 1,
           time: 1,
-          before_scenario: fn(input) ->
+          before_scenario: fn input ->
             send(me, {:global_scenario, input})
             input + 1
           end,
-          before_each: fn(input) ->
+          before_each: fn input ->
             send(me, {:global_each, input})
             input + 1
           end,
-          after_scenario: fn(input) ->
+          after_scenario: fn input ->
             send(me, {:global_after_scenario, input})
           end,
           inputs: %{"basic input" => 0}
         }
       }
       |> test_suite
-      |> Benchmark.benchmark("job", {fn (input)->
-           # still keep to make sure we only get one iteration and not too fast
-           :timer.sleep 1
-           send(me, {:runner, input})
-         end,
-         before_scenario: fn(input) ->
-           send(me, {:local_scenario, input})
-           input + 1
-         end,
-         before_each: fn(input) ->
-           send(me, {:local_each, input})
-           input + 1
-         end,
-         after_scenario: fn(input) ->
-           send(me, {:local_after_scenario, input})
-         end})
+      |> Benchmark.benchmark("job", {fn input ->
+         # still keep to make sure we only get one iteration and not too fast
+         :timer.sleep(1)
+         send(me, {:runner, input})
+       end,
+       before_scenario: fn input ->
+         send(me, {:local_scenario, input})
+         input + 1
+       end,
+       before_each: fn input ->
+         send(me, {:local_each, input})
+         input + 1
+       end,
+       after_scenario: fn input ->
+         send(me, {:local_after_scenario, input})
+       end})
       |> Benchmark.measure(TestPrinter)
 
-      assert_received_exactly [
-        {:global_scenario, 0}, {:local_scenario, 1},
-        {:global_each, 2}, {:local_each, 3}, {:runner, 4},
-        {:global_each, 2}, {:local_each, 3}, {:runner, 4},
-        {:local_after_scenario, 2}, {:global_after_scenario, 2}
-      ]
+      assert_received_exactly([
+        {:global_scenario, 0},
+        {:local_scenario, 1},
+        {:global_each, 2},
+        {:local_each, 3},
+        {:runner, 4},
+        {:global_each, 2},
+        {:local_each, 3},
+        {:runner, 4},
+        {:local_after_scenario, 2},
+        {:global_after_scenario, 2}
+      ])
     end
 
     test "hooks dealing with inputs still work when there is no input given" do
       me = self()
+
       %Suite{
         configuration: %{
           warmup: 1,
           time: 1,
-          before_scenario: fn(input) ->
+          before_scenario: fn input ->
             send(me, {:global_scenario, input})
             input
           end,
-          before_each: fn(input) ->
+          before_each: fn input ->
             send(me, {:global_each, input})
             input
           end,
-          after_scenario: fn(input) ->
+          after_scenario: fn input ->
             send(me, {:global_after_scenario, input})
           end
         }
       }
       |> test_suite
       |> Benchmark.benchmark("job", {fn ->
-           # still keep to make sure we only get one iteration and not too fast
-           :timer.sleep 1
-         end,
-         before_scenario: fn(input) ->
-           send(me, {:local_scenario, input})
-           input
-         end,
-         before_each: fn(input) ->
-           send(me, {:local_each, input})
-           input
-         end,
-         after_scenario: fn(input) ->
-           send(me, {:local_after_scenario, input})
-         end})
+         # still keep to make sure we only get one iteration and not too fast
+         :timer.sleep(1)
+       end,
+       before_scenario: fn input ->
+         send(me, {:local_scenario, input})
+         input
+       end,
+       before_each: fn input ->
+         send(me, {:local_each, input})
+         input
+       end,
+       after_scenario: fn input ->
+         send(me, {:local_after_scenario, input})
+       end})
       |> Benchmark.measure(TestPrinter)
 
       no_input = Benchmark.no_input()
 
-      assert_received_exactly [
-        {:global_scenario, no_input}, {:local_scenario, no_input},
-        {:global_each, no_input}, {:local_each, no_input},
-        {:global_each, no_input}, {:local_each, no_input},
-        {:local_after_scenario, no_input}, {:global_after_scenario, no_input}
-      ]
+      assert_received_exactly([
+        {:global_scenario, no_input},
+        {:local_scenario, no_input},
+        {:global_each, no_input},
+        {:local_each, no_input},
+        {:global_each, no_input},
+        {:local_each, no_input},
+        {:local_after_scenario, no_input},
+        {:global_after_scenario, no_input}
+      ])
     end
 
+    test "runs all benchmarks with all inputs exactly once as a dry run" do
+      ref = self()
+
+      inputs = %{"small" => 1, "big" => 100}
+
+      config = %{time: 0, warmup: 0, inputs: inputs, dry_run: true}
+
+      %Suite{configuration: config}
+      |> test_suite
+      |> Benchmark.benchmark("first", fn input -> send(ref, {:first, input}) end)
+      |> Benchmark.benchmark("second", fn input -> send(ref, {:second, input}) end)
+      |> Benchmark.measure(TestPrinter)
+
+      assert_received_exactly([{:first, 100}, {:first, 1}, {:second, 100}, {:second, 1}])
+    end
   end
 end

From b81d76f6a45f5ae03b9337d2f1dd3222a69b1900 Mon Sep 17 00:00:00 2001
From: Devon Estes <devon.c.estes@gmail.com>
Date: Mon, 29 Jan 2018 14:40:47 +0100
Subject: [PATCH 2/5] Update changelog and documentation

---
 CHANGELOG.md | 10 ++++++++++
 README.md    |  1 +
 2 files changed, 11 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3497cc86..647825c9 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,13 @@
+## 0.13.0 (2018-??-??)
+
+Adds the ability to run a `dry_run` of your benchmarks if you want to make sure
+everything will run without error before running the full set of benchmarks.
+
+### Features (User Facing)
+* new `dry_run` configuration option which allows users to add a dry run of all
+  benchmarks with each input before running the actual suite. This should save
+time while actually writing the code for your benchmarks.
+
 ## 0.12.0 (2018-01-20)
 
 Adds the ability to save benchmarking results and load them again to compare
diff --git a/README.md b/README.md
index 76f82093..13ecf975 100644
--- a/README.md
+++ b/README.md
@@ -143,6 +143,7 @@ The available options are the following (also documented in [hexdocs](https://he
 
 * `warmup` - the time in seconds for which a benchmarking job should be run without measuring times before real measurements start. This simulates a _"warm"_ running system. Defaults to 2.
 * `time` - the time in seconds for how long each individual benchmarking job should be run and measured. Defaults to 5.
+* `dry_run` - whether or not to run each job with each input to ensure that your code executes without error. This can save time while developing your suites.
 * `inputs` - a map from descriptive input names to some different input, your benchmarking jobs will then be run with each of these inputs. For this to work your benchmarking function gets the current input passed in as an argument into the function. Defaults to `nil`, aka no input specified and functions are called without an argument. See [Inputs](#inputs).
 * `parallel` - the function of each benchmarking job will be executed in `parallel` number processes. If `parallel: 4` then 4 processes will be spawned that all execute the _same_ function for the given time. When these finish/the time is up 4 new processes will be spawned for the next job/function. This gives you more data in the same time, but also puts a load on the system interfering with benchmark results. For more on the pros and cons of parallel benchmarking [check the wiki](https://github.com/PragTob/benchee/wiki/Parallel-Benchmarking). Defaults to 1 (no parallel execution).
 * `formatters` - list of formatters either as module implementing the formatter behaviour or formatter functions. They are run when using `Benchee.run/2`. Functions need to accept one argument (which is the benchmarking suite with all data) and then use that to produce output. Used for plugins. Defaults to the builtin console formatter `Benchee.Formatters.Console`. See [Formatters](#formatters).

From aefe802730cdd3c3ea134c45e34f9a3f817c8440 Mon Sep 17 00:00:00 2001
From: Devon Estes <devon.c.estes@gmail.com>
Date: Mon, 29 Jan 2018 15:00:21 +0100
Subject: [PATCH 3/5] Add config for Credo

Now that the official Elixir formatter has a line length of 96, we need
to make sure Credo doesn't complain about that.
---
 .credo.exs | 146 +++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 146 insertions(+)
 create mode 100644 .credo.exs

diff --git a/.credo.exs b/.credo.exs
new file mode 100644
index 00000000..9c0846e2
--- /dev/null
+++ b/.credo.exs
@@ -0,0 +1,146 @@
+# This file contains the configuration for Credo and you are probably reading
+# this after creating it with `mix credo.gen.config`.
+#
+# If you find anything wrong or unclear in this file, please report an
+# issue on GitHub: https://github.com/rrrene/credo/issues
+#
+%{
+  #
+  # You can have as many configs as you like in the `configs:` field.
+  configs: [
+    %{
+      #
+      # Run any exec using `mix credo -C <name>`. If no exec name is given
+      # "default" is used.
+      #
+      name: "default",
+      #
+      # These are the files included in the analysis:
+      files: %{
+        #
+        # You can give explicit globs or simply directories.
+        # In the latter case `**/*.{ex,exs}` will be used.
+      #
+        included: ["lib/", "src/", "web/", "apps/"],
+        excluded: [~r"/_build/", ~r"/deps/"]
+      },
+      #
+      # If you create your own checks, you must specify the source files for
+      # them here, so they can be loaded by Credo before running the analysis.
+      #
+      requires: [],
+      #
+      # If you want to enforce a style guide and need a more traditional linting
+      # experience, you can change `strict` to `true` below:
+      #
+      strict: true,
+      #
+      # If you want to use uncolored output by default, you can change `color`
+      # to `false` below:
+      #
+      color: true,
+      #
+      # You can customize the parameters of any check by adding a second element
+      # to the tuple.
+      #
+      # To disable a check put `false` as second element:
+      #
+      #     {Credo.Check.Design.DuplicatedCode, false}
+      #
+      checks: [
+        {Credo.Check.Consistency.ExceptionNames},
+        {Credo.Check.Consistency.LineEndings},
+        {Credo.Check.Consistency.ParameterPatternMatching},
+        {Credo.Check.Consistency.SpaceAroundOperators},
+        {Credo.Check.Consistency.SpaceInParentheses},
+        {Credo.Check.Consistency.TabsOrSpaces},
+
+        # You can customize the priority of any check
+        # Priority values are: `low, normal, high, higher`
+        #
+        {Credo.Check.Design.AliasUsage, priority: :low},
+
+        # For some checks, you can also set other parameters
+        #
+        # If you don't want the `setup` and `test` macro calls in ExUnit tests
+        # or the `schema` macro in Ecto schemas to trigger DuplicatedCode, just
+        # set the `excluded_macros` parameter to `[:schema, :setup, :test]`.
+        #
+        {Credo.Check.Design.DuplicatedCode, excluded_macros: []},
+
+        # You can also customize the exit_status of each check.
+        # If you don't want TODO comments to cause `mix credo` to fail, just
+        # set this value to 0 (zero).
+        #
+        {Credo.Check.Design.TagTODO, exit_status: 2},
+        {Credo.Check.Design.TagFIXME},
+
+        {Credo.Check.Readability.FunctionNames},
+        {Credo.Check.Readability.LargeNumbers},
+        {Credo.Check.Readability.MaxLineLength, priority: :low, max_length: 96},
+        {Credo.Check.Readability.ModuleAttributeNames},
+        {Credo.Check.Readability.ModuleDoc},
+        {Credo.Check.Readability.ModuleNames},
+        {Credo.Check.Readability.ParenthesesOnZeroArityDefs},
+        {Credo.Check.Readability.ParenthesesInCondition},
+        {Credo.Check.Readability.PredicateFunctionNames},
+        {Credo.Check.Readability.PreferImplicitTry},
+        {Credo.Check.Readability.RedundantBlankLines},
+        {Credo.Check.Readability.StringSigils},
+        {Credo.Check.Readability.TrailingBlankLine},
+        {Credo.Check.Readability.TrailingWhiteSpace},
+        {Credo.Check.Readability.VariableNames},
+        {Credo.Check.Readability.Semicolons},
+        {Credo.Check.Readability.SpaceAfterCommas},
+
+        {Credo.Check.Refactor.DoubleBooleanNegation},
+        {Credo.Check.Refactor.CondStatements},
+        {Credo.Check.Refactor.CyclomaticComplexity},
+        {Credo.Check.Refactor.FunctionArity},
+        {Credo.Check.Refactor.LongQuoteBlocks},
+        {Credo.Check.Refactor.MatchInCondition},
+        {Credo.Check.Refactor.NegatedConditionsInUnless},
+        {Credo.Check.Refactor.NegatedConditionsWithElse},
+        {Credo.Check.Refactor.Nesting},
+        {Credo.Check.Refactor.PipeChainStart},
+        {Credo.Check.Refactor.UnlessWithElse},
+
+        {Credo.Check.Warning.BoolOperationOnSameValues},
+        {Credo.Check.Warning.ExpensiveEmptyEnumCheck},
+        {Credo.Check.Warning.IExPry},
+        {Credo.Check.Warning.IoInspect},
+        {Credo.Check.Warning.LazyLogging},
+        {Credo.Check.Warning.OperationOnSameValues},
+        {Credo.Check.Warning.OperationWithConstantResult},
+        {Credo.Check.Warning.UnusedEnumOperation},
+        {Credo.Check.Warning.UnusedFileOperation},
+        {Credo.Check.Warning.UnusedKeywordOperation},
+        {Credo.Check.Warning.UnusedListOperation},
+        {Credo.Check.Warning.UnusedPathOperation},
+        {Credo.Check.Warning.UnusedRegexOperation},
+        {Credo.Check.Warning.UnusedStringOperation},
+        {Credo.Check.Warning.UnusedTupleOperation},
+        {Credo.Check.Warning.RaiseInsideRescue},
+
+        # Controversial and experimental checks (opt-in, just remove `, false`)
+        #
+        {Credo.Check.Refactor.ABCSize, false},
+        {Credo.Check.Refactor.AppendSingleItem, false},
+        {Credo.Check.Refactor.VariableRebinding, false},
+        {Credo.Check.Warning.MapGetUnsafePass, false},
+        {Credo.Check.Consistency.MultiAliasImportRequireUse, false},
+
+        # Deprecated checks (these will be deleted after a grace period)
+        #
+        {Credo.Check.Readability.Specs, false},
+        {Credo.Check.Warning.NameRedeclarationByAssignment, false},
+        {Credo.Check.Warning.NameRedeclarationByCase, false},
+        {Credo.Check.Warning.NameRedeclarationByDef, false},
+        {Credo.Check.Warning.NameRedeclarationByFn, false},
+
+        # Custom checks can be created using `mix credo.gen.check`.
+        #
+      ]
+    }
+  ]
+}

From 7a8ef14a70bb2ea83e5d095f949bd32ff6e75063 Mon Sep 17 00:00:00 2001
From: Devon Estes <devon.c.estes@gmail.com>
Date: Wed, 31 Jan 2018 11:18:57 +0100
Subject: [PATCH 4/5] Implement feedback and add additional test

This adds a test to make sure callbacks are run in the dry runs, and
also makes sure it's working properly.
---
 .tool-versions                         |  2 +-
 CHANGELOG.md                           |  2 +-
 README.md                              |  2 +-
 lib/benchee/benchmark/runner.ex        |  6 ++--
 lib/benchee/configuration.ex           |  5 +++
 test/benchee/benchmark/runner_test.exs | 47 +++++++++++++++++++++-----
 6 files changed, 48 insertions(+), 16 deletions(-)

diff --git a/.tool-versions b/.tool-versions
index 48f07ad7..b6a4778d 100644
--- a/.tool-versions
+++ b/.tool-versions
@@ -1,2 +1,2 @@
-elixir master
+elixir 1.6.0
 erlang 20.2
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 647825c9..b9f72431 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,7 +5,7 @@ everything will run without error before running the full set of benchmarks.
 
 ### Features (User Facing)
 * new `dry_run` configuration option which allows users to add a dry run of all
-  benchmarks with each input before running the actual suite. This should save
+benchmarks with each input before running the actual suite. This should save
 time while actually writing the code for your benchmarks.
 
 ## 0.12.0 (2018-01-20)
diff --git a/README.md b/README.md
index 13ecf975..dc9323ff 100644
--- a/README.md
+++ b/README.md
@@ -143,7 +143,7 @@ The available options are the following (also documented in [hexdocs](https://he
 
 * `warmup` - the time in seconds for which a benchmarking job should be run without measuring times before real measurements start. This simulates a _"warm"_ running system. Defaults to 2.
 * `time` - the time in seconds for how long each individual benchmarking job should be run and measured. Defaults to 5.
-* `dry_run` - whether or not to run each job with each input to ensure that your code executes without error. This can save time while developing your suites.
+* `dry_run` - whether or not to run each job with each input - including all given before or after scenario or each hooks - before the benchmarks are measured to ensure that your code executes without error. This can save time while developing your suites. Defaults to `false`.
 * `inputs` - a map from descriptive input names to some different input, your benchmarking jobs will then be run with each of these inputs. For this to work your benchmarking function gets the current input passed in as an argument into the function. Defaults to `nil`, aka no input specified and functions are called without an argument. See [Inputs](#inputs).
 * `parallel` - the function of each benchmarking job will be executed in `parallel` number processes. If `parallel: 4` then 4 processes will be spawned that all execute the _same_ function for the given time. When these finish/the time is up 4 new processes will be spawned for the next job/function. This gives you more data in the same time, but also puts a load on the system interfering with benchmark results. For more on the pros and cons of parallel benchmarking [check the wiki](https://github.com/PragTob/benchee/wiki/Parallel-Benchmarking). Defaults to 1 (no parallel execution).
 * `formatters` - list of formatters either as module implementing the formatter behaviour or formatter functions. They are run when using `Benchee.run/2`. Functions need to accept one argument (which is the benchmarking suite with all data) and then use that to produce output. Used for plugins. Defaults to the builtin console formatter `Benchee.Formatters.Console`. See [Formatters](#formatters).
diff --git a/lib/benchee/benchmark/runner.ex b/lib/benchee/benchmark/runner.ex
index 8d32e0c6..65f6f8d2 100644
--- a/lib/benchee/benchmark/runner.ex
+++ b/lib/benchee/benchmark/runner.ex
@@ -25,9 +25,8 @@ defmodule Benchee.Benchmark.Runner do
   """
   @spec run_scenarios([Scenario.t()], ScenarioContext.t()) :: [Scenario.t()]
   def run_scenarios(scenarios, scenario_context) do
-    Enum.map(scenarios, fn scenario ->
-      parallel_benchmark(scenario, scenario_context)
-    end)
+    Enum.each(scenarios, fn scenario -> dry_run(scenario, scenario_context) end)
+    Enum.map(scenarios, fn scenario -> parallel_benchmark(scenario, scenario_context) end)
   end
 
   defp parallel_benchmark(
@@ -38,7 +37,6 @@ defmodule Benchee.Benchmark.Runner do
          }
        ) do
     printer.benchmarking(job_name, input_name, config)
-    dry_run(scenario, scenario_context)
 
     measurements =
       1..config.parallel
diff --git a/lib/benchee/configuration.ex b/lib/benchee/configuration.ex
index 307c5f6d..51f27863 100644
--- a/lib/benchee/configuration.ex
+++ b/lib/benchee/configuration.ex
@@ -45,6 +45,7 @@ defmodule Benchee.Configuration do
           parallel: integer,
           time: number,
           warmup: number,
+          dry_run: boolean,
           formatters: [(Suite.t() -> Suite.t())],
           print: map,
           inputs: %{Suite.key() => any} | nil,
@@ -75,6 +76,10 @@ defmodule Benchee.Configuration do
     how often it is executed). Defaults to 5.
     * `warmup`     - the time in seconds for which the benchmarking function
     should be run without gathering results. Defaults to 2.
+    * `dry_run` - whether or not to run each job with each input - including all
+    given before or after scenario or each hooks - before the benchmarks are
+    measured to ensure that your code executes without error. This can save time
+    while developing your suites. Defaults to `false`.
     * `inputs` - a map from descriptive input names to some different input,
     your benchmarking jobs will then be run with each of these inputs. For this
     to work your benchmarking function gets the current input passed in as an
diff --git a/test/benchee/benchmark/runner_test.exs b/test/benchee/benchmark/runner_test.exs
index 163b5ee0..fa569d89 100644
--- a/test/benchee/benchmark/runner_test.exs
+++ b/test/benchee/benchmark/runner_test.exs
@@ -357,12 +357,7 @@ defmodule Benchee.Benchmark.RunnerTest do
       )
       |> Benchmark.measure(TestPrinter)
 
-      assert_received_exactly([
-        :before_scenario,
-        :before,
-        :after,
-        :after_scenario
-      ])
+      assert_received_exactly([:before_scenario, :before, :after, :after_scenario])
     end
 
     test "hooks trigger during warmup and runtime but scenarios once" do
@@ -839,7 +834,7 @@ defmodule Benchee.Benchmark.RunnerTest do
     end
 
     test "runs all benchmarks with all inputs exactly once as a dry run" do
-      ref = self()
+      me = self()
 
       inputs = %{"small" => 1, "big" => 100}
 
@@ -847,11 +842,45 @@ defmodule Benchee.Benchmark.RunnerTest do
 
       %Suite{configuration: config}
       |> test_suite
-      |> Benchmark.benchmark("first", fn input -> send(ref, {:first, input}) end)
-      |> Benchmark.benchmark("second", fn input -> send(ref, {:second, input}) end)
+      |> Benchmark.benchmark("first", fn input -> send(me, {:first, input}) end)
+      |> Benchmark.benchmark("second", fn input -> send(me, {:second, input}) end)
       |> Benchmark.measure(TestPrinter)
 
       assert_received_exactly([{:first, 100}, {:first, 1}, {:second, 100}, {:second, 1}])
     end
+
+    test "runs all hooks as part of a dry run" do
+      me = self()
+
+      config = %{time: 100, warmup: 100, dry_run: true}
+
+      try do
+        %Suite{configuration: config}
+        |> test_suite
+        |> Benchmark.benchmark("first", fn -> send(me, :first) end)
+        |> Benchmark.benchmark(
+          "job",
+          {fn -> send(me, :second) end,
+           before_each: fn input ->
+             send(me, :before)
+             input
+           end,
+           after_each: fn _ -> send(me, :after) end,
+           before_scenario: fn input ->
+             send(me, :before_scenario)
+             input
+           end,
+           after_scenario: fn _ ->
+             send(me, :after_scenario)
+             raise "This fails!"
+           end}
+        )
+        |> Benchmark.measure(TestPrinter)
+      rescue
+        RuntimeError -> nil
+      end
+
+      assert_received_exactly([:first, :before_scenario, :before, :second, :after, :after_scenario])
+    end
   end
 end

From 3c7118ffa6f601d3f66c2c895b997fc0243028ba Mon Sep 17 00:00:00 2001
From: Devon Estes <devon.c.estes@gmail.com>
Date: Fri, 2 Feb 2018 09:58:01 +0100
Subject: [PATCH 5/5] Rename dry_run to pre_check

---
 CHANGELOG.md                           |  4 ++--
 README.md                              |  2 +-
 lib/benchee/benchmark/runner.ex        |  6 +++---
 lib/benchee/configuration.ex           |  6 +++---
 test/benchee/benchmark/runner_test.exs | 10 +++++-----
 5 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b9f72431..0a60bc35 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,10 +1,10 @@
 ## 0.13.0 (2018-??-??)
 
-Adds the ability to run a `dry_run` of your benchmarks if you want to make sure
+Adds the ability to run a `pre_check` of your benchmarks if you want to make sure
 everything will run without error before running the full set of benchmarks.
 
 ### Features (User Facing)
-* new `dry_run` configuration option which allows users to add a dry run of all
+* new `pre_check` configuration option which allows users to add a dry run of all
 benchmarks with each input before running the actual suite. This should save
 time while actually writing the code for your benchmarks.
 
diff --git a/README.md b/README.md
index dc9323ff..681cb55b 100644
--- a/README.md
+++ b/README.md
@@ -143,7 +143,7 @@ The available options are the following (also documented in [hexdocs](https://he
 
 * `warmup` - the time in seconds for which a benchmarking job should be run without measuring times before real measurements start. This simulates a _"warm"_ running system. Defaults to 2.
 * `time` - the time in seconds for how long each individual benchmarking job should be run and measured. Defaults to 5.
-* `dry_run` - whether or not to run each job with each input - including all given before or after scenario or each hooks - before the benchmarks are measured to ensure that your code executes without error. This can save time while developing your suites. Defaults to `false`.
+* `pre_check` - whether or not to run each job with each input - including all given before or after scenario or each hooks - before the benchmarks are measured to ensure that your code executes without error. This can save time while developing your suites. Defaults to `false`.
 * `inputs` - a map from descriptive input names to some different input, your benchmarking jobs will then be run with each of these inputs. For this to work your benchmarking function gets the current input passed in as an argument into the function. Defaults to `nil`, aka no input specified and functions are called without an argument. See [Inputs](#inputs).
 * `parallel` - the function of each benchmarking job will be executed in `parallel` number processes. If `parallel: 4` then 4 processes will be spawned that all execute the _same_ function for the given time. When these finish/the time is up 4 new processes will be spawned for the next job/function. This gives you more data in the same time, but also puts a load on the system interfering with benchmark results. For more on the pros and cons of parallel benchmarking [check the wiki](https://github.com/PragTob/benchee/wiki/Parallel-Benchmarking). Defaults to 1 (no parallel execution).
 * `formatters` - list of formatters either as module implementing the formatter behaviour or formatter functions. They are run when using `Benchee.run/2`. Functions need to accept one argument (which is the benchmarking suite with all data) and then use that to produce output. Used for plugins. Defaults to the builtin console formatter `Benchee.Formatters.Console`. See [Formatters](#formatters).
diff --git a/lib/benchee/benchmark/runner.ex b/lib/benchee/benchmark/runner.ex
index 65f6f8d2..03dbd3f7 100644
--- a/lib/benchee/benchmark/runner.ex
+++ b/lib/benchee/benchmark/runner.ex
@@ -25,7 +25,7 @@ defmodule Benchee.Benchmark.Runner do
   """
   @spec run_scenarios([Scenario.t()], ScenarioContext.t()) :: [Scenario.t()]
   def run_scenarios(scenarios, scenario_context) do
-    Enum.each(scenarios, fn scenario -> dry_run(scenario, scenario_context) end)
+    Enum.each(scenarios, fn scenario -> pre_check(scenario, scenario_context) end)
     Enum.map(scenarios, fn scenario -> parallel_benchmark(scenario, scenario_context) end)
   end
 
@@ -48,7 +48,7 @@ defmodule Benchee.Benchmark.Runner do
 
   # This will run the given scenario exactly once, including the before and
   # after hooks, to ensure the function can execute without raising an error.
-  defp dry_run(scenario, scenario_context = %ScenarioContext{config: %{dry_run: true}}) do
+  defp pre_check(scenario, scenario_context = %ScenarioContext{config: %{pre_check: true}}) do
     scenario_input = run_before_scenario(scenario, scenario_context)
     scenario_context = %ScenarioContext{scenario_context | scenario_input: scenario_input}
     measure_iteration(scenario, scenario_context)
@@ -56,7 +56,7 @@ defmodule Benchee.Benchmark.Runner do
     nil
   end
 
-  defp dry_run(_, _), do: nil
+  defp pre_check(_, _), do: nil
 
   defp measure_scenario(scenario, scenario_context) do
     scenario_input = run_before_scenario(scenario, scenario_context)
diff --git a/lib/benchee/configuration.ex b/lib/benchee/configuration.ex
index 51f27863..e757bd62 100644
--- a/lib/benchee/configuration.ex
+++ b/lib/benchee/configuration.ex
@@ -15,7 +15,7 @@ defmodule Benchee.Configuration do
   defstruct parallel: 1,
             time: 5,
             warmup: 2,
-            dry_run: false,
+            pre_check: false,
             formatters: [Console],
             print: %{
               benchmarking: true,
@@ -45,7 +45,7 @@ defmodule Benchee.Configuration do
           parallel: integer,
           time: number,
           warmup: number,
-          dry_run: boolean,
+          pre_check: boolean,
           formatters: [(Suite.t() -> Suite.t())],
           print: map,
           inputs: %{Suite.key() => any} | nil,
@@ -76,7 +76,7 @@ defmodule Benchee.Configuration do
     how often it is executed). Defaults to 5.
     * `warmup`     - the time in seconds for which the benchmarking function
     should be run without gathering results. Defaults to 2.
-    * `dry_run` - whether or not to run each job with each input - including all
+    * `pre_check` - whether or not to run each job with each input - including all
     given before or after scenario or each hooks - before the benchmarks are
     measured to ensure that your code executes without error. This can save time
     while developing your suites. Defaults to `false`.
diff --git a/test/benchee/benchmark/runner_test.exs b/test/benchee/benchmark/runner_test.exs
index fa569d89..e252a0b1 100644
--- a/test/benchee/benchmark/runner_test.exs
+++ b/test/benchee/benchmark/runner_test.exs
@@ -10,7 +10,7 @@ defmodule Benchee.Benchmark.RunnerTest do
     time: 40_000,
     warmup: 20_000,
     inputs: nil,
-    dry_run: false,
+    pre_check: false,
     print: %{fast_warning: false, configuration: true}
   }
   @system %{
@@ -833,12 +833,12 @@ defmodule Benchee.Benchmark.RunnerTest do
       ])
     end
 
-    test "runs all benchmarks with all inputs exactly once as a dry run" do
+    test "runs all benchmarks with all inputs exactly once as a pre check" do
       me = self()
 
       inputs = %{"small" => 1, "big" => 100}
 
-      config = %{time: 0, warmup: 0, inputs: inputs, dry_run: true}
+      config = %{time: 0, warmup: 0, inputs: inputs, pre_check: true}
 
       %Suite{configuration: config}
       |> test_suite
@@ -849,10 +849,10 @@ defmodule Benchee.Benchmark.RunnerTest do
       assert_received_exactly([{:first, 100}, {:first, 1}, {:second, 100}, {:second, 1}])
     end
 
-    test "runs all hooks as part of a dry run" do
+    test "runs all hooks as part of a pre check" do
       me = self()
 
-      config = %{time: 100, warmup: 100, dry_run: true}
+      config = %{time: 100, warmup: 100, pre_check: true}
 
       try do
         %Suite{configuration: config}