From 3c7e737a7bcf2d2059b1db2a5a221f76806ae8b0 Mon Sep 17 00:00:00 2001 From: khyperia Date: Mon, 27 Jan 2020 15:43:27 +0100 Subject: [PATCH] Allow configuring of benchmark warmup time --- docs/command-line.md | 10 +++++++ src/catch2/benchmark/catch_benchmark.hpp | 4 +-- src/catch2/catch_commandline.cpp | 3 ++ src/catch2/catch_config.cpp | 9 +++--- src/catch2/catch_config.hpp | 2 ++ src/catch2/catch_interfaces_config.h | 2 ++ .../Baselines/compact.sw.approved.txt | 2 ++ .../Baselines/console.std.approved.txt | 2 +- .../Baselines/console.sw.approved.txt | 24 +++++++++++++-- .../SelfTest/Baselines/junit.sw.approved.txt | 7 +++-- .../Baselines/sonarqube.sw.approved.txt | 5 ++-- tests/SelfTest/Baselines/tap.sw.approved.txt | 6 +++- tests/SelfTest/Baselines/xml.sw.approved.txt | 30 ++++++++++++++++--- .../IntrospectiveTests/CmdLine.tests.cpp | 10 +++++-- 14 files changed, 94 insertions(+), 22 deletions(-) diff --git a/docs/command-line.md b/docs/command-line.md index 742d84b660..7ec2fbb31d 100644 --- a/docs/command-line.md +++ b/docs/command-line.md @@ -24,6 +24,7 @@ [Specify the number of resamples for bootstrapping](#specify-the-number-of-resamples-for-bootstrapping)
[Specify the confidence-interval for bootstrapping](#specify-the-confidence-interval-for-bootstrapping)
[Disable statistical analysis of collected benchmark samples](#disable-statistical-analysis-of-collected-benchmark-samples)
+[Specify the amount of time in milliseconds spent on warming up each test](#specify-the-amount-of-time-in-milliseconds-spent-on-warming-up-each-test)
[Usage](#usage)
[Specify the section to run](#specify-the-section-to-run)
[Filenames as tags](#filenames-as-tags)
@@ -64,6 +65,7 @@ Click one of the following links to take you straight to that option - or scroll ` --benchmark-resamples`
` --benchmark-confidence-interval`
` --benchmark-no-analysis`
+ ` --benchmark-warmup-time`
` --use-colour`

@@ -317,6 +319,14 @@ Must be between 0 and 1 and defaults to 0.95. When this flag is specified no bootstrapping or any other statistical analysis is performed. Instead the user code is only measured and the plain mean from the samples is reported. + +## Specify the amount of time in milliseconds spent on warming up each test +
--benchmark-warmup-time
+ +> [Introduced](https://github.com/catchorg/Catch2/pull/1844) in Catch X.Y.Z. + +Configure the amount of time spent warming up each test. + ## Usage 
-h, -?, --help
diff --git a/src/catch2/benchmark/catch_benchmark.hpp b/src/catch2/benchmark/catch_benchmark.hpp index a6ffcf7ca9..154105c5eb 100644 --- a/src/catch2/benchmark/catch_benchmark.hpp +++ b/src/catch2/benchmark/catch_benchmark.hpp @@ -44,10 +44,10 @@ namespace Catch { template ExecutionPlan> prepare(const IConfig &cfg, Environment> env) const { auto min_time = env.clock_resolution.mean * Detail::minimum_ticks; - auto run_time = std::max(min_time, std::chrono::duration_cast(Detail::warmup_time)); + auto run_time = std::max(min_time, std::chrono::duration_cast(cfg.benchmarkWarmupTime())); auto&& test = Detail::run_for_at_least(std::chrono::duration_cast>(run_time), 1, fun); int new_iters = static_cast(std::ceil(min_time * test.iterations / test.elapsed)); - return { new_iters, test.elapsed / test.iterations * new_iters * cfg.benchmarkSamples(), fun, std::chrono::duration_cast>(Detail::warmup_time), Detail::warmup_iterations }; + return { new_iters, test.elapsed / test.iterations * new_iters * cfg.benchmarkSamples(), fun, std::chrono::duration_cast>(cfg.benchmarkWarmupTime()), Detail::warmup_iterations }; } template diff --git a/src/catch2/catch_commandline.cpp b/src/catch2/catch_commandline.cpp index 3640a2267f..3c453f481b 100644 --- a/src/catch2/catch_commandline.cpp +++ b/src/catch2/catch_commandline.cpp @@ -210,6 +210,9 @@ namespace Catch { | Opt( config.benchmarkNoAnalysis ) ["--benchmark-no-analysis"] ( "perform only measurements; do not perform any analysis" ) + | Opt( config.benchmarkWarmupTime, "benchmarkWarmupTime" ) + ["--benchmark-warmup-time"] + ( "amount of time in milliseconds spent on warming up each test (default: 100)" ) | Arg( config.testsOrTags, "test name|pattern|tags" ) ( "which test or tests to use" ); diff --git a/src/catch2/catch_config.cpp b/src/catch2/catch_config.cpp index ea7ec3def2..7c0393daba 100644 --- a/src/catch2/catch_config.cpp +++ b/src/catch2/catch_config.cpp @@ -75,10 +75,11 @@ namespace Catch { bool Config::showInvisibles() const { return m_data.showInvisibles; } Verbosity Config::verbosity() const { return m_data.verbosity; } - bool Config::benchmarkNoAnalysis() const { return m_data.benchmarkNoAnalysis; } - int Config::benchmarkSamples() const { return m_data.benchmarkSamples; } - double Config::benchmarkConfidenceInterval() const { return m_data.benchmarkConfidenceInterval; } - unsigned int Config::benchmarkResamples() const { return m_data.benchmarkResamples; } + bool Config::benchmarkNoAnalysis() const { return m_data.benchmarkNoAnalysis; } + int Config::benchmarkSamples() const { return m_data.benchmarkSamples; } + double Config::benchmarkConfidenceInterval() const { return m_data.benchmarkConfidenceInterval; } + unsigned int Config::benchmarkResamples() const { return m_data.benchmarkResamples; } + std::chrono::milliseconds Config::benchmarkWarmupTime() const { return std::chrono::milliseconds(m_data.benchmarkWarmupTime); } IStream const* Config::openStream() { return Catch::makeStream(m_data.outputFilename); diff --git a/src/catch2/catch_config.hpp b/src/catch2/catch_config.hpp index 55fa5ed874..1497212122 100644 --- a/src/catch2/catch_config.hpp +++ b/src/catch2/catch_config.hpp @@ -43,6 +43,7 @@ namespace Catch { unsigned int benchmarkSamples = 100; double benchmarkConfidenceInterval = 0.95; unsigned int benchmarkResamples = 100000; + std::chrono::milliseconds::rep benchmarkWarmupTime = 100; Verbosity verbosity = Verbosity::Normal; WarnAbout::What warnings = WarnAbout::Nothing; @@ -108,6 +109,7 @@ namespace Catch { int benchmarkSamples() const override; double benchmarkConfidenceInterval() const override; unsigned int benchmarkResamples() const override; + std::chrono::milliseconds benchmarkWarmupTime() const override; private: diff --git a/src/catch2/catch_interfaces_config.h b/src/catch2/catch_interfaces_config.h index 1ef601fe85..ddaf2e8e58 100644 --- a/src/catch2/catch_interfaces_config.h +++ b/src/catch2/catch_interfaces_config.h @@ -11,6 +11,7 @@ #include #include +#include #include #include #include @@ -81,6 +82,7 @@ namespace Catch { virtual int benchmarkSamples() const = 0; virtual double benchmarkConfidenceInterval() const = 0; virtual unsigned int benchmarkResamples() const = 0; + virtual std::chrono::milliseconds benchmarkWarmupTime() const = 0; }; using IConfigPtr = std::shared_ptr; diff --git a/tests/SelfTest/Baselines/compact.sw.approved.txt b/tests/SelfTest/Baselines/compact.sw.approved.txt index 0d6cd16051..480078224a 100644 --- a/tests/SelfTest/Baselines/compact.sw.approved.txt +++ b/tests/SelfTest/Baselines/compact.sw.approved.txt @@ -1152,6 +1152,8 @@ CmdLine.tests.cpp:: passed: cli.parse({ "test", "--benchmark-confid CmdLine.tests.cpp:: passed: config.benchmarkConfidenceInterval == Catch::Approx(0.99) for: 0.99 == Approx( 0.99 ) CmdLine.tests.cpp:: passed: cli.parse({ "test", "--benchmark-no-analysis" }) for: {?} CmdLine.tests.cpp:: passed: config.benchmarkNoAnalysis for: true +CmdLine.tests.cpp:: passed: cli.parse({ "test", "--benchmark-warmup-time=10" }) for: {?} +CmdLine.tests.cpp:: passed: config.benchmarkWarmupTime == 10 for: 10 == 10 Misc.tests.cpp:: passed: std::tuple_size::value >= 1 for: 3 >= 1 Misc.tests.cpp:: passed: std::tuple_size::value >= 1 for: 2 >= 1 Misc.tests.cpp:: passed: std::tuple_size::value >= 1 for: 1 >= 1 diff --git a/tests/SelfTest/Baselines/console.std.approved.txt b/tests/SelfTest/Baselines/console.std.approved.txt index 7056db0b72..69ba403517 100644 --- a/tests/SelfTest/Baselines/console.std.approved.txt +++ b/tests/SelfTest/Baselines/console.std.approved.txt @@ -1381,5 +1381,5 @@ due to unexpected exception with message: =============================================================================== test cases: 328 | 254 passed | 70 failed | 4 failed as expected -assertions: 1827 | 1675 passed | 131 failed | 21 failed as expected +assertions: 1829 | 1677 passed | 131 failed | 21 failed as expected diff --git a/tests/SelfTest/Baselines/console.sw.approved.txt b/tests/SelfTest/Baselines/console.sw.approved.txt index 595acde66e..d6285a8185 100644 --- a/tests/SelfTest/Baselines/console.sw.approved.txt +++ b/tests/SelfTest/Baselines/console.sw.approved.txt @@ -8284,7 +8284,7 @@ with expansion: ------------------------------------------------------------------------------- Process can be configured on command line Benchmark options - resamples + confidence-interval ------------------------------------------------------------------------------- CmdLine.tests.cpp: ............................................................................... @@ -8302,7 +8302,7 @@ with expansion: ------------------------------------------------------------------------------- Process can be configured on command line Benchmark options - resamples + no-analysis ------------------------------------------------------------------------------- CmdLine.tests.cpp: ............................................................................... @@ -8317,6 +8317,24 @@ CmdLine.tests.cpp:: PASSED: with expansion: true +------------------------------------------------------------------------------- +Process can be configured on command line + Benchmark options + warmup-time +------------------------------------------------------------------------------- +CmdLine.tests.cpp: +............................................................................... + +CmdLine.tests.cpp:: PASSED: + CHECK( cli.parse({ "test", "--benchmark-warmup-time=10" }) ) +with expansion: + {?} + +CmdLine.tests.cpp:: PASSED: + REQUIRE( config.benchmarkWarmupTime == 10 ) +with expansion: + 10 == 10 + ------------------------------------------------------------------------------- Product with differing arities - std::tuple ------------------------------------------------------------------------------- @@ -14283,5 +14301,5 @@ Misc.tests.cpp:: PASSED: =============================================================================== test cases: 328 | 238 passed | 86 failed | 4 failed as expected -assertions: 1844 | 1675 passed | 148 failed | 21 failed as expected +assertions: 1846 | 1677 passed | 148 failed | 21 failed as expected diff --git a/tests/SelfTest/Baselines/junit.sw.approved.txt b/tests/SelfTest/Baselines/junit.sw.approved.txt index f7214ed739..0be5cc07d0 100644 --- a/tests/SelfTest/Baselines/junit.sw.approved.txt +++ b/tests/SelfTest/Baselines/junit.sw.approved.txt @@ -1,7 +1,7 @@ - + @@ -1024,8 +1024,9 @@ Message.tests.cpp: - - + + + diff --git a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt index 517c5f0b6f..07c512f00a 100644 --- a/tests/SelfTest/Baselines/sonarqube.sw.approved.txt +++ b/tests/SelfTest/Baselines/sonarqube.sw.approved.txt @@ -64,8 +64,9 @@ - - + + + diff --git a/tests/SelfTest/Baselines/tap.sw.approved.txt b/tests/SelfTest/Baselines/tap.sw.approved.txt index f6c539618c..b22d58c50d 100644 --- a/tests/SelfTest/Baselines/tap.sw.approved.txt +++ b/tests/SelfTest/Baselines/tap.sw.approved.txt @@ -2226,6 +2226,10 @@ ok {test-number} - config.benchmarkConfidenceInterval == Catch::Approx(0.99) for ok {test-number} - cli.parse({ "test", "--benchmark-no-analysis" }) for: {?} # Process can be configured on command line ok {test-number} - config.benchmarkNoAnalysis for: true +# Process can be configured on command line +ok {test-number} - cli.parse({ "test", "--benchmark-warmup-time=10" }) for: {?} +# Process can be configured on command line +ok {test-number} - config.benchmarkWarmupTime == 10 for: 10 == 10 # Product with differing arities - std::tuple ok {test-number} - std::tuple_size::value >= 1 for: 3 >= 1 # Product with differing arities - std::tuple @@ -3680,5 +3684,5 @@ ok {test-number} - q3 == 23. for: 23.0 == 23.0 ok {test-number} - # xmlentitycheck ok {test-number} - -1..1836 +1..1838 diff --git a/tests/SelfTest/Baselines/xml.sw.approved.txt b/tests/SelfTest/Baselines/xml.sw.approved.txt index 6abee4b7e5..d2d0ce5470 100644 --- a/tests/SelfTest/Baselines/xml.sw.approved.txt +++ b/tests/SelfTest/Baselines/xml.sw.approved.txt @@ -10354,7 +10354,7 @@ Nor would this
-
+
cli.parse({ "test", "--benchmark-confidence-interval=0.99" }) @@ -10376,7 +10376,7 @@ Nor would this
-
+
cli.parse({ "test", "--benchmark-no-analysis" }) @@ -10397,6 +10397,28 @@ Nor would this
+
+
+ + + cli.parse({ "test", "--benchmark-warmup-time=10" }) + + + {?} + + + + + config.benchmarkWarmupTime == 10 + + + 10 == 10 + + + +
+ +
@@ -17117,7 +17139,7 @@ loose text artifact
- + - + diff --git a/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp b/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp index 68dafad1f4..b7d5c9c27d 100644 --- a/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp +++ b/tests/SelfTest/IntrospectiveTests/CmdLine.tests.cpp @@ -497,17 +497,23 @@ TEST_CASE( "Process can be configured on command line", "[config][command-line]" REQUIRE(config.benchmarkResamples == 20000); } - SECTION("resamples") { + SECTION("confidence-interval") { CHECK(cli.parse({ "test", "--benchmark-confidence-interval=0.99" })); REQUIRE(config.benchmarkConfidenceInterval == Catch::Approx(0.99)); } - SECTION("resamples") { + SECTION("no-analysis") { CHECK(cli.parse({ "test", "--benchmark-no-analysis" })); REQUIRE(config.benchmarkNoAnalysis); } + + SECTION("warmup-time") { + CHECK(cli.parse({ "test", "--benchmark-warmup-time=10" })); + + REQUIRE(config.benchmarkWarmupTime == 10); + } } }