Merge c4246ac into 5159967

include/benchmark/benchmark.h

@@ -438,9 +438,12 @@ enum AggregationReportMode
     : unsigned
 #else
 #endif
-{ ARM_Unspecified,  // The mode has not been manually specified
-  ARM_Default,      // The mode is user-specified as default.
-  ARM_ReportAggregatesOnly };
+{ ARM_Unspecified,           // The mode has not been manually specified
+  ARM_Default,               // The mode is user-specified as default.
+  ARM_ReportAggregatesOnly,  // All reporters only display aggregates.
+  ARM_DisplayAggregatesOnly  // Display reporter displays aggregates only,
+                             // but file reporter still contains everything.
+};
 }  // namespace internal
 
 // State is passed to a running Benchmark and contains state for the
@@ -862,8 +865,12 @@ class Benchmark {
   // Specify if each repetition of the benchmark should be reported separately
   // or if only the final statistics should be reported. If the benchmark
   // is not repeated then the single result is always reported.
+  // Applies to *ALL* reporters (display and file).
   Benchmark* ReportAggregatesOnly(bool value = true);
 
+  // Same as ReportAggregatesOnly(), but applies to display reporter only.
+  Benchmark* DisplayAggregatesOnly(bool value = true);
+
   // If a particular benchmark is I/O bound, runs multiple threads internally or
   // if for some reason CPU timings are not representative, call this method. If
   // called, the elapsed time will be used to control how many iterations are

src/benchmark.cc

@@ -34,6 +34,7 @@
 #include <memory>
 #include <string>
 #include <thread>
+#include <utility>
 
 #include "check.h"
 #include "colorprint.h"
@@ -72,10 +73,19 @@ DEFINE_int32(benchmark_repetitions, 1,
              "The number of runs of each benchmark. If greater than 1, the "
              "mean and standard deviation of the runs will be reported.");
 
-DEFINE_bool(benchmark_report_aggregates_only, false,
-            "Report the result of each benchmark repetitions. When 'true' is "
-            "specified only the mean, standard deviation, and other statistics "
-            "are reported for repeated benchmarks.");
+DEFINE_bool(
+    benchmark_report_aggregates_only, false,
+    "Report the result of each benchmark repetitions. When 'true' is specified "
+    "only the mean, standard deviation, and other statistics are reported for "
+    "repeated benchmarks. Affects all reporters.");
+
+DEFINE_bool(
+    benchmark_display_aggregates_only, false,
+    "Display the result of each benchmark repetitions. When 'true' is "
+    "specified only the mean, standard deviation, and other statistics are "
+    "displayed for repeated benchmarks. Unlike "
+    "benchmark_report_aggregates_only, only affects the display reporter, but "
+    "*NOT* file reporter, which will still contain all the output.");
 
 DEFINE_string(benchmark_format, "console",
               "The format to use for console output. Valid values are "
@@ -193,9 +203,11 @@ void RunInThread(const benchmark::internal::Benchmark::Instance* b,
   manager->NotifyThreadComplete();
 }
 
-std::vector<BenchmarkReporter::Run> RunBenchmark(
-    const benchmark::internal::Benchmark::Instance& b,
-    std::vector<BenchmarkReporter::Run>* complexity_reports) {
+std::pair<std::vector<BenchmarkReporter::Run> /*everything*/,
+          std::vector<BenchmarkReporter::Run> /*aggregates_only*/>
+RunBenchmark(const benchmark::internal::Benchmark::Instance& b,
+             std::vector<BenchmarkReporter::Run>* complexity_reports,
+             bool& display_aggregates_only) {
   std::vector<BenchmarkReporter::Run> reports;  // return value
 
   const bool has_explicit_iteration_count = b.iterations != 0;
@@ -209,6 +221,11 @@ std::vector<BenchmarkReporter::Run> RunBenchmark(
       (b.aggregation_report_mode == internal::ARM_Unspecified
            ? FLAGS_benchmark_report_aggregates_only
            : b.aggregation_report_mode == internal::ARM_ReportAggregatesOnly);
+  display_aggregates_only =
+      repeats != 1 &&
+      (b.aggregation_report_mode == internal::ARM_Unspecified
+           ? FLAGS_benchmark_display_aggregates_only
+           : b.aggregation_report_mode == internal::ARM_DisplayAggregatesOnly);
   for (int repetition_num = 0; repetition_num < repeats; repetition_num++) {
     for (;;) {
       // Try benchmark
@@ -304,18 +321,26 @@ std::vector<BenchmarkReporter::Run> RunBenchmark(
       iters = static_cast<int>(next_iters + 0.5);
     }
   }
+
   // Calculate additional statistics
-  auto stat_reports = ComputeStats(reports);
+  auto aggregate_reports = ComputeStats(reports);
+
+  // Maybe calculate complexity report
   if ((b.complexity != oNone) && b.last_benchmark_instance) {
     auto additional_run_stats = ComputeBigO(*complexity_reports);
-    stat_reports.insert(stat_reports.end(), additional_run_stats.begin(),
-                        additional_run_stats.end());
+    aggregate_reports.insert(aggregate_reports.end(),
+                             additional_run_stats.begin(),
+                             additional_run_stats.end());
     complexity_reports->clear();
   }
 
   if (report_aggregates_only) reports.clear();
-  reports.insert(reports.end(), stat_reports.begin(), stat_reports.end());
-  return reports;
+
+  // Append aggregate statistics to the report
+  reports.insert(reports.end(), aggregate_reports.begin(),
+                 aggregate_reports.end());
+
+  return std::make_pair(reports, aggregate_reports);
 }
 
 }  // namespace
@@ -463,10 +488,18 @@ void RunBenchmarks(const std::vector<Benchmark::Instance>& benchmarks,
     flushStreams(display_reporter);
     flushStreams(file_reporter);
     for (const auto& benchmark : benchmarks) {
-      std::vector<BenchmarkReporter::Run> reports =
-          RunBenchmark(benchmark, &complexity_reports);
-      display_reporter->ReportRuns(reports);
-      if (file_reporter) file_reporter->ReportRuns(reports);
+      bool display_aggregates_only;
+      std::pair<std::vector<BenchmarkReporter::Run> /*everything*/,
+                std::vector<BenchmarkReporter::Run> /*aggregates_only*/>
+          reports = RunBenchmark(benchmark, &complexity_reports,
+                                 display_aggregates_only);
+
+      const std::vector<BenchmarkReporter::Run>& report_for_display =
+          display_aggregates_only ? reports.second /*aggregates_only*/
+                                  : reports.first /*everything*/;
+
+      display_reporter->ReportRuns(report_for_display);
+      if (file_reporter) file_reporter->ReportRuns(reports.first);
       flushStreams(display_reporter);
       flushStreams(file_reporter);
     }
@@ -596,6 +629,7 @@ void PrintUsageAndExit() {
           "          [--benchmark_min_time=<min_time>]\n"
           "          [--benchmark_repetitions=<num_repetitions>]\n"
           "          [--benchmark_report_aggregates_only={true|false}\n"
+          "          [--benchmark_display_aggregates_only={true|false}\n"
           "          [--benchmark_format=<console|json|csv>]\n"
           "          [--benchmark_out=<filename>]\n"
           "          [--benchmark_out_format=<json|console|csv>]\n"
@@ -619,6 +653,8 @@ void ParseCommandLineFlags(int* argc, char** argv) {
                        &FLAGS_benchmark_repetitions) ||
         ParseBoolFlag(argv[i], "benchmark_report_aggregates_only",
                       &FLAGS_benchmark_report_aggregates_only) ||
+        ParseBoolFlag(argv[i], "benchmark_display_aggregates_only",
+                      &FLAGS_benchmark_display_aggregates_only) ||
         ParseStringFlag(argv[i], "benchmark_format", &FLAGS_benchmark_format) ||
         ParseStringFlag(argv[i], "benchmark_out", &FLAGS_benchmark_out) ||
         ParseStringFlag(argv[i], "benchmark_out_format",

src/benchmark_register.cc

@@ -373,6 +373,11 @@ Benchmark* Benchmark::ReportAggregatesOnly(bool value) {
   return this;
 }
 
+Benchmark* Benchmark::DisplayAggregatesOnly(bool value) {
+  aggregation_report_mode_ = value ? ARM_DisplayAggregatesOnly : ARM_Default;
+  return this;
+}
+
 Benchmark* Benchmark::UseRealTime() {
   CHECK(!use_manual_time_)
       << "Cannot set UseRealTime and UseManualTime simultaneously.";

test/reporter_output_test.cc

@@ -342,6 +342,33 @@ ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
                       {"^\"BM_SummaryRepeat/repeats:3_median\",%csv_report$"},
                       {"^\"BM_SummaryRepeat/repeats:3_stddev\",%csv_report$"}});
 
+// Test that non-aggregate data is not displayed.
+// NOTE: this test is kinda bad. we are only testing the display output.
+//       But we don't check that the file output still contains everything...
+void BM_SummaryDisplay(benchmark::State& state) {
+  for (auto _ : state) {
+  }
+}
+BENCHMARK(BM_SummaryDisplay)->Repetitions(2)->DisplayAggregatesOnly();
+ADD_CASES(TC_ConsoleOut,
+          {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
+           {"^BM_SummaryDisplay/repeats:2_mean %console_report$"},
+           {"^BM_SummaryDisplay/repeats:2_median %console_report$"},
+           {"^BM_SummaryDisplay/repeats:2_stddev %console_report$"}});
+ADD_CASES(TC_JSONOut, {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
+                       {"\"name\": \"BM_SummaryDisplay/repeats:2_mean\",$"},
+                       {"\"run_type\": \"aggregate\",$", MR_Next},
+                       {"\"name\": \"BM_SummaryDisplay/repeats:2_median\",$"},
+                       {"\"run_type\": \"aggregate\",$", MR_Next},
+                       {"\"name\": \"BM_SummaryDisplay/repeats:2_stddev\",$"},
+                       {"\"run_type\": \"aggregate\",$", MR_Next}});
+ADD_CASES(TC_CSVOut,
+          {{".*BM_SummaryDisplay/repeats:2 ", MR_Not},
+           {"^\"BM_SummaryDisplay/repeats:2_mean\",%csv_report$"},
+           {"^\"BM_SummaryDisplay/repeats:2_median\",%csv_report$"},
+           {"^\"BM_SummaryDisplay/repeats:2_stddev\",%csv_report$"}});
+
+// Test repeats with custom time unit.
 void BM_RepeatTimeUnit(benchmark::State& state) {
   for (auto _ : state) {
   }