From b4b44347e5208e51781e3e9f20d7dd87d07dbbfc Mon Sep 17 00:00:00 2001
From: Russell Brown <russell.brown@mac.com>
Date: Wed, 4 Oct 2017 14:39:55 +0100
Subject: [PATCH] Non-deterministic test needs a little ?SOMETIMES

prop_quantile/0 is by it's nature non-deterministic, and the
comments/notes make it clear. Rather than tolerate it breaking the build
periodically, this commit changes the property by using eqc's
`?SOMETIMES` macro. If a test fails 3 times in a row then we assume that
it is a deterministic failure, a bug. Otherwise, assume it is the noted
"loosy goosy" nature of the comparison and pass the property. NOTE:
still think we should find and fit the error bounds.
---
 src/basho_stats_histogram.erl | 35 ++++++++++++++++++++---------------
 1 file changed, 20 insertions(+), 15 deletions(-)

diff --git a/src/basho_stats_histogram.erl b/src/basho_stats_histogram.erl
index 26ce6b4..9c6562e 100644
--- a/src/basho_stats_histogram.erl
+++ b/src/basho_stats_histogram.erl
@@ -245,23 +245,28 @@ prop_quantile() ->
     %% XXX since we try to generate the quantile from the histogram, not the
     %% original data, our results and Rs don't always agree and this means the
     %% test will occasionally fail. There's not an easy way to fix this.
-    ?FORALL({Min, Bins, Xlen, Q}, {choose(1, 99), choose(50, 200), choose(100, 500),
-                                    choose(0,100)},
-             ?LET(Max, choose(Min+1, 100),
-                  ?LET(Xs, vector(Xlen, choose(Min, Max)),
-                      ?WHENFAIL(
-                          begin
-                                  io:format("Min ~p, Max ~p, Bins ~p, Q ~p, Xs ~w~n",
-                                      [Min, Max, Bins, Q, Xs]),
-                                  Command = ?FMT("quantile(x, ~4.2f, type=4)", [Q * 0.01]),
-                                  InputStr = [integer_to_list(I) || I <- Xs],
-                                  io:format(?FMT("x <- c(~s)\n", [string:join(InputStr, ",")])),
-                                  io:format(?FMT("write(~s, ncolumns=1, file=stdout())\n", [Command]))
-                          end,
-                          qc_quantile_check(Q, Min, Max, Bins, Xs))))).
+    ?SOMETIMES(3,
+               %% as the comment above states, this is
+               %% non-deterministic, but it should _never_ fail 3
+               %% times of 3
+               ?FORALL({Min, Bins, Xlen, Q}, {choose(1, 99), choose(50, 200), choose(100, 500),
+                                              choose(0,100)},
+            ?LET(Max, choose(Min+1, 100),
+                 ?LET(Xs, vector(Xlen, choose(Min, Max)),
+                                 ?WHENFAIL(
+                                    begin
+                                        io:format("Min ~p, Max ~p, Bins ~p, Q ~p, Xs ~w~n",
+                                                  [Min, Max, Bins, Q, Xs]),
+                                        Command = ?FMT("quantile(x, ~4.2f, type=4)", [Q * 0.01]),
+                                        InputStr = [integer_to_list(I) || I <- Xs],
+                                        io:format(?FMT("x <- c(~s)\n", [string:join(InputStr, ",")])),
+                                        io:format(?FMT("write(~s, ncolumns=1, file=stdout())\n", [Command]))
+                                    end,
+
+                                    qc_quantile_check(Q, Min, Max, Bins, Xs)))))).
 
 qc_quantile_test() ->
     true = eqc:quickcheck(prop_quantile()).
 
 -endif.
--endif. 
+-endif.