From e90a5af6985dbcf1eabfc56a9acf1fc98fb6c8ca Mon Sep 17 00:00:00 2001
From: Paul Koch <code@koch.ninja>
Date: Mon, 23 Dec 2024 17:54:41 -0800
Subject: [PATCH] compare exact non-accelerated final metric vs metric when run
 with acceleration

---
 .../libebm/tests/boosting_unusual_inputs.cpp  | 84 ++++++++++++++++++-
 1 file changed, 80 insertions(+), 4 deletions(-)

diff --git a/shared/libebm/tests/boosting_unusual_inputs.cpp b/shared/libebm/tests/boosting_unusual_inputs.cpp
index 82c1f9918..1773ef0c6 100644
--- a/shared/libebm/tests/boosting_unusual_inputs.cpp
+++ b/shared/libebm/tests/boosting_unusual_inputs.cpp
@@ -2063,8 +2063,10 @@ TEST_CASE("lossguide, boosting, regression") {
 
 TEST_CASE("stress test, boosting") {
    auto rng = MakeRng(0);
-   const IntEbm cTrainSamples = 200;
-   const IntEbm cValidationSamples = 100;
+   AccelerationFlags acceleration = AccelerationFlags_NONE;
+
+   const IntEbm cTrainSamples = 211; // have some non-SIMD residuals
+   const IntEbm cValidationSamples = 101; // have some non-SIMD residuals
    const std::vector<FeatureTest> features = {
          FeatureTest(10, false, false, false),
          FeatureTest(10, false, false, true),
@@ -2117,7 +2119,81 @@ TEST_CASE("stress test, boosting") {
                validation,
                innerBagCount,
                k_testCreateBoosterFlags_Default,
-               AccelerationFlags_NONE);
+               acceleration);
+
+         double validationMetricIteration = 0.0;
+         for(size_t iRound = 0; iRound < cRounds; ++iRound) {
+            for(IntEbm iTerm = 0; iTerm < static_cast<IntEbm>(terms.size()); ++iTerm) {
+               const IntEbm cRealBins = features[terms[iTerm][0]].CountRealBins();
+               const IntEbm cDimensions = terms[iTerm].size();
+
+               const TermBoostFlags boostFlags =
+                     static_cast<TermBoostFlags>(ChooseAny(rng, boostFlagsAny) | ChooseFrom(rng, boostFlagsChoose));
+
+               const double learningRate = 0.015625;
+               const IntEbm minSamplesLeaf = TestRand(rng, 5) + 1;
+               const double minHessian = 0 == TestRand(rng, 5) ? 0.015625 : 0.0;
+               const double regAlpha = 0 == TestRand(rng, 5) ? 0.015625 : 0.0;
+               const double regLambda = 0 == TestRand(rng, 5) ? 0.015625 : 0.0;
+               const double maxDeltaStep = 0 == TestRand(rng, 5) ? 1.0 : 0.0;
+               const double categoricalSmoothing = 10.0;
+               const IntEbm maxCategoricalThreshold = 1 + TestRand(rng, cRealBins + 1);
+               const double categoricalInclusionPercent = 0 == TestRand(rng, 2) ? 0.75 : 1.0;
+
+               // we allow 1 cut more than the number of bins to test excessive leaves.
+               const IntEbm cLeaves = 1 + TestRand(rng, cRealBins + 1);
+               const std::vector<IntEbm> leaves(cDimensions, cLeaves);
+               const MonotoneDirection direction =
+                     0 == TestRand(rng, 5) ? static_cast<MonotoneDirection>(TestRand(rng, 2) * 2 - 1) : 0;
+               const std::vector<MonotoneDirection> monotonicity(cDimensions, direction);
+
+               validationMetricIteration = test.Boost(iTerm,
+                                                     boostFlags,
+                                                     learningRate,
+                                                     minSamplesLeaf,
+                                                     minHessian,
+                                                     regAlpha,
+                                                     regLambda,
+                                                     maxDeltaStep,
+                                                     categoricalSmoothing,
+                                                     maxCategoricalThreshold,
+                                                     categoricalInclusionPercent,
+                                                     leaves,
+                                                     monotonicity)
+                                                 .validationMetric;
+            }
+         }
+         if(classesCount == 1) {
+            CHECK(std::numeric_limits<double>::infinity() == validationMetricIteration);
+         } else {
+            validationMetric *= validationMetricIteration;
+         }
+      }
+   }
+
+   const double expected = 26838942758406.215;
+   CHECK(validationMetric == expected);
+
+   // Now redo the above, but using acceleration. Results will likely be slightly different but similar.
+   rng = MakeRng(0);
+   acceleration = AccelerationFlags_ALL;
+   validationMetric = 1.0;
+
+   for(IntEbm classesCount = Task_Regression; classesCount < 5; ++classesCount) {
+      if(classesCount != Task_Regression && classesCount < 1) {
+         continue;
+      }
+      const auto train = MakeRandomDataset(rng, classesCount, cTrainSamples, features);
+      const auto validation = MakeRandomDataset(rng, classesCount, cValidationSamples, features);
+      for(IntEbm innerBagCount = 0; innerBagCount < 3; ++innerBagCount) {
+         TestBoost test = TestBoost(classesCount,
+               features,
+               terms,
+               train,
+               validation,
+               innerBagCount,
+               k_testCreateBoosterFlags_Default,
+               acceleration);
 
          double validationMetricIteration = 0.0;
          for(size_t iRound = 0; iRound < cRounds; ++iRound) {
@@ -2169,5 +2245,5 @@ TEST_CASE("stress test, boosting") {
       }
    }
 
-   CHECK(validationMetric == 30885317143376.566);
+   CHECK_APPROX_TOLERANCE(validationMetric, expected, 1e-2);
 }