From c694515ded8c708d599b4566c7731f4ce1f93af4 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Mon, 31 Aug 2020 13:53:41 -0400 Subject: [PATCH 1/2] [ML] do not fail to start analytics process of memory estimate is low --- lib/api/CDataFrameAnalysisRunner.cc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/api/CDataFrameAnalysisRunner.cc b/lib/api/CDataFrameAnalysisRunner.cc index c4492558a5..c95604085e 100644 --- a/lib/api/CDataFrameAnalysisRunner.cc +++ b/lib/api/CDataFrameAnalysisRunner.cc @@ -99,14 +99,14 @@ void CDataFrameAnalysisRunner::computeAndSaveExecutionStrategy() { auto roundMb = [](std::size_t memory) { return 0.01 * static_cast((100 * memory) / BYTES_IN_MB); }; - - // Report rounded up to the nearest MB. - HANDLE_FATAL(<< "Input error: memory limit " << roundMb(memoryLimit) - << "MB is too low to perform analysis. You need to give the process" - << " at least " << std::ceil(roundMb(memoryUsage)) - << "MB, but preferably more."); - - } else if (m_NumberPartitions > 1) { + // Simply log the limit being configured too low. + // If we exceed the limit during the process, we will fail and the user + // will have to update the limit and attempt to re-run + LOG_DEBUG(<< "Memory limit " << roundMb(memoryLimit) << "MB is configured lower than estimate " + << std::ceil(roundMb(memoryUsage)) << "MB." + << "Analytics process may fail due to low memory limit"); + } + if (m_NumberPartitions > 1) { // The maximum number of rows is found by binary search in the interval // [numberRows / m_NumberPartitions, numberRows / (m_NumberPartitions - 1)). From c384d9f63c2a97685b274a23c16376f5826dfc03 Mon Sep 17 00:00:00 2001 From: Benjamin Trent <4357155+benwtrent@users.noreply.github.com> Date: Tue, 1 Sep 2020 07:35:28 -0400 Subject: [PATCH 2/2] fixing test --- lib/api/unittest/CDataFrameAnalysisRunnerTest.cc | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/api/unittest/CDataFrameAnalysisRunnerTest.cc b/lib/api/unittest/CDataFrameAnalysisRunnerTest.cc index f6ab101fbb..6a6183b12a 100644 --- a/lib/api/unittest/CDataFrameAnalysisRunnerTest.cc +++ b/lib/api/unittest/CDataFrameAnalysisRunnerTest.cc @@ -95,12 +95,8 @@ BOOST_AUTO_TEST_CASE(testComputeAndSaveExecutionStrategyDiskUsageFlag) { .diskUsageAllowed(false) .outlierSpec(); - // single error is registered that the memory limit is to low - LOG_DEBUG(<< "errors = " << core::CContainerPrinter::print(errors)); - core::CRegex re; - re.init("Input error: memory limit.*"); - BOOST_REQUIRE_EQUAL(1, static_cast(errors.size())); - BOOST_TEST_REQUIRE(re.matches(errors[0])); + // no error should be registered + BOOST_REQUIRE_EQUAL(0, static_cast(errors.size())); } // Test large memory requirement with disk usage