From 7392137d62526266be791d3a10aafc48e0857606 Mon Sep 17 00:00:00 2001 From: Eduardo Arias Date: Thu, 24 Oct 2024 09:53:42 -0300 Subject: [PATCH] Add support to regressions tests for matching multiple values in error_log. - Update `Variable offset - FILES_NAMES` regression test to specify expected error log output with two strings, because the order in which they're processed can vary in different systems. - The test currently expects a specific order, which is not guaranteed as the underlying structure in `AnchoredSetVariable::resolve` is a `std::unordered_multimap`. - The test is now enabled on Windows builds too. - Simplified `Test match variable (*/n)` tests not to depend on order and enabled them on Windows builds too. --- .github/workflows/ci.yml | 8 -------- build/win32/docker/Dockerfile | 15 --------------- test/regression/regression.cc | 6 ++++-- test/regression/regression_test.cc | 9 ++++++++- test/regression/regression_test.h | 3 ++- .../regression/issue-2423-msg-in-chain.json | 3 --- test/test-cases/regression/offset-variable.json | 5 ++++- 7 files changed, 18 insertions(+), 31 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 12075aa423..73da98321b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -166,14 +166,6 @@ jobs: md \bin copy "C:\Program Files\Git\usr\bin\echo.exe" \bin copy "C:\Program Files\Git\usr\bin\echo.exe" \bin\echo - - name: Disable tests that don't work on Windows - working-directory: test\test-cases\regression - shell: cmd - run: | - jq "map(if .title == \"Test match variable (1/n)\" then .enabled = 0 else . end)" issue-2423-msg-in-chain.json > tmp.json && move /Y tmp.json issue-2423-msg-in-chain.json - jq "map(if .title == \"Test match variable (2/n)\" then .enabled = 0 else . end)" issue-2423-msg-in-chain.json > tmp.json && move /Y tmp.json issue-2423-msg-in-chain.json - jq "map(if .title == \"Test match variable (3/n)\" then .enabled = 0 else . end)" issue-2423-msg-in-chain.json > tmp.json && move /Y tmp.json issue-2423-msg-in-chain.json - jq "map(if .title == \"Variable offset - FILES_NAMES\" then .enabled = 0 else . end)" offset-variable.json > tmp.json && move /Y tmp.json offset-variable.json - name: Run tests working-directory: build\win32\build run: | diff --git a/build/win32/docker/Dockerfile b/build/win32/docker/Dockerfile index a33d12f816..67deffff1c 100644 --- a/build/win32/docker/Dockerfile +++ b/build/win32/docker/Dockerfile @@ -86,21 +86,6 @@ RUN cmd.exe /C md \bin RUN cmd.exe /C copy "C:\Program Files\GIT\usr\bin" \bin > NUL RUN cmd.exe /C copy "C:\Program Files\GIT\usr\bin\echo.exe" \bin\echo > NUL -# disable tests that don't work on windows -ARG JQ_VERSION=1.7.1 -ARG JQ_BINARY=jq-windows-amd64.exe -ARG JQ_URL=https://github.com/jqlang/jq/releases/download/jq-${JQ_VERSION}/${JQ_BINARY} - -ARG JQ_BIN=C:\TEMP\jq.exe -ADD ${JQ_URL} ${JQ_BIN} - -WORKDIR ${MOD_SECURITY_DIR}\test\test-cases\regression - -RUN %JQ_BIN% "map(if .title == \"Test match variable (1/n)\" then .enabled = 0 else . end)" issue-2423-msg-in-chain.json > tmp.json && move /Y tmp.json issue-2423-msg-in-chain.json -RUN %JQ_BIN% "map(if .title == \"Test match variable (2/n)\" then .enabled = 0 else . end)" issue-2423-msg-in-chain.json > tmp.json && move /Y tmp.json issue-2423-msg-in-chain.json -RUN %JQ_BIN% "map(if .title == \"Test match variable (3/n)\" then .enabled = 0 else . end)" issue-2423-msg-in-chain.json > tmp.json && move /Y tmp.json issue-2423-msg-in-chain.json -RUN %JQ_BIN% "map(if .title == \"Variable offset - FILES_NAMES\" then .enabled = 0 else . end)" offset-variable.json > tmp.json && move /Y tmp.json offset-variable.json - # run tests WORKDIR ${MOD_SECURITY_DIR}\build\win32\build diff --git a/test/regression/regression.cc b/test/regression/regression.cc index 34d43b82a5..28e75e4196 100644 --- a/test/regression/regression.cc +++ b/test/regression/regression.cc @@ -341,7 +341,9 @@ void perform_unit_test(const ModSecurityTest &test, std::to_string(t->http_code) + " got: " + std::to_string(r.status) + "\n"; testRes->passed = false; - } else if (!contains(context.m_server_log.str(), t->error_log)) { + } else if (auto errit = std::find_if(t->error_log.begin(), t->error_log.end(), + [&context](const auto &x) { return !contains(context.m_server_log.str(), x); }); + errit != t->error_log.end()) { if (test.m_automake_output) { std::cout << ":test-result: FAIL " << filename \ << ":" << t->name << std::endl; @@ -351,7 +353,7 @@ void perform_unit_test(const ModSecurityTest &test, testRes->reason << "Error log was not matching the " \ << "expected results." << std::endl; testRes->reason << KWHT << "Expecting: " << RESET \ - << t->error_log + ""; + << *errit + ""; testRes->passed = false; } else if (!t->audit_log.empty() && !contains(getAuditLogContent(modsec_transaction.m_rules->m_auditLog->m_path1), t->audit_log)) { if (test.m_automake_output) { diff --git a/test/regression/regression_test.cc b/test/regression/regression_test.cc index 01ad2aacc9..5ac8601e1e 100644 --- a/test/regression/regression_test.cc +++ b/test/regression/regression_test.cc @@ -189,7 +189,14 @@ RegressionTest *RegressionTest::from_yajl_node(const yajl_val &node) { u->debug_log = YAJL_GET_STRING(val2); } if (strcmp(key2, "error_log") == 0) { - u->error_log = YAJL_GET_STRING(val2); + if (val2->u.array.len == 0) + u->error_log.insert(YAJL_GET_STRING(val2)); + else { + for (int k = 0; k < val2->u.array.len; k++) { + yajl_val vale = val2->u.array.values[k]; + u->error_log.insert(YAJL_GET_STRING(vale)); + } + } } if (strcmp(key2, "http_code") == 0) { u->http_code = YAJL_GET_INTEGER(val2); diff --git a/test/regression/regression_test.h b/test/regression/regression_test.h index eb37986723..de5919208d 100644 --- a/test/regression/regression_test.h +++ b/test/regression/regression_test.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -54,7 +55,7 @@ class RegressionTest { std::string audit_log; std::string debug_log; - std::string error_log; + std::set error_log; std::string parser_error; std::string clientIp; diff --git a/test/test-cases/regression/issue-2423-msg-in-chain.json b/test/test-cases/regression/issue-2423-msg-in-chain.json index c667de0542..8f43d20aa9 100644 --- a/test/test-cases/regression/issue-2423-msg-in-chain.json +++ b/test/test-cases/regression/issue-2423-msg-in-chain.json @@ -14,7 +14,6 @@ }, "request":{ "headers":{ - "Host":"localhost", "Transfer-Encoding": "deflate" }, "uri":"/match-this", @@ -45,7 +44,6 @@ }, "request":{ "headers":{ - "Host":"localhost", "Transfer-Encoding": "deflate" }, "uri":"/match-this", @@ -76,7 +74,6 @@ }, "request":{ "headers":{ - "Host":"localhost", "Transfer-Encoding": "deflate" }, "uri":"/match-this", diff --git a/test/test-cases/regression/offset-variable.json b/test/test-cases/regression/offset-variable.json index 7ffe9299ba..d454e21d12 100644 --- a/test/test-cases/regression/offset-variable.json +++ b/test/test-cases/regression/offset-variable.json @@ -1509,7 +1509,10 @@ ] }, "expected":{ - "error_log":"o0,8o0,8v491,8t:trimo0,16o0,16v709,16t:trim" + "error_log":[ + "o0,8o0,8v491,8t:trim", + "o0,16o0,16v709,16t:trim" + ] }, "rules":[ "SecRequestBodyAccess On",