From fbd2ef26551edd266cd6cda1a195296c052fc2a3 Mon Sep 17 00:00:00 2001 From: "Damien Buhl (alias daminetreg)" Date: Wed, 18 Aug 2021 08:50:03 +0200 Subject: [PATCH 1/5] :wrench: Handle padding correctly instead of all encoded \0 --- test/base64.cpp | 31 +++++++++++++++++++++++++++++++ xxhr/util.hpp | 12 +++++++++--- 2 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 test/base64.cpp diff --git a/test/base64.cpp b/test/base64.cpp new file mode 100644 index 000000000..1aa572bd0 --- /dev/null +++ b/test/base64.cpp @@ -0,0 +1,31 @@ +#define BOOST_TEST_MODULE base64 +#include + +#include + +std::map test_data { + { "", "" }, + { "TQ==" , "M" }, + { "TWE=" , "Ma" }, + { "TWFu" , "Man" }, + { "cGxlYXN1cmUu" , "pleasure." }, + { "bGVhc3VyZS4=" , "leasure." }, + { "ZWFzdXJlLg==" , "easure." }, + { "YXN1cmUu" , "asure." }, + { "c3VyZS4=" , "sure." }, + { "c3VyZS4=" , "sure." }, + { "AQABAFEAAAD5AAAAAA==", { 0x01, 0x00, 0x01, 0x00 0x51 0x00, 0x00, 0x00, 0xf9, 0x00, 0x00, 0x00, 0x00 } } +}; + + +BOOST_AUTO_TEST_CASE(decode) { + for (const auto& data : test_data) { + BOOST_REQUIRE_EQUAL(xxhr::util::decode64(data.first), data.second); + } +} + +BOOST_AUTO_TEST_CASE(encode) { + for (const auto& data : test_data) { + BOOST_REQUIRE_EQUAL(xxhr::util::encode64(data.second), data.first); + } +} diff --git a/xxhr/util.hpp b/xxhr/util.hpp index d0af9f5a5..d9597eabd 100644 --- a/xxhr/util.hpp +++ b/xxhr/util.hpp @@ -168,11 +168,17 @@ namespace util { inline std::string decode64(const std::string &val) { using namespace boost::archive::iterators; using It = transform_width, 8, 6>; - return boost::algorithm::trim_right_copy_if(std::string(It(std::begin(val)), It(std::end(val))), [](char c) { - return c == '\0'; - }); + // Remove padding and not all encoded \0 + // See https://svn.boost.org/trac10/ticket/5629#comment:9 + auto decoded_with_padding_chars_as_nulls = std::string(It(std::begin(val)), It(std::end(val))); + auto padding_count = std::count(val.end()-((std::min)(2, val.size())), val.end()); + auto decoded = decoded_with_padding_chars_as_nulls.resize(decoded_with_padding_chars_as_nulls.size()-decoded_with_padding_chars_as_nulls); + decoded_with_padding_chars_as_nulls.erase(decoded.end() - padding_count, '='), decoded.end()); + auto decoded = std::string(decoded_with_padding_chars_as_nulls + return ; } + inline std::string encode64(const std::string &val) { using namespace boost::archive::iterators; using It = base64_from_binary>; From 513239dfff73111f2ced6fdb085e6576e746fcfd Mon Sep 17 00:00:00 2001 From: "Damien Buhl (alias daminetreg)" Date: Wed, 18 Aug 2021 09:20:53 +0200 Subject: [PATCH 2/5] :wrench: Handle padding correctly instead of removing all encoded \0 at the end. --- test/base64.cpp | 6 +++++- xxhr/util.hpp | 12 +++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/test/base64.cpp b/test/base64.cpp index 1aa572bd0..3eae12f55 100644 --- a/test/base64.cpp +++ b/test/base64.cpp @@ -14,18 +14,22 @@ std::map test_data { { "YXN1cmUu" , "asure." }, { "c3VyZS4=" , "sure." }, { "c3VyZS4=" , "sure." }, - { "AQABAFEAAAD5AAAAAA==", { 0x01, 0x00, 0x01, 0x00 0x51 0x00, 0x00, 0x00, 0xf9, 0x00, 0x00, 0x00, 0x00 } } + { "AQABAFEAAAD5AAAAAA==", { 0x01, 0x00, 0x01, 0x00, 0x51, 0x00, 0x00, 0x00, char(0xf9), 0x00, 0x00, 0x00, 0x00 } } }; BOOST_AUTO_TEST_CASE(decode) { + BOOST_REQUIRE(test_data.size() > 0); for (const auto& data : test_data) { + std::cout << data.first << " should be " << data.second << std::endl; BOOST_REQUIRE_EQUAL(xxhr::util::decode64(data.first), data.second); } } BOOST_AUTO_TEST_CASE(encode) { + BOOST_REQUIRE(test_data.size() > 0); for (const auto& data : test_data) { + std::cout << data.first << " should be " << data.second << std::endl; BOOST_REQUIRE_EQUAL(xxhr::util::encode64(data.second), data.first); } } diff --git a/xxhr/util.hpp b/xxhr/util.hpp index d9597eabd..44018493b 100644 --- a/xxhr/util.hpp +++ b/xxhr/util.hpp @@ -168,14 +168,12 @@ namespace util { inline std::string decode64(const std::string &val) { using namespace boost::archive::iterators; using It = transform_width, 8, 6>; - // Remove padding and not all encoded \0 // See https://svn.boost.org/trac10/ticket/5629#comment:9 - auto decoded_with_padding_chars_as_nulls = std::string(It(std::begin(val)), It(std::end(val))); - auto padding_count = std::count(val.end()-((std::min)(2, val.size())), val.end()); - auto decoded = decoded_with_padding_chars_as_nulls.resize(decoded_with_padding_chars_as_nulls.size()-decoded_with_padding_chars_as_nulls); - decoded_with_padding_chars_as_nulls.erase(decoded.end() - padding_count, '='), decoded.end()); - auto decoded = std::string(decoded_with_padding_chars_as_nulls - return ; + // Boost binary_from_base64 transforms '=' into '\0', they need to be removed to support binary data + auto decoded_with_zeroed_padding = std::string(It(std::begin(val)), It(std::end(val))); + auto padding_count = std::count(val.end() - std::min(std::size_t{2}, val.size()), val.end() , '='); + auto decoded = decoded_with_zeroed_padding.substr(0,decoded_with_zeroed_padding.size()-padding_count); + return decoded; } From 1d154c1b960f767dccda4ad15f95678a85ca47c2 Mon Sep 17 00:00:00 2001 From: "Damien Buhl (alias daminetreg)" Date: Wed, 18 Aug 2021 11:15:11 +0200 Subject: [PATCH 3/5] :wrench: wasm doesn't provide sigaltstack impl, fix boost test on the platform. --- test/base64.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/test/base64.cpp b/test/base64.cpp index 3eae12f55..5107f7ccb 100644 --- a/test/base64.cpp +++ b/test/base64.cpp @@ -1,5 +1,6 @@ #define BOOST_TEST_MODULE base64 #include +#include "wasm_boost_test.hpp" #include From 44df7b30f28f0d1be51f2dcb0275ac030ecbafdd Mon Sep 17 00:00:00 2001 From: "Damien Buhl (alias daminetreg)" Date: Wed, 18 Aug 2021 13:40:26 +0200 Subject: [PATCH 4/5] :lipstick: compiles better with the missing file. --- test/wasm_boost_test.hpp | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 test/wasm_boost_test.hpp diff --git a/test/wasm_boost_test.hpp b/test/wasm_boost_test.hpp new file mode 100644 index 000000000..ab6176307 --- /dev/null +++ b/test/wasm_boost_test.hpp @@ -0,0 +1,9 @@ +#pragma once + + +#ifdef __EMSCRIPTEN__ + #include + extern "C" { + int sigaltstack(const stack_t *__restrict, stack_t *__restrict) { return 0; } //XXX: Check in newer emcc version if we can remove this. + } +#endif \ No newline at end of file From fb72f7da681209b1e6d9c944e0bc034d152e1cd9 Mon Sep 17 00:00:00 2001 From: "Damien Buhl (alias daminetreg)" Date: Wed, 25 Aug 2021 17:40:55 +0200 Subject: [PATCH 5/5] :lipstick: less allocation, simpler decode64 as of boost patch. --- test/base64.cpp | 4 ++-- xxhr/util.hpp | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/test/base64.cpp b/test/base64.cpp index 5107f7ccb..0d1448392 100644 --- a/test/base64.cpp +++ b/test/base64.cpp @@ -22,7 +22,7 @@ std::map test_data { BOOST_AUTO_TEST_CASE(decode) { BOOST_REQUIRE(test_data.size() > 0); for (const auto& data : test_data) { - std::cout << data.first << " should be " << data.second << std::endl; + BOOST_TEST_MESSAGE( data.first << " check if equals " << data.second); BOOST_REQUIRE_EQUAL(xxhr::util::decode64(data.first), data.second); } } @@ -30,7 +30,7 @@ BOOST_AUTO_TEST_CASE(decode) { BOOST_AUTO_TEST_CASE(encode) { BOOST_REQUIRE(test_data.size() > 0); for (const auto& data : test_data) { - std::cout << data.first << " should be " << data.second << std::endl; + BOOST_TEST_MESSAGE( data.first << " check if equals " << data.second); BOOST_REQUIRE_EQUAL(xxhr::util::encode64(data.second), data.first); } } diff --git a/xxhr/util.hpp b/xxhr/util.hpp index 44018493b..db86f6085 100644 --- a/xxhr/util.hpp +++ b/xxhr/util.hpp @@ -170,10 +170,8 @@ namespace util { using It = transform_width, 8, 6>; // See https://svn.boost.org/trac10/ticket/5629#comment:9 // Boost binary_from_base64 transforms '=' into '\0', they need to be removed to support binary data - auto decoded_with_zeroed_padding = std::string(It(std::begin(val)), It(std::end(val))); auto padding_count = std::count(val.end() - std::min(std::size_t{2}, val.size()), val.end() , '='); - auto decoded = decoded_with_zeroed_padding.substr(0,decoded_with_zeroed_padding.size()-padding_count); - return decoded; + return std::string(It(std::begin(val)), It(std::end(val) - padding_count)); }