From 88a37010d6cfa808ddfa9559a15285b8291ad187 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Wed, 17 Jun 2020 21:14:23 +0200 Subject: [PATCH 1/3] :bug: serialize 32-bit floating-point numbers as float 32 in MessagePack (0xCA) #2196 --- .../features/binary_formats/messagepack.md | 7 ++-- .../nlohmann/detail/output/binary_writer.hpp | 14 ++++++-- include/nlohmann/json.hpp | 6 ++-- single_include/nlohmann/json.hpp | 20 +++++++---- test/src/unit-msgpack.cpp | 34 +++++++++++++++++++ 5 files changed, 64 insertions(+), 17 deletions(-) diff --git a/doc/mkdocs/docs/features/binary_formats/messagepack.md b/doc/mkdocs/docs/features/binary_formats/messagepack.md index ed06105693..3e041bb708 100644 --- a/doc/mkdocs/docs/features/binary_formats/messagepack.md +++ b/doc/mkdocs/docs/features/binary_formats/messagepack.md @@ -31,7 +31,8 @@ number_unsigned | 128..255 | uint 8 | 0xCC number_unsigned | 256..65535 | uint 16 | 0xCD number_unsigned | 65536..4294967295 | uint 32 | 0xCE number_unsigned | 4294967296..18446744073709551615 | uint 64 | 0xCF -number_float | *any value* | float 64 | 0xCB +number_float | *any value representable by a float* | float 32 | 0xCA +number_float | *any value NOT representable by a float* | float 64 | 0xCB string | *length*: 0..31 | fixstr | 0xA0..0xBF string | *length*: 32..255 | str 8 | 0xD9 string | *length*: 256..65535 | str 16 | 0xDA @@ -61,10 +62,6 @@ binary | *size*: 65536..4294967295 | bin 32 | 0xC6 - arrays with more than 4294967295 elements - objects with more than 4294967295 elements -!!! info "Unused MessagePack types" - - The following MessagePack types are not used in the conversion: float 32 (0xCA) - !!! info "NaN/infinity handling" If NaN or Infinity are stored inside a JSON number, they are serialized properly. function which serializes NaN or Infinity to `null`. diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index 269df0d554..f322119119 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -504,8 +504,18 @@ class binary_writer case value_t::number_float: { - oa->write_character(get_msgpack_float_prefix(j.m_value.number_float)); - write_number(j.m_value.number_float); + if (static_cast(j.m_value.number_float) >= static_cast(std::numeric_limits::lowest()) and + static_cast(j.m_value.number_float) <= static_cast((std::numeric_limits::max)()) and + static_cast(static_cast(j.m_value.number_float)) == static_cast(j.m_value.number_float)) + { + oa->write_character(get_msgpack_float_prefix(static_cast(j.m_value.number_float))); + write_number(static_cast(j.m_value.number_float)); + } + else + { + oa->write_character(get_msgpack_float_prefix(j.m_value.number_float)); + write_number(j.m_value.number_float); + } break; } diff --git a/include/nlohmann/json.hpp b/include/nlohmann/json.hpp index 409a6e7993..790ecd4b25 100644 --- a/include/nlohmann/json.hpp +++ b/include/nlohmann/json.hpp @@ -7040,7 +7040,8 @@ class basic_json number_unsigned | 256..65535 | uint 16 | 0xCD number_unsigned | 65536..4294967295 | uint 32 | 0xCE number_unsigned | 4294967296..18446744073709551615 | uint 64 | 0xCF - number_float | *any value* | float 64 | 0xCB + number_float | *any value representable by a float* | float 32 | 0xCA + number_float | *any value NOT representable by a float* | float 64 | 0xCB string | *length*: 0..31 | fixstr | 0xA0..0xBF string | *length*: 32..255 | str 8 | 0xD9 string | *length*: 256..65535 | str 16 | 0xDA @@ -7064,9 +7065,6 @@ class basic_json - arrays with more than 4294967295 elements - objects with more than 4294967295 elements - @note The following MessagePack types are not used in the conversion: - - float 32 (0xCA) - @note Any MessagePack output created @ref to_msgpack can be successfully parsed by @ref from_msgpack. diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index cc822a543b..85fffc7c7e 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -12714,8 +12714,18 @@ class binary_writer case value_t::number_float: { - oa->write_character(get_msgpack_float_prefix(j.m_value.number_float)); - write_number(j.m_value.number_float); + if (static_cast(j.m_value.number_float) >= static_cast(std::numeric_limits::lowest()) and + static_cast(j.m_value.number_float) <= static_cast((std::numeric_limits::max)()) and + static_cast(static_cast(j.m_value.number_float)) == static_cast(j.m_value.number_float)) + { + oa->write_character(get_msgpack_float_prefix(static_cast(j.m_value.number_float))); + write_number(static_cast(j.m_value.number_float)); + } + else + { + oa->write_character(get_msgpack_float_prefix(j.m_value.number_float)); + write_number(j.m_value.number_float); + } break; } @@ -22821,7 +22831,8 @@ class basic_json number_unsigned | 256..65535 | uint 16 | 0xCD number_unsigned | 65536..4294967295 | uint 32 | 0xCE number_unsigned | 4294967296..18446744073709551615 | uint 64 | 0xCF - number_float | *any value* | float 64 | 0xCB + number_float | *any value representable by a float* | float 32 | 0xCA + number_float | *any value NOT representable by a float* | float 64 | 0xCB string | *length*: 0..31 | fixstr | 0xA0..0xBF string | *length*: 32..255 | str 8 | 0xD9 string | *length*: 256..65535 | str 16 | 0xDA @@ -22845,9 +22856,6 @@ class basic_json - arrays with more than 4294967295 elements - objects with more than 4294967295 elements - @note The following MessagePack types are not used in the conversion: - - float 32 (0xCA) - @note Any MessagePack output created @ref to_msgpack can be successfully parsed by @ref from_msgpack. diff --git a/test/src/unit-msgpack.cpp b/test/src/unit-msgpack.cpp index 2744a67bf7..5baecf0274 100644 --- a/test/src/unit-msgpack.cpp +++ b/test/src/unit-msgpack.cpp @@ -783,6 +783,40 @@ TEST_CASE("MessagePack") CHECK(json::from_msgpack(result) == v); CHECK(json::from_msgpack(result, true, false) == j); } + + SECTION("1.0") + { + double v = 1.0; + json j = v; + std::vector expected = + { + 0xca, 0x3f, 0x80, 0x00, 0x00 + }; + const auto result = json::to_msgpack(j); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_msgpack(result) == j); + CHECK(json::from_msgpack(result) == v); + CHECK(json::from_msgpack(result, true, false) == j); + } + + SECTION("128.128") + { + double v = 128.1280059814453125; + json j = v; + std::vector expected = + { + 0xca, 0x43, 0x00, 0x20, 0xc5 + }; + const auto result = json::to_msgpack(j); + CHECK(result == expected); + + // roundtrip + CHECK(json::from_msgpack(result) == j); + CHECK(json::from_msgpack(result) == v); + CHECK(json::from_msgpack(result, true, false) == j); + } } } From cd115cbc33d074e231d3a703c8120b72223cb1e4 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Thu, 18 Jun 2020 12:50:32 +0200 Subject: [PATCH 2/3] :white_check_mark: update test suite --- cmake/download_test_data.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/download_test_data.cmake b/cmake/download_test_data.cmake index fe95235f89..b023d8b458 100644 --- a/cmake/download_test_data.cmake +++ b/cmake/download_test_data.cmake @@ -1,7 +1,7 @@ find_package(Git) set(JSON_TEST_DATA_URL https://github.com/nlohmann/json_test_data) -set(JSON_TEST_DATA_VERSION 2.0.0) +set(JSON_TEST_DATA_VERSION 3.0.0) # target to download test data add_custom_target(download_test_data From b64002bbca497e79f56466ced8f99f90ca9cf347 Mon Sep 17 00:00:00 2001 From: Niels Lohmann Date: Fri, 19 Jun 2020 13:24:08 +0200 Subject: [PATCH 3/3] :recycle: extract common code to function --- .../nlohmann/detail/output/binary_writer.hpp | 47 +++++++++---------- single_include/nlohmann/json.hpp | 47 +++++++++---------- 2 files changed, 46 insertions(+), 48 deletions(-) diff --git a/include/nlohmann/detail/output/binary_writer.hpp b/include/nlohmann/detail/output/binary_writer.hpp index f322119119..7289da3dce 100644 --- a/include/nlohmann/detail/output/binary_writer.hpp +++ b/include/nlohmann/detail/output/binary_writer.hpp @@ -28,6 +28,7 @@ class binary_writer { using string_t = typename BasicJsonType::string_t; using binary_t = typename BasicJsonType::binary_t; + using number_float_t = typename BasicJsonType::number_float_t; public: /*! @@ -194,18 +195,7 @@ class binary_writer } else { - if (static_cast(j.m_value.number_float) >= static_cast(std::numeric_limits::lowest()) and - static_cast(j.m_value.number_float) <= static_cast((std::numeric_limits::max)()) and - static_cast(static_cast(j.m_value.number_float)) == static_cast(j.m_value.number_float)) - { - oa->write_character(get_cbor_float_prefix(static_cast(j.m_value.number_float))); - write_number(static_cast(j.m_value.number_float)); - } - else - { - oa->write_character(get_cbor_float_prefix(j.m_value.number_float)); - write_number(j.m_value.number_float); - } + write_compact_float(j.m_value.number_float, detail::input_format_t::cbor); } break; } @@ -504,18 +494,7 @@ class binary_writer case value_t::number_float: { - if (static_cast(j.m_value.number_float) >= static_cast(std::numeric_limits::lowest()) and - static_cast(j.m_value.number_float) <= static_cast((std::numeric_limits::max)()) and - static_cast(static_cast(j.m_value.number_float)) == static_cast(j.m_value.number_float)) - { - oa->write_character(get_msgpack_float_prefix(static_cast(j.m_value.number_float))); - write_number(static_cast(j.m_value.number_float)); - } - else - { - oa->write_character(get_msgpack_float_prefix(j.m_value.number_float)); - write_number(j.m_value.number_float); - } + write_compact_float(j.m_value.number_float, detail::input_format_t::msgpack); break; } @@ -1528,6 +1507,26 @@ class binary_writer oa->write_characters(vec.data(), sizeof(NumberType)); } + void write_compact_float(const number_float_t n, detail::input_format_t format) + { + if (static_cast(n) >= static_cast(std::numeric_limits::lowest()) and + static_cast(n) <= static_cast((std::numeric_limits::max)()) and + static_cast(static_cast(n)) == static_cast(n)) + { + oa->write_character(format == detail::input_format_t::cbor + ? get_cbor_float_prefix(static_cast(n)) + : get_msgpack_float_prefix(static_cast(n))); + write_number(static_cast(n)); + } + else + { + oa->write_character(format == detail::input_format_t::cbor + ? get_cbor_float_prefix(n) + : get_msgpack_float_prefix(n)); + write_number(n); + } + } + public: // The following to_char_type functions are implement the conversion // between uint8_t and CharType. In case CharType is not unsigned, diff --git a/single_include/nlohmann/json.hpp b/single_include/nlohmann/json.hpp index 85fffc7c7e..33d0c35776 100644 --- a/single_include/nlohmann/json.hpp +++ b/single_include/nlohmann/json.hpp @@ -12238,6 +12238,7 @@ class binary_writer { using string_t = typename BasicJsonType::string_t; using binary_t = typename BasicJsonType::binary_t; + using number_float_t = typename BasicJsonType::number_float_t; public: /*! @@ -12404,18 +12405,7 @@ class binary_writer } else { - if (static_cast(j.m_value.number_float) >= static_cast(std::numeric_limits::lowest()) and - static_cast(j.m_value.number_float) <= static_cast((std::numeric_limits::max)()) and - static_cast(static_cast(j.m_value.number_float)) == static_cast(j.m_value.number_float)) - { - oa->write_character(get_cbor_float_prefix(static_cast(j.m_value.number_float))); - write_number(static_cast(j.m_value.number_float)); - } - else - { - oa->write_character(get_cbor_float_prefix(j.m_value.number_float)); - write_number(j.m_value.number_float); - } + write_compact_float(j.m_value.number_float, detail::input_format_t::cbor); } break; } @@ -12714,18 +12704,7 @@ class binary_writer case value_t::number_float: { - if (static_cast(j.m_value.number_float) >= static_cast(std::numeric_limits::lowest()) and - static_cast(j.m_value.number_float) <= static_cast((std::numeric_limits::max)()) and - static_cast(static_cast(j.m_value.number_float)) == static_cast(j.m_value.number_float)) - { - oa->write_character(get_msgpack_float_prefix(static_cast(j.m_value.number_float))); - write_number(static_cast(j.m_value.number_float)); - } - else - { - oa->write_character(get_msgpack_float_prefix(j.m_value.number_float)); - write_number(j.m_value.number_float); - } + write_compact_float(j.m_value.number_float, detail::input_format_t::msgpack); break; } @@ -13738,6 +13717,26 @@ class binary_writer oa->write_characters(vec.data(), sizeof(NumberType)); } + void write_compact_float(const number_float_t n, detail::input_format_t format) + { + if (static_cast(n) >= static_cast(std::numeric_limits::lowest()) and + static_cast(n) <= static_cast((std::numeric_limits::max)()) and + static_cast(static_cast(n)) == static_cast(n)) + { + oa->write_character(format == detail::input_format_t::cbor + ? get_cbor_float_prefix(static_cast(n)) + : get_msgpack_float_prefix(static_cast(n))); + write_number(static_cast(n)); + } + else + { + oa->write_character(format == detail::input_format_t::cbor + ? get_cbor_float_prefix(n) + : get_msgpack_float_prefix(n)); + write_number(n); + } + } + public: // The following to_char_type functions are implement the conversion // between uint8_t and CharType. In case CharType is not unsigned,