Skip to content

Commit

Permalink
Merge pull request #2201 from nlohmann/issue2196
Browse files Browse the repository at this point in the history 
Serialize floating-point numbers with 32 bit when possible (MessagePack)
  • Loading branch information
@nlohmann
nlohmann committed Jun 20, 2020
2 parents e7452d8 + b64002b commit 74c6e42
Show file tree
Hide file tree
Showing 6 changed files with 87 additions and 42 deletions.
2 changes: 1 addition & 1 deletion cmake/download_test_data.cmake
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
find_package(Git)

set(JSON_TEST_DATA_URL https://github.com/nlohmann/json_test_data)
set(JSON_TEST_DATA_VERSION 2.0.0)
set(JSON_TEST_DATA_VERSION 3.0.0)

# target to download test data
add_custom_target(download_test_data
Expand Down
7 changes: 2 additions & 5 deletions doc/mkdocs/docs/features/binary_formats/messagepack.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,8 @@ number_unsigned | 128..255 | uint 8 | 0xCC
number_unsigned | 256..65535 | uint 16 | 0xCD
number_unsigned | 65536..4294967295 | uint 32 | 0xCE
number_unsigned | 4294967296..18446744073709551615 | uint 64 | 0xCF
number_float | *any value* | float 64 | 0xCB
number_float | *any value representable by a float* | float 32 | 0xCA
number_float | *any value NOT representable by a float* | float 64 | 0xCB
string | *length*: 0..31 | fixstr | 0xA0..0xBF
string | *length*: 32..255 | str 8 | 0xD9
string | *length*: 256..65535 | str 16 | 0xDA
Expand Down Expand Up @@ -61,10 +62,6 @@ binary | *size*: 65536..4294967295 | bin 32 | 0xC6
- arrays with more than 4294967295 elements
- objects with more than 4294967295 elements

!!! info "Unused MessagePack types"

The following MessagePack types are not used in the conversion: float 32 (0xCA)

!!! info "NaN/infinity handling"

If NaN or Infinity are stored inside a JSON number, they are serialized properly. function which serializes NaN or Infinity to `null`.
Expand Down
37 changes: 23 additions & 14 deletions include/nlohmann/detail/output/binary_writer.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ class binary_writer
{
using string_t = typename BasicJsonType::string_t;
using binary_t = typename BasicJsonType::binary_t;
using number_float_t = typename BasicJsonType::number_float_t;

public:
/*!
Expand Down Expand Up @@ -194,18 +195,7 @@ class binary_writer
}
else
{
if (static_cast<double>(j.m_value.number_float) >= static_cast<double>(std::numeric_limits<float>::lowest()) and
static_cast<double>(j.m_value.number_float) <= static_cast<double>((std::numeric_limits<float>::max)()) and
static_cast<double>(static_cast<float>(j.m_value.number_float)) == static_cast<double>(j.m_value.number_float))
{
oa->write_character(get_cbor_float_prefix(static_cast<float>(j.m_value.number_float)));
write_number(static_cast<float>(j.m_value.number_float));
}
else
{
oa->write_character(get_cbor_float_prefix(j.m_value.number_float));
write_number(j.m_value.number_float);
}
write_compact_float(j.m_value.number_float, detail::input_format_t::cbor);
}
break;
}
Expand Down Expand Up @@ -504,8 +494,7 @@ class binary_writer

case value_t::number_float:
{
oa->write_character(get_msgpack_float_prefix(j.m_value.number_float));
write_number(j.m_value.number_float);
write_compact_float(j.m_value.number_float, detail::input_format_t::msgpack);
break;
}

Expand Down Expand Up @@ -1518,6 +1507,26 @@ class binary_writer
oa->write_characters(vec.data(), sizeof(NumberType));
}

void write_compact_float(const number_float_t n, detail::input_format_t format)
{
if (static_cast<double>(n) >= static_cast<double>(std::numeric_limits<float>::lowest()) and
static_cast<double>(n) <= static_cast<double>((std::numeric_limits<float>::max)()) and
static_cast<double>(static_cast<float>(n)) == static_cast<double>(n))
{
oa->write_character(format == detail::input_format_t::cbor
? get_cbor_float_prefix(static_cast<float>(n))
: get_msgpack_float_prefix(static_cast<float>(n)));
write_number(static_cast<float>(n));
}
else
{
oa->write_character(format == detail::input_format_t::cbor
? get_cbor_float_prefix(n)
: get_msgpack_float_prefix(n));
write_number(n);
}
}

public:
// The following to_char_type functions are implement the conversion
// between uint8_t and CharType. In case CharType is not unsigned,
Expand Down
6 changes: 2 additions & 4 deletions include/nlohmann/json.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7040,7 +7040,8 @@ class basic_json
number_unsigned | 256..65535 | uint 16 | 0xCD
number_unsigned | 65536..4294967295 | uint 32 | 0xCE
number_unsigned | 4294967296..18446744073709551615 | uint 64 | 0xCF
number_float | *any value* | float 64 | 0xCB
number_float | *any value representable by a float* | float 32 | 0xCA
number_float | *any value NOT representable by a float* | float 64 | 0xCB
string | *length*: 0..31 | fixstr | 0xA0..0xBF
string | *length*: 32..255 | str 8 | 0xD9
string | *length*: 256..65535 | str 16 | 0xDA
Expand All @@ -7064,9 +7065,6 @@ class basic_json
- arrays with more than 4294967295 elements
- objects with more than 4294967295 elements
@note The following MessagePack types are not used in the conversion:
- float 32 (0xCA)
@note Any MessagePack output created @ref to_msgpack can be successfully
parsed by @ref from_msgpack.
Expand Down
43 changes: 25 additions & 18 deletions single_include/nlohmann/json.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12238,6 +12238,7 @@ class binary_writer
{
using string_t = typename BasicJsonType::string_t;
using binary_t = typename BasicJsonType::binary_t;
using number_float_t = typename BasicJsonType::number_float_t;

public:
/*!
Expand Down Expand Up @@ -12404,18 +12405,7 @@ class binary_writer
}
else
{
if (static_cast<double>(j.m_value.number_float) >= static_cast<double>(std::numeric_limits<float>::lowest()) and
static_cast<double>(j.m_value.number_float) <= static_cast<double>((std::numeric_limits<float>::max)()) and
static_cast<double>(static_cast<float>(j.m_value.number_float)) == static_cast<double>(j.m_value.number_float))
{
oa->write_character(get_cbor_float_prefix(static_cast<float>(j.m_value.number_float)));
write_number(static_cast<float>(j.m_value.number_float));
}
else
{
oa->write_character(get_cbor_float_prefix(j.m_value.number_float));
write_number(j.m_value.number_float);
}
write_compact_float(j.m_value.number_float, detail::input_format_t::cbor);
}
break;
}
Expand Down Expand Up @@ -12714,8 +12704,7 @@ class binary_writer

case value_t::number_float:
{
oa->write_character(get_msgpack_float_prefix(j.m_value.number_float));
write_number(j.m_value.number_float);
write_compact_float(j.m_value.number_float, detail::input_format_t::msgpack);
break;
}

Expand Down Expand Up @@ -13728,6 +13717,26 @@ class binary_writer
oa->write_characters(vec.data(), sizeof(NumberType));
}

void write_compact_float(const number_float_t n, detail::input_format_t format)
{
if (static_cast<double>(n) >= static_cast<double>(std::numeric_limits<float>::lowest()) and
static_cast<double>(n) <= static_cast<double>((std::numeric_limits<float>::max)()) and
static_cast<double>(static_cast<float>(n)) == static_cast<double>(n))
{
oa->write_character(format == detail::input_format_t::cbor
? get_cbor_float_prefix(static_cast<float>(n))
: get_msgpack_float_prefix(static_cast<float>(n)));
write_number(static_cast<float>(n));
}
else
{
oa->write_character(format == detail::input_format_t::cbor
? get_cbor_float_prefix(n)
: get_msgpack_float_prefix(n));
write_number(n);
}
}

public:
// The following to_char_type functions are implement the conversion
// between uint8_t and CharType. In case CharType is not unsigned,
Expand Down Expand Up @@ -22821,7 +22830,8 @@ class basic_json
number_unsigned | 256..65535 | uint 16 | 0xCD
number_unsigned | 65536..4294967295 | uint 32 | 0xCE
number_unsigned | 4294967296..18446744073709551615 | uint 64 | 0xCF
number_float | *any value* | float 64 | 0xCB
number_float | *any value representable by a float* | float 32 | 0xCA
number_float | *any value NOT representable by a float* | float 64 | 0xCB
string | *length*: 0..31 | fixstr | 0xA0..0xBF
string | *length*: 32..255 | str 8 | 0xD9
string | *length*: 256..65535 | str 16 | 0xDA
Expand All @@ -22845,9 +22855,6 @@ class basic_json
- arrays with more than 4294967295 elements
- objects with more than 4294967295 elements

@note The following MessagePack types are not used in the conversion:
- float 32 (0xCA)

@note Any MessagePack output created @ref to_msgpack can be successfully
parsed by @ref from_msgpack.

Expand Down
34 changes: 34 additions & 0 deletions test/src/unit-msgpack.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -783,6 +783,40 @@ TEST_CASE("MessagePack")
CHECK(json::from_msgpack(result) == v);
CHECK(json::from_msgpack(result, true, false) == j);
}

SECTION("1.0")
{
double v = 1.0;
json j = v;
std::vector<uint8_t> expected =
{
0xca, 0x3f, 0x80, 0x00, 0x00
};
const auto result = json::to_msgpack(j);
CHECK(result == expected);

// roundtrip
CHECK(json::from_msgpack(result) == j);
CHECK(json::from_msgpack(result) == v);
CHECK(json::from_msgpack(result, true, false) == j);
}

SECTION("128.128")
{
double v = 128.1280059814453125;
json j = v;
std::vector<uint8_t> expected =
{
0xca, 0x43, 0x00, 0x20, 0xc5
};
const auto result = json::to_msgpack(j);
CHECK(result == expected);

// roundtrip
CHECK(json::from_msgpack(result) == j);
CHECK(json::from_msgpack(result) == v);
CHECK(json::from_msgpack(result, true, false) == j);
}
}
}

Expand Down

0 comments on commit 74c6e42

Please sign in to comment.