Skip to content

Commit

Permalink
json_pointer::array_index: Use unsigned values for the array index wh…
Browse files Browse the repository at this point in the history 
…en parsing

The current code uses std::stoi to convert the input string to an int
array_index. This limits the maximum addressable array size to ~2GB on
most platforms.

But all callers immediately convert the result of array_index to
BasicJsonType::size_type.

So let's parse it as unsigned long long, which allows us to have as
big arrays as available memory. And also makes the call sites nicer to
read.

One complication arises on platforms where size_type is smaller than
unsigned long long. We need to bail out on these if the parsed array
index does not fit into size_type.
  • Loading branch information
@t-b
t-b committed Jun 22, 2020
1 parent f0e7316 commit ecbb275
Show file tree
Hide file tree
Showing 4 changed files with 69 additions and 34 deletions.
33 changes: 21 additions & 12 deletions include/nlohmann/detail/json_pointer.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
#include <algorithm> // all_of
#include <cassert> // assert
#include <cctype> // isdigit
#include <limits> // max
#include <numeric> // accumulate
#include <string> // string
#include <utility> // move
Expand Down Expand Up @@ -328,9 +329,12 @@ class json_pointer
@throw parse_error.106 if an array index begins with '0'
@throw parse_error.109 if an array index begins not with a digit
@throw out_of_range.404 if string @a s could not be converted to an integer
@throw out_of_range.410 if an array index exceeds size_type
*/
static int array_index(const std::string& s)
static typename BasicJsonType::size_type array_index(const std::string& s)
{
using size_type = typename BasicJsonType::size_type;

// error condition (cf. RFC 6901, Sect. 4)
if (JSON_HEDLEY_UNLIKELY(s.size() > 1 and s[0] == '0'))
{
Expand All @@ -346,10 +350,10 @@ class json_pointer
}

std::size_t processed_chars = 0;
int res = 0;
unsigned long long res = 0;
JSON_TRY
{
res = std::stoi(s, &processed_chars);
res = std::stoull(s, &processed_chars);
}
JSON_CATCH(std::out_of_range&)
{
Expand All @@ -362,7 +366,14 @@ class json_pointer
JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + s + "'"));
}

return res;
// only triggered on special platforms (like 32bit), see also
// https://github.com/nlohmann/json/pull/2203
if (res >= static_cast<unsigned long long>((std::numeric_limits<size_type>::max)()))
{
JSON_THROW(detail::out_of_range::create(410, "array index " + s + " exceeds size_type")); // LCOV_EXCL_LINE
}

return static_cast<size_type>(res);
}

json_pointer top() const
Expand Down Expand Up @@ -421,7 +432,7 @@ class json_pointer
case detail::value_t::array:
{
// create an entry in the array
result = &result->operator[](static_cast<size_type>(array_index(reference_token)));
result = &result->operator[](array_index(reference_token));
break;
}

Expand Down Expand Up @@ -499,8 +510,7 @@ class json_pointer
else
{
// convert array index to number; unchecked access
ptr = &ptr->operator[](
static_cast<size_type>(array_index(reference_token)));
ptr = &ptr->operator[](array_index(reference_token));
}
break;
}
Expand Down Expand Up @@ -544,7 +554,7 @@ class json_pointer
}

// note: at performs range check
ptr = &ptr->at(static_cast<size_type>(array_index(reference_token)));
ptr = &ptr->at(array_index(reference_token));
break;
}

Expand Down Expand Up @@ -594,8 +604,7 @@ class json_pointer
}

// use unchecked array access
ptr = &ptr->operator[](
static_cast<size_type>(array_index(reference_token)));
ptr = &ptr->operator[](array_index(reference_token));
break;
}

Expand Down Expand Up @@ -638,7 +647,7 @@ class json_pointer
}

// note: at performs range check
ptr = &ptr->at(static_cast<size_type>(array_index(reference_token)));
ptr = &ptr->at(array_index(reference_token));
break;
}

Expand Down Expand Up @@ -702,7 +711,7 @@ class json_pointer
}
}

const auto idx = static_cast<size_type>(array_index(reference_token));
const auto idx = array_index(reference_token);
if (idx >= ptr->size())
{
// index out of range
Expand Down
4 changes: 2 additions & 2 deletions include/nlohmann/json.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8179,7 +8179,7 @@ class basic_json
else
{
const auto idx = json_pointer::array_index(last_path);
if (JSON_HEDLEY_UNLIKELY(static_cast<size_type>(idx) > parent.size()))
if (JSON_HEDLEY_UNLIKELY(idx > parent.size()))
{
// avoid undefined behavior
JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range"));
Expand Down Expand Up @@ -8222,7 +8222,7 @@ class basic_json
else if (parent.is_array())
{
// note erase performs range check
parent.erase(static_cast<size_type>(json_pointer::array_index(last_path)));
parent.erase(json_pointer::array_index(last_path));
}
};

Expand Down
37 changes: 23 additions & 14 deletions single_include/nlohmann/json.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11032,6 +11032,7 @@ class json_reverse_iterator : public std::reverse_iterator<Base>
#include <algorithm> // all_of
#include <cassert> // assert
#include <cctype> // isdigit
#include <limits> // max
#include <numeric> // accumulate
#include <string> // string
#include <utility> // move
Expand Down Expand Up @@ -11360,9 +11361,12 @@ class json_pointer
@throw parse_error.106 if an array index begins with '0'
@throw parse_error.109 if an array index begins not with a digit
@throw out_of_range.404 if string @a s could not be converted to an integer
@throw out_of_range.410 if an array index exceeds size_type
*/
static int array_index(const std::string& s)
static typename BasicJsonType::size_type array_index(const std::string& s)
{
using size_type = typename BasicJsonType::size_type;

// error condition (cf. RFC 6901, Sect. 4)
if (JSON_HEDLEY_UNLIKELY(s.size() > 1 and s[0] == '0'))
{
Expand All @@ -11378,10 +11382,10 @@ class json_pointer
}

std::size_t processed_chars = 0;
int res = 0;
unsigned long long res = 0;
JSON_TRY
{
res = std::stoi(s, &processed_chars);
res = std::stoull(s, &processed_chars);
}
JSON_CATCH(std::out_of_range&)
{
Expand All @@ -11394,7 +11398,14 @@ class json_pointer
JSON_THROW(detail::out_of_range::create(404, "unresolved reference token '" + s + "'"));
}

return res;
// only triggered on special platforms (like 32bit), see also
// https://github.com/nlohmann/json/pull/2203
if (res >= static_cast<unsigned long long>((std::numeric_limits<size_type>::max)()))
{
JSON_THROW(detail::out_of_range::create(410, "array index " + s + " exceeds size_type")); // LCOV_EXCL_LINE
}

return static_cast<size_type>(res);
}

json_pointer top() const
Expand Down Expand Up @@ -11453,7 +11464,7 @@ class json_pointer
case detail::value_t::array:
{
// create an entry in the array
result = &result->operator[](static_cast<size_type>(array_index(reference_token)));
result = &result->operator[](array_index(reference_token));
break;
}

Expand Down Expand Up @@ -11531,8 +11542,7 @@ class json_pointer
else
{
// convert array index to number; unchecked access
ptr = &ptr->operator[](
static_cast<size_type>(array_index(reference_token)));
ptr = &ptr->operator[](array_index(reference_token));
}
break;
}
Expand Down Expand Up @@ -11576,7 +11586,7 @@ class json_pointer
}

// note: at performs range check
ptr = &ptr->at(static_cast<size_type>(array_index(reference_token)));
ptr = &ptr->at(array_index(reference_token));
break;
}

Expand Down Expand Up @@ -11626,8 +11636,7 @@ class json_pointer
}

// use unchecked array access
ptr = &ptr->operator[](
static_cast<size_type>(array_index(reference_token)));
ptr = &ptr->operator[](array_index(reference_token));
break;
}

Expand Down Expand Up @@ -11670,7 +11679,7 @@ class json_pointer
}

// note: at performs range check
ptr = &ptr->at(static_cast<size_type>(array_index(reference_token)));
ptr = &ptr->at(array_index(reference_token));
break;
}

Expand Down Expand Up @@ -11734,7 +11743,7 @@ class json_pointer
}
}

const auto idx = static_cast<size_type>(array_index(reference_token));
const auto idx = array_index(reference_token);
if (idx >= ptr->size())
{
// index out of range
Expand Down Expand Up @@ -23971,7 +23980,7 @@ class basic_json
else
{
const auto idx = json_pointer::array_index(last_path);
if (JSON_HEDLEY_UNLIKELY(static_cast<size_type>(idx) > parent.size()))
if (JSON_HEDLEY_UNLIKELY(idx > parent.size()))
{
// avoid undefined behavior
JSON_THROW(out_of_range::create(401, "array index " + std::to_string(idx) + " is out of range"));
Expand Down Expand Up @@ -24014,7 +24023,7 @@ class basic_json
else if (parent.is_array())
{
// note erase performs range check
parent.erase(static_cast<size_type>(json_pointer::array_index(last_path)));
parent.erase(json_pointer::array_index(last_path));
}
};

Expand Down
29 changes: 23 additions & 6 deletions test/src/unit-json_pointer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -348,12 +348,29 @@ TEST_CASE("JSON pointers")
CHECK_THROWS_WITH(j_const["/1+1"_json_pointer] == 1,
"[json.exception.out_of_range.404] unresolved reference token '1+1'");

CHECK_THROWS_AS(j["/111111111111111111111111"_json_pointer] = 1, json::out_of_range&);
CHECK_THROWS_WITH(j["/111111111111111111111111"_json_pointer] = 1,
"[json.exception.out_of_range.404] unresolved reference token '111111111111111111111111'");
CHECK_THROWS_AS(j_const["/111111111111111111111111"_json_pointer] == 1, json::out_of_range&);
CHECK_THROWS_WITH(j_const["/111111111111111111111111"_json_pointer] == 1,
"[json.exception.out_of_range.404] unresolved reference token '111111111111111111111111'");
{
auto too_large_index = std::to_string((std::numeric_limits<unsigned long long>::max)()) + "1";
json::json_pointer jp(std::string("/") + too_large_index);
std::string throw_msg = std::string("[json.exception.out_of_range.404] unresolved reference token '") + too_large_index + "'";

CHECK_THROWS_AS(j[jp] = 1, json::out_of_range&);
CHECK_THROWS_WITH(j[jp] = 1, throw_msg.c_str());
CHECK_THROWS_AS(j_const[jp] == 1, json::out_of_range&);
CHECK_THROWS_WITH(j_const[jp] == 1, throw_msg.c_str());
}

if (sizeof(typename json::size_type) < sizeof(unsigned long long))
{
auto size_type_max_uul = static_cast<unsigned long long>((std::numeric_limits<json::size_type>::max)());
auto too_large_index = std::to_string(size_type_max_uul);
json::json_pointer jp(std::string("/") + too_large_index);
std::string throw_msg = std::string("[json.exception.out_of_range.410] array index ") + too_large_index + " exceeds size_type";

CHECK_THROWS_AS(j[jp] = 1, json::out_of_range&);
CHECK_THROWS_WITH(j[jp] = 1, throw_msg.c_str());
CHECK_THROWS_AS(j_const[jp] == 1, json::out_of_range&);
CHECK_THROWS_WITH(j_const[jp] == 1, throw_msg.c_str());
}

CHECK_THROWS_AS(j.at("/one"_json_pointer) = 1, json::parse_error&);
CHECK_THROWS_WITH(j.at("/one"_json_pointer) = 1,
Expand Down

0 comments on commit ecbb275

Please sign in to comment.