Add support for non-contiguous buffers within JSON builder using a ca…

…llback mechanism. (Azure#902)

* Add support for non-contiguous buffers within JSON builder using a
callback mechanism.

* Reword a comment.

* Adding the callback method signature on the top of the test file to fix
missing-declarations warning.

* Add tests where allocator returns null or first destination is null.

* Move init to be a suffix in chunked API.

* Change the init APIs to accept void* user_context instead of the
allocator callback struct.

* Revert "Change the init APIs to accept void* user_context instead of the"

This reverts commit 35f3ccb92e44cd3cae4610f43aafaef298d815ef.

* Revert "Move init to be a suffix in chunked API."

This reverts commit d58bb4b2a4a57363595b5165287b8a6f4fca02a9.

* Revert "Add tests where allocator returns null or first destination is null."

This reverts commit 2358db8a6da5f6a3431995177a168e81ae3613a1.

* Revert "Merge branch 'master' of https://github.com/Azure/azure-sdk-for-c into BuilderNonContiguous"

This reverts commit 542ccb35a10114aa61903e55704251ed7bdeaa70, reversing
changes made to dd03bf2c7037ffa2c29fb81481de55c7fe22ec69.

* Revert "Adding the callback method signature on the top of the test file to fix"

This reverts commit dd03bf2c7037ffa2c29fb81481de55c7fe22ec69.

* Revert "Reword a comment."

This reverts commit 4fa831667525fafd725550205a0fb3d01aba8a7f.

* Revert "Add support for non-contiguous buffers within JSON builder using a"

This reverts commit d88dd9557be60ba732342a6e1855fda832bf5808.

* Re-apply all the changes on top of master, after the recent rename and
API updates.

* Update allocator context field names, and flip remaining_size to
bytes_used.

* Update get_json API name and simplify internal fields of the writer
struct.

* Enable appending strings in limited size chunks, rather than requiring
the entire destination up-front.

* Remove unused local, fix some spacing, formatting, and typos.

sdk/inc/azure/core/az_json.h

@@ -227,6 +227,10 @@ typedef struct
   {
     az_span destination_buffer;
     int32_t bytes_written;
+    // For single contiguous buffer, bytes_written == total_bytes_written
+    int32_t total_bytes_written; // Currently, this is primarily used for testing.
+    az_span_allocator_fn allocator_callback;
+    void* user_context;
     bool need_comma;
     az_json_token_kind token_kind; // needed for validation, potentially #if/def with preconditions.
     _az_json_bit_stack bit_stack; // needed for validation, potentially #if/def with preconditions.
@@ -253,17 +257,50 @@ AZ_NODISCARD az_result az_json_writer_init(
     az_json_writer_options const* options);
 
 /**
- * @brief Returns the #az_span containing the JSON text written to the underlying buffer so far.
+ * @brief Initializes an #az_json_writer which writes JSON text into a destination that can contain
+ * non-contiguous buffers.
  *
- * @param[in] json_writer A pointer to an #az_json_writer instance wrapping the destination
- * buffer.
+ * @param[out] json_writer A pointer to an #az_json_writer the instance to initialize.
+ * @param[in] first_destination_buffer An #az_span over the byte buffer where the JSON text is to be
+ * written at the start.
+ * @param[in] allocator_callback An #az_span_allocator_fn callback function that provides the
+ * destination span to write the JSON text to once the previous buffer is full or too small to
+ * contain the next token.
+ * @param[in] user_context A context specific user-defined struct or set of fields that is passed
+ * through to calls to the #az_span_allocator_fn.
+ * @param[in] options __[nullable]__ A reference to an #az_json_writer_options
+ * structure which defines custom behavior of the #az_json_writer. If `NULL` is passed, the writer
+ * will use the default options (i.e. #az_json_writer_options_default()).
+ *
+ * @return An #az_result value indicating the result of the operation:
+ *         - #AZ_OK if the az_json_writer is initialized successfully
+ */
+AZ_NODISCARD az_result az_json_writer_chunked_init(
+    az_json_writer* json_writer,
+    az_span first_destination_buffer,
+    az_span_allocator_fn allocator_callback,
+    void* user_context,
+    az_json_writer_options const* options);
+
+/**
+ * @brief Returns the #az_span containing the JSON text written to the underlying buffer so far, in
+ * the last provided destination buffer.
+ *
+ * @param[in] json_writer A pointer to an #az_json_writer instance wrapping the destination buffer.
  *
  * @note Do NOT modify or override the contents of the returned #az_span unless you are no longer
  * writing JSON text into it.
  *
  * @return An #az_span containing the JSON text built so far.
+ *
+ * @remarks This method returns the entire JSON tet when it fits in the first provided buffer, where
+ * the destination is a single, contiguous buffer. When the destination can be a set of
+ * non-contiguous buffers (using `az_json_writer_chunked_init`), and the JSON is larger than the
+ * first provided destination span, this method only returns the text written into the last provided
+ * destination buffer from the allocator callback.
  */
-AZ_NODISCARD AZ_INLINE az_span az_json_writer_get_json(az_json_writer const* json_writer)
+AZ_NODISCARD AZ_INLINE az_span
+az_json_writer_get_bytes_used_in_destination(az_json_writer const* json_writer)
 {
   return az_span_slice(
       json_writer->_internal.destination_buffer, 0, json_writer->_internal.bytes_written);

sdk/inc/azure/core/az_span.h

@@ -445,6 +445,47 @@ AZ_NODISCARD az_result az_span_u64toa(az_span destination, uint64_t source, az_s
 AZ_NODISCARD az_result
 az_span_dtoa(az_span destination, double source, int32_t fractional_digits, az_span* out_span);
 
+/******************************  NON-CONTIGUOUS SPAN  */
+
+/**
+ * @brief Defines a container of required and user-defined fields that provide the
+ * necessary information and parameters for the implementation of the #az_span_allocator_fn
+ * callback.
+ */
+typedef struct
+{
+  int32_t bytes_used; ///< The amount of space consumed (i.e. written into) within the previously
+                      ///< provided destination, which can be used to infer the remaining number of
+                      ///< bytes of the #az_span that are leftover.
+  int32_t minimum_required_size; ///< The minimum length of the destination #az_span required to be
+                                 ///< provided by the callback. If 0, any non-empty sized buffer
+                                 ///< must be returned.
+  void* user_context; ///< Any struct or set of fields that are provided by the user for their
+                      ///< specific implementation, passed through to the #az_span_allocator_fn.
+} az_allocator_context;
+
+/**
+ * @brief Defines the signature of the callback function that the caller must implement to provide
+ * the potentially discontiguous destination buffers where output can be written into.
+ *
+ * @param[in] allocator_context A container of required and user-defined fields that provide the
+ * necessary information and parameters for the implementation of the callback.
+ * @param[out] out_next_destination A pointer to an #az_span that can be used as a destination to
+ * write data into, that is at least the required size specified within the allocator_context.
+ *
+ * @remarks The caller must no longer hold onto, use, or write to the previously provided #az_span
+ * after this allocator returns a new destination #az_span.
+ *
+ * @remarks There is no guarantee that successive calls will return the same or same-sized buffer.
+ * This method must never return an empty #az_span, unless the requested buffer size is not
+ * available. In which case, it must return an error #az_result.
+ *
+ * @remarks The caller must check the return value using #az_succeeded() before continuing to use
+ * the \p out_next_destination.
+ */
+typedef az_result (
+    *az_span_allocator_fn)(az_allocator_context* allocator_context, az_span* out_next_destination);
+
 /******************************  SPAN PAIR  */
 
 /**

sdk/samples/iot/hub/src/paho_iot_hub_pnp_example.c

@@ -407,7 +407,7 @@ static az_result build_command_response_payload(
   AZ_RETURN_IF_FAILED(az_json_writer_append_string(json_builder, end_time_span));
   AZ_RETURN_IF_FAILED(az_json_writer_append_end_object(json_builder));
 
-  *response_payload = az_json_writer_get_json(json_builder);
+  *response_payload = az_json_writer_get_bytes_used_in_destination(json_builder);
 
   return AZ_OK;
 }
@@ -586,7 +586,7 @@ static int send_reported_temperature_property(
       return rc;
     }
   }
-  az_span json_payload = az_json_writer_get_json(&json_builder);
+  az_span json_payload = az_json_writer_get_bytes_used_in_destination(&json_builder);
 
   printf("Payload: %.*s\n", az_span_size(json_payload), (char*)az_span_ptr(json_payload));
 
@@ -878,8 +878,8 @@ static int connect_device(void)
 
   // Get the MQTT username used to connect to IoT Hub
   if (az_failed(
-          rc = az_iot_hub_client_get_user_name(
-              &client, mqtt_username, sizeof(mqtt_username), NULL)))
+          rc
+          = az_iot_hub_client_get_user_name(&client, mqtt_username, sizeof(mqtt_username), NULL)))
 
   {
     printf("Failed to get MQTT username, return code %d\n", rc);
@@ -975,7 +975,7 @@ static az_result build_telemetry_message(az_span* out_payload)
   AZ_RETURN_IF_FAILED(az_json_writer_append_double(
       &json_builder, current_device_temp, DOUBLE_DECIMAL_PLACE_DIGITS));
   AZ_RETURN_IF_FAILED(az_json_writer_append_end_object(&json_builder));
-  *out_payload = az_json_writer_get_json(&json_builder);
+  *out_payload = az_json_writer_get_bytes_used_in_destination(&json_builder);
 
   return AZ_OK;
 }
@@ -1017,4 +1017,3 @@ static az_span get_request_id(void)
   (void)result;
   return out_span;
 }
-

sdk/samples/iot/hub/src/paho_iot_hub_twin_example.c

@@ -432,15 +432,15 @@ static az_result build_reported_property(az_span* reported_property_payload)
   AZ_RETURN_IF_FAILED(az_json_writer_append_int32(&json_writer, reported_property_value));
   AZ_RETURN_IF_FAILED(az_json_writer_append_end_object(&json_writer));
 
-  *reported_property_payload = az_json_writer_get_json(&json_writer);
+  *reported_property_payload = az_json_writer_get_bytes_used_in_destination(&json_writer);
 
   return AZ_OK;
 }
 
 static void receive_desired_property()
 {
   // Wait until max # messages received
-  for(uint8_t message_count = 0; message_count < MAX_MESSAGE_COUNT; message_count++)
+  for (uint8_t message_count = 0; message_count < MAX_MESSAGE_COUNT; message_count++)
   {
     LOG("Device waiting for desired property twin message #%d from service.", message_count + 1);
     receive_message();

sdk/src/azure/core/az_json_private.h

@@ -9,10 +9,11 @@
 
 #include <azure/core/_az_cfg_prefix.h>
 
-#define _az_JSON_TOKEN_DEFAULT (az_json_token){ \
-  .kind = AZ_JSON_TOKEN_NONE, \
-  ._internal = { 0 } \
-}
+#define _az_JSON_TOKEN_DEFAULT \
+  (az_json_token) \
+  { \
+    .kind = AZ_JSON_TOKEN_NONE, ._internal = { 0 } \
+  }
 
 enum
 {
@@ -36,6 +37,26 @@ enum
 
   _az_MAX_UNESCAPED_STRING_SIZE
   = _az_MAX_ESCAPED_STRING_SIZE / _az_MAX_EXPANSION_FACTOR_WHILE_ESCAPING, // 166_666_666 bytes
+
+  _az_MAX_SIZE_FOR_INT32 = 11, // 10 + sign (i.e. -2147483648)
+
+  // [-][0-9]{16}.[0-9]{15}, i.e. 1+16+1+15 since _az_MAX_SUPPORTED_FRACTIONAL_DIGITS is 15
+  _az_MAX_SIZE_FOR_DOUBLE = 33,
+
+  // When writing large JSON strings in chunks, ask for at least 64 bytes, to avoid writing one
+  // character at a time.
+  // This value should be between 12 and 512 (inclusive).
+  // In the worst case, a 4-byte UTF-8 character, that needs to be escaped using the \uXXXX UTF-16
+  // format, will need 12 bytes, for the two UTF-16 escaped characters (high/low surrogate pairs).
+  // Anything larger than 512 is not feasible since it is difficult for embedded devices to have
+  // such large blocks of contiguous memory available.
+  _az_MINIMUM_STRING_CHUNK_SIZE = 64,
+
+  // We need 2 bytes for the quotes, potentially one more for the comma to separate items, and one
+  // more for the colon if writing a property name. Therefore, only a maximum of 10 character
+  // strings are guaranteed to fit into a single 64 byte chunk, if all 10 needed to be escaped (i.e.
+  // multiply by 6). 10 * 6 + 4 = 64, and that fits within _az_MINIMUM_STRING_CHUNK_SIZE
+  _az_MAX_UNESCAPED_STRING_SIZE_PER_CHUNK = 10,
 };
 
 typedef enum
@@ -61,7 +82,8 @@ AZ_INLINE _az_json_stack_item _az_json_stack_pop(_az_json_bit_stack* json_stack)
   }
 
   // true (i.e. 1) means _az_JSON_STACK_OBJECT, while false (i.e. 0) means _az_JSON_STACK_ARRAY
-  return (json_stack->_internal.az_json_stack & 1) != 0 ? _az_JSON_STACK_OBJECT : _az_JSON_STACK_ARRAY;
+  return (json_stack->_internal.az_json_stack & 1) != 0 ? _az_JSON_STACK_OBJECT
+                                                        : _az_JSON_STACK_ARRAY;
 }
 
 AZ_INLINE void _az_json_stack_push(_az_json_bit_stack* json_stack, _az_json_stack_item item)
@@ -82,7 +104,8 @@ AZ_NODISCARD AZ_INLINE _az_json_stack_item _az_json_stack_peek(_az_json_bit_stac
       && json_stack->_internal.current_depth <= _az_MAX_JSON_STACK_SIZE);
 
   // true (i.e. 1) means _az_JSON_STACK_OBJECT, while false (i.e. 0) means _az_JSON_STACK_ARRAY
-  return (json_stack->_internal.az_json_stack & 1) != 0 ? _az_JSON_STACK_OBJECT : _az_JSON_STACK_ARRAY;
+  return (json_stack->_internal.az_json_stack & 1) != 0 ? _az_JSON_STACK_OBJECT
+                                                        : _az_JSON_STACK_ARRAY;
 }
 
 #include <azure/core/_az_cfg_suffix.h>