[POC, Do not merge] input_chunk: split incoming buffer when it's too big

include/fluent-bit/flb_input_log.h

@@ -25,17 +25,17 @@
 
 int flb_input_log_append(struct flb_input_instance *ins,
                          const char *tag, size_t tag_len,
-                         const void *buf, size_t buf_size);
+                         void *buf, size_t buf_size);
 
 int flb_input_log_append_records(struct flb_input_instance *ins,
                                  size_t records,
                                  const char *tag, size_t tag_len,
-                                 const void *buf, size_t buf_size);
+                                 void *buf, size_t buf_size);
 
 int flb_input_log_append_skip_processor_stages(struct flb_input_instance *ins,
                                                size_t processor_starting_stage,
                                                const char *tag,
                                                size_t tag_len,
-                                               const void *buf,
+                                               void *buf,
                                                size_t buf_size);
 #endif

plugins/in_tail/tail_file.c

@@ -615,8 +615,9 @@ static int process_content(struct flb_tail_file *file, size_t *bytes)
                                          file->tag_len,
                                          file->sl_log_event_encoder->output_buffer,
                                          file->sl_log_event_encoder->output_length);
-
             flb_log_event_encoder_reset(file->sl_log_event_encoder);
+            flb_free(file->sl_log_event_encoder->output_buffer);
+            flb_log_event_encoder_claim_internal_buffer_ownership(file->sl_log_event_encoder);
         }
     }
     else if (file->skip_next) {

src/flb_input_chunk.c

@@ -36,6 +36,7 @@
 #include <fluent-bit/flb_metrics.h>
 #include <fluent-bit/stream_processor/flb_sp.h>
 #include <fluent-bit/flb_ring_buffer.h>
+#include <fluent-bit/flb_log_event.h>
 #include <chunkio/chunkio.h>
 #include <monkey/mk_core.h>
 
@@ -1124,7 +1125,9 @@ static struct flb_input_chunk *input_chunk_get(struct flb_input_instance *in,
     }
 
     if (id >= 0) {
-        if (ic->busy == FLB_TRUE || cio_chunk_is_locked(ic->chunk)) {
+        if (ic->busy == FLB_TRUE || cio_chunk_is_locked(ic->chunk) ||
+            (flb_input_chunk_get_real_size(ic) + chunk_size) >
+                FLB_INPUT_CHUNK_FS_MAX_SIZE) {
             ic = NULL;
         }
         else if (cio_chunk_is_up(ic->chunk) == CIO_FALSE) {

src/flb_input_log.c

@@ -23,17 +23,137 @@
 #include <fluent-bit/flb_input_log.h>
 #include <fluent-bit/flb_input_plugin.h>
 #include <fluent-bit/flb_processor.h>
+#include <fluent-bit/flb_log_event_decoder.h>
+#include <fluent-bit/flb_log_event_encoder.h>
+
+#include <monkey/mk_core/mk_list.h>
+
+struct buffer_entry {
+    char *buf;
+    size_t buf_size;
+    struct mk_list _head;
+};
+
+static struct buffer_entry *new_buffer_entry(void *buf, size_t buf_size)
+{
+    struct buffer_entry *new_entry = flb_malloc(sizeof(struct buffer_entry));
+    new_entry->buf_size = buf_size;
+    new_entry->buf = buf;
+    return new_entry;
+}
+
+static void buffer_entry_destroy(struct buffer_entry *entry) {
+    if (!entry) {
+        return;
+    }
+    if (entry->buf) {
+        flb_free(entry->buf);
+    }
+    mk_list_del(&entry->_head);
+    flb_free(entry);
+}
+
+static int split_buffer_entry(struct buffer_entry *entry,
+                              struct mk_list *entries)
+{
+    int ret;
+    int encoder_result;
+    void *tmp_encoder_buf;
+    size_t tmp_encoder_buf_size;
+    struct flb_log_event_encoder log_encoder;
+    struct flb_log_event_decoder log_decoder;
+    struct flb_log_event log_event;
+    int entries_processed;
+    struct buffer_entry *new_buffer;
+
+    ret = flb_log_event_decoder_init(&log_decoder, entry->buf, entry->buf_size);
+    if (ret != FLB_EVENT_DECODER_SUCCESS) {
+        flb_error("Log event decoder initialization error : %d", ret);
+
+        return FLB_FALSE;
+    }
+
+    ret = flb_log_event_encoder_init(&log_encoder,
+                                    FLB_LOG_EVENT_FORMAT_DEFAULT);
+    if (ret != FLB_EVENT_ENCODER_SUCCESS) {
+        flb_error("Log event encoder initialization error : %d", ret);
+
+        flb_log_event_decoder_destroy(&log_decoder);
+
+        return FLB_FALSE;
+    }
+
+    entries_processed = 0;
+    while ((ret = flb_log_event_decoder_next(
+                    &log_decoder,
+                    &log_event)) == FLB_EVENT_DECODER_SUCCESS) {
+        encoder_result = flb_log_event_encoder_begin_record(&log_encoder);
+        if (encoder_result == FLB_EVENT_ENCODER_SUCCESS) {
+            encoder_result = flb_log_event_encoder_set_timestamp(
+                                     &log_encoder, &log_event.timestamp);
+        }
+
+        if (encoder_result == FLB_EVENT_ENCODER_SUCCESS) {
+            encoder_result = \
+                flb_log_event_encoder_set_metadata_from_msgpack_object(
+                    &log_encoder, log_event.metadata);
+        }
+
+        if (encoder_result == FLB_EVENT_ENCODER_SUCCESS) {
+            encoder_result = \
+                flb_log_event_encoder_set_body_from_msgpack_object(
+                    &log_encoder, log_event.body);
+        }
+
+        if (encoder_result == FLB_EVENT_ENCODER_SUCCESS) {
+            encoder_result = flb_log_event_encoder_commit_record(&log_encoder);
+        }
+
+        if (encoder_result != FLB_EVENT_ENCODER_SUCCESS) {
+            flb_error("log event encoder error : %d", encoder_result);
+            continue;
+        }
+
+        if (log_encoder.output_length >= FLB_INPUT_CHUNK_FS_MAX_SIZE) {
+            tmp_encoder_buf_size = log_encoder.output_length;
+            tmp_encoder_buf = log_encoder.output_buffer;
+            flb_log_event_encoder_claim_internal_buffer_ownership(&log_encoder);
+            new_buffer = new_buffer_entry(tmp_encoder_buf, tmp_encoder_buf_size);
+            mk_list_add(&new_buffer->_head, entries);
+        }
+
+        entries_processed++;
+    }
+
+    if (log_encoder.output_length >= 0) {
+        tmp_encoder_buf_size = log_encoder.output_length;
+        tmp_encoder_buf = flb_malloc(tmp_encoder_buf_size);
+        memcpy(tmp_encoder_buf, log_encoder.output_buffer, tmp_encoder_buf_size);
+        new_buffer = new_buffer_entry(tmp_encoder_buf, tmp_encoder_buf_size);
+        mk_list_add(&new_buffer->_head, entries);
+    }
+
+    flb_log_event_encoder_destroy(&log_encoder);
+    flb_log_event_decoder_destroy(&log_decoder);
+    return FLB_TRUE;
+}
+
 
 static int input_log_append(struct flb_input_instance *ins,
                             size_t processor_starting_stage,
                             size_t records,
                             const char *tag, size_t tag_len,
-                            const void *buf, size_t buf_size)
+                            void *buf, size_t buf_size)
 {
     int ret;
     int processor_is_active;
     void *out_buf = (void *) buf;
     size_t out_size = buf_size;
+    struct mk_list buffers;
+    struct mk_list *head;
+    struct mk_list *tmp;
+    struct buffer_entry *start_buffer;
+    struct buffer_entry *iter_buffer;
 
     processor_is_active = flb_processor_is_active(ins->processor);
     if (processor_is_active) {
@@ -68,9 +188,22 @@ static int input_log_append(struct flb_input_instance *ins,
         }
     }
 
-    ret = flb_input_chunk_append_raw(ins, FLB_INPUT_LOGS, records,
-                                     tag, tag_len, out_buf, out_size);
-
+    if (buf_size > FLB_INPUT_CHUNK_FS_MAX_SIZE) {
+        mk_list_init(&buffers);
+        start_buffer = new_buffer_entry(buf, buf_size);
+        split_buffer_entry(start_buffer, &buffers);
+        flb_free(start_buffer);
+        mk_list_foreach_safe(head, tmp, &buffers) {
+            iter_buffer = mk_list_entry(head, struct buffer_entry, _head);
+            records = flb_mp_count(iter_buffer->buf, iter_buffer->buf_size);
+            ret = flb_input_chunk_append_raw(ins, FLB_INPUT_LOGS, records,
+                                            tag, tag_len, iter_buffer->buf, iter_buffer->buf_size);
+            buffer_entry_destroy(iter_buffer);
+        }
+    } else {
+        ret = flb_input_chunk_append_raw(ins, FLB_INPUT_LOGS, records,
+                                        tag, tag_len, buf, buf_size);
+    }
 
     if (processor_is_active && buf != out_buf) {
         flb_free(out_buf);
@@ -81,7 +214,7 @@ static int input_log_append(struct flb_input_instance *ins,
 /* Take a msgpack serialized record and enqueue it as a chunk */
 int flb_input_log_append(struct flb_input_instance *ins,
                          const char *tag, size_t tag_len,
-                         const void *buf, size_t buf_size)
+                         void *buf, size_t buf_size)
 {
     int ret;
     size_t records;
@@ -96,7 +229,7 @@ int flb_input_log_append_skip_processor_stages(struct flb_input_instance *ins,
                                                size_t processor_starting_stage,
                                                const char *tag,
                                                size_t tag_len,
-                                               const void *buf,
+                                               void *buf,
                                                size_t buf_size)
 {
     return input_log_append(ins,
@@ -112,7 +245,7 @@ int flb_input_log_append_skip_processor_stages(struct flb_input_instance *ins,
 int flb_input_log_append_records(struct flb_input_instance *ins,
                                  size_t records,
                                  const char *tag, size_t tag_len,
-                                 const void *buf, size_t buf_size)
+                                 void *buf, size_t buf_size)
 {
     int ret;