incremental decode (#50)

* checkpoint

* it works

* remove unused header

* remove unused field

* golf

* new wiki example

* golf

* golf

* golf

* golf

* use local variables

* api improvement

* const

* rename enum

* tidy

* matrix clang

* pretty name

* all-checks-pass

* arm

* golf

* order

* fix sanitizers

* start decode_inc tests

* 1 byte at a time decoding works

* tidy

* cleanup

* c++20 for cpp designated initializer syntax

* cstring for memset

* fix windows

* fix windows

* fix windows

* fix windows

* rename

* more sanitizers

* more sanitizers

* shut up memory sanitizer about stl_tree in doctest

* shut up memory sanitizer about stl_tree in doctest

* shut up memory sanitizer about stl_tree in doctest

* ignore ostream

* iomanip

* basic_string

* simplify yaml

* makefile adds -fsanitize

* fix makefile

* rename

* cleanup the sanitizer stuff

* windows names

* more incremental decode tests

.github/workflows/presubmit.yml

@@ -8,7 +8,8 @@ on:
     - cron: '0 2 * * 0'  # Weekly
 
 jobs:
-  sanitize:
+  sanitizers:
+    name: sanitizer (${{ matrix.sanitizer }})
     runs-on: ubuntu-latest
 
     permissions:
@@ -20,14 +21,14 @@ jobs:
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}
 
-    env:
-      CC: /usr/bin/clang
-      CXX: /usr/bin/clang++
+    strategy:
+      matrix:
+        sanitizer: [ "memory", "address", "safe-stack", "undefined" ]
 
     steps:
       - uses: actions/checkout@v3
       - name: Build
-        run: COBS_SANITIZE=1 make -j
+        run: CC=clang CXX=clang++ COBS_SANITIZER=${{ matrix.sanitizer }} make -j
 
   arm-cm4:
     runs-on: ubuntu-latest
@@ -46,7 +47,8 @@ jobs:
       - name: Build
         run: arm-none-eabi-gcc -mcpu=cortex-m4 -Os -Werror -Wall -Wextra -Wconversion -c cobs.c
 
-  linux-gcc:
+  linux:
+    name: linux (${{ matrix.compiler.name }}, ${{ matrix.architecture }})
     runs-on: ubuntu-latest
 
     permissions:
@@ -60,12 +62,15 @@ jobs:
 
     strategy:
       matrix:
+        compiler:
+          - { name: "clang", env: "CC=clang CXX=clang++" }
+          - { name: "gcc", env: "CC=gcc CXX=g++" }
         architecture: [32, 64]
 
     steps:
       - uses: actions/checkout@v3
       - name: Build
-        run: COBS_LINUXARCH=${{ matrix.architecture }} make -j
+        run: ${{ matrix.compiler.env }} COBS_LINUXARCH=${{ matrix.architecture }} make -j
 
   macos:
     runs-on: macos-latest
@@ -76,6 +81,7 @@ jobs:
         run: make -j
 
   win:
+    name: windows (msvc, ${{ matrix.architecture }})
     runs-on: windows-latest
 
     strategy:
@@ -89,3 +95,10 @@ jobs:
         run: |
           call "C:\Program Files\Microsoft Visual Studio\2022\Enterprise\VC\Auxiliary\Build\vcvars${{ matrix.architecture }}.bat"
           call make-win.bat
+
+  all-checks-pass:
+    needs: [sanitizers, linux, macos, win, arm-cm4]
+    runs-on: ubuntu-latest
+    steps:
+    - run: echo Done
+

.gitignore

@@ -1,4 +1,5 @@
 build
+.vscode
 
 # Prerequisites
 *.d

Makefile

@@ -4,6 +4,7 @@ SRCS := tests/cobs_encode_max_c.c \
 		tests/test_cobs_encode_inc.cc \
 		tests/test_cobs_encode_inplace.cc \
 		tests/test_cobs_decode.cc \
+		tests/test_cobs_decode_inc.cc \
 		tests/test_cobs_decode_inplace.cc \
 		tests/test_paper_figures.cc \
 		tests/test_wikipedia.cc \
@@ -16,7 +17,7 @@ OS := $(shell uname)
 COMPILER_VERSION := $(shell $(CXX) --version)
 
 CFLAGS = --std=c99
-CXXFLAGS = --std=c++17
+CXXFLAGS = --std=c++20
 
 CPPFLAGS += -MMD -MP -Os -g
 
@@ -27,11 +28,6 @@ LDFLAGS += -m32
 endif
 endif
 
-ifeq ($(COBS_SANITIZE),1)
-CPPFLAGS_SAN += -fsanitize=undefined,address
-LDFLAGS_SAN += -fsanitize=undefined,address
-endif
-
 CPPFLAGS += -Wall -Werror -Wextra
 
 ifneq '' '$(findstring clang,$(COMPILER_VERSION))'
@@ -40,25 +36,31 @@ CPPFLAGS += -Weverything \
 			-Wno-unsafe-buffer-usage \
 			-Wno-poison-system-directories \
 			-Wno-format-pedantic \
-			-Wno-c++98-compat-bind-to-temporary-copy
+			-Wno-c++98-compat-bind-to-temporary-copy \
+			-Wno-pre-c++20-compat-pedantic
 CFLAGS += -Wno-declaration-after-statement
 else
 CPPFLAGS += -Wconversion
 endif
 
 CPPFLAGS += -Wno-c++98-compat -Wno-padded
 
+ifdef COBS_SANITIZER
+CPPFLAGS += -fsanitize=$(COBS_SANITIZER) -fsanitize-ignorelist=sanitize-ignorelist.txt
+LDFLAGS += -fsanitize=$(COBS_SANITIZER) -fsanitize-ignorelist=sanitize-ignorelist.txt
+endif
+
 $(BUILD_DIR)/cobs_unittests: $(OBJS) $(BUILD_DIR)/cobs.c.o Makefile
-	$(CXX) $(LDFLAGS) $(LDFLAGS_SAN) $(OBJS) $(BUILD_DIR)/cobs.c.o -o $@
+	$(CXX) $(LDFLAGS) $(OBJS) $(BUILD_DIR)/cobs.c.o -o $@
 
 $(BUILD_DIR)/cobs.c.o: cobs.c cobs.h Makefile
-	mkdir -p $(dir $@) && $(CC) $(CPPFLAGS) $(CFLAGS) $(CPPFLAGS_SAN) -c $< -o $@
+	mkdir -p $(dir $@) && $(CC) $(CPPFLAGS) $(CFLAGS) -c $< -o $@
 
 $(BUILD_DIR)/%.c.o: %.c Makefile
-	mkdir -p $(dir $@) && $(CC) $(CPPFLAGS) $(CFLAGS) $(CPPFLAGS_SAN) -c $< -o $@
+	mkdir -p $(dir $@) && $(CC) $(CPPFLAGS) $(CFLAGS) -c $< -o $@
 
 $(BUILD_DIR)/%.cc.o: %.cc Makefile
-	mkdir -p $(dir $@) && $(CXX) $(CPPFLAGS) $(CXXFLAGS) $(CPPFLAGS_SAN) -c $< -o $@
+	mkdir -p $(dir $@) && $(CXX) $(CPPFLAGS) $(CXXFLAGS) -c $< -o $@
 
 $(BUILD_DIR)/cobs_unittests.timestamp: $(BUILD_DIR)/cobs_unittests
 	$(BUILD_DIR)/cobs_unittests -m && touch $(BUILD_DIR)/cobs_unittests.timestamp

cobs.c

@@ -42,7 +42,7 @@ cobs_ret_t cobs_decode_inplace(void *buf, size_t const len) {
 
   cobs_byte_t *const src = (cobs_byte_t *)buf;
   size_t ofs, cur = 0;
-  while (cur < len && ((ofs = src[cur]) != COBS_FRAME_DELIMITER)) {
+  while ((cur < len) && ((ofs = src[cur]) != COBS_FRAME_DELIMITER)) {
     src[cur] = 0;
     for (size_t i = 1; i < ofs; ++i) {
       if (src[cur + i] == 0) {
@@ -183,42 +183,102 @@ cobs_ret_t cobs_decode(void const *enc,
     return COBS_RET_ERR_BAD_ARG;
   }
 
-  cobs_byte_t const *const src = (cobs_byte_t const *)enc;
-  cobs_byte_t *const dst = (cobs_byte_t *)out_dec;
+  cobs_decode_inc_ctx_t ctx;
+  cobs_ret_t r = cobs_decode_inc_begin(&ctx);
+  if (r != COBS_RET_SUCCESS) {
+    return r;
+  }
 
-  if ((src[0] == COBS_FRAME_DELIMITER) || (src[enc_len - 1] != COBS_FRAME_DELIMITER)) {
-    return COBS_RET_ERR_BAD_PAYLOAD;
+  size_t src_len;
+  bool decode_complete;
+  if ((r = cobs_decode_inc(&ctx,
+                           &(cobs_decode_inc_args_t){ .enc_src = enc,
+                                                      .dec_dst = out_dec,
+                                                      .enc_src_max = enc_len,
+                                                      .dec_dst_max = dec_max },
+                           &src_len,
+                           out_dec_len,
+                           &decode_complete)) != COBS_RET_SUCCESS) {
+    return r;
   }
+  return decode_complete ? COBS_RET_SUCCESS : COBS_RET_ERR_EXHAUSTED;
+}
 
+cobs_ret_t cobs_decode_inc_begin(cobs_decode_inc_ctx_t *ctx) {
+  if (!ctx) {
+    return COBS_RET_ERR_BAD_ARG;
+  }
+  ctx->state = COBS_DECODE_READ_CODE;
+  return COBS_RET_SUCCESS;
+}
+
+cobs_ret_t cobs_decode_inc(cobs_decode_inc_ctx_t *ctx,
+                           cobs_decode_inc_args_t const *args,
+                           size_t *out_enc_src_len,
+                           size_t *out_dec_dst_len,
+                           bool *out_decode_complete) {
+  if (!ctx || !args || !out_enc_src_len || !out_dec_dst_len || !out_decode_complete ||
+      !args->dec_dst || !args->enc_src) {
+    return COBS_RET_ERR_BAD_ARG;
+  }
+
+  bool decode_complete = false;
   size_t src_idx = 0, dst_idx = 0;
 
-  while (src_idx < (enc_len - 1)) {
-    unsigned const code = src[src_idx++];
-    if (!code) {
-      return COBS_RET_ERR_BAD_PAYLOAD;
-    }
-    if ((src_idx + code) > enc_len) {
-      return COBS_RET_ERR_BAD_PAYLOAD;
-    }
+  size_t const src_max = args->enc_src_max;
+  size_t const dst_max = args->dec_dst_max;
+  cobs_byte_t const *src_b = (cobs_byte_t const *)args->enc_src;
+  cobs_byte_t *dst_b = (cobs_byte_t *)args->dec_dst;
+  unsigned block = ctx->block, code = ctx->code;
+  enum cobs_decode_inc_state state = ctx->state;
 
-    if ((dst_idx + code - 1) > dec_max) {
-      return COBS_RET_ERR_EXHAUSTED;
-    }
-    for (size_t i = 0; i < code - 1; ++i) {
-      if (src[src_idx] == 0) {
-        return COBS_RET_ERR_BAD_PAYLOAD;
-      }
-      dst[dst_idx++] = src[src_idx++];
-    }
+  while (src_idx < src_max) {
+    switch (state) {
+      case COBS_DECODE_READ_CODE: {
+        block = code = src_b[src_idx++];
+        state = COBS_DECODE_RUN;
+      } break;
 
-    if ((src_idx < (enc_len - 1)) && (code < 0xFF)) {
-      if (dst_idx >= dec_max) {
-        return COBS_RET_ERR_EXHAUSTED;
-      }
-      dst[dst_idx++] = 0;
+      case COBS_DECODE_FINISH_RUN: {
+        if (!src_b[src_idx]) {
+          decode_complete = true;
+          goto done;
+        }
+
+        if (code != 0xFF) {
+          if (dst_idx >= dst_max) {
+            goto done;
+          }
+          dst_b[dst_idx++] = 0;
+        }
+        state = COBS_DECODE_READ_CODE;
+      } break;
+
+      case COBS_DECODE_RUN: {
+        while (block - 1) {
+          if ((src_idx >= src_max) || (dst_idx >= dst_max)) {
+            goto done;
+          }
+
+          --block;
+          cobs_byte_t const b = src_b[src_idx++];
+          if (!b) {
+            return COBS_RET_ERR_BAD_PAYLOAD;
+          }
+
+          dst_b[dst_idx++] = b;
+        }
+        state = COBS_DECODE_FINISH_RUN;
+      } break;
     }
   }
 
-  *out_dec_len = dst_idx;
+done:
+  ctx->state = state;
+  ctx->code = (uint8_t)code;
+  ctx->block = (uint8_t)block;
+  *out_dec_dst_len = dst_idx;
+  *out_enc_src_len = src_idx;
+  *out_decode_complete = decode_complete;
   return COBS_RET_SUCCESS;
 }

cobs.h

@@ -2,6 +2,7 @@
 
 #include <stdbool.h>
 #include <stddef.h>
+#include <stdint.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -171,6 +172,31 @@ cobs_ret_t cobs_encode_inc(cobs_enc_ctx_t *ctx, void const *dec_src, size_t dec_
 // If null pointers are provided, the function returns COBS_RET_ERR_BAD_ARG.
 cobs_ret_t cobs_encode_inc_end(cobs_enc_ctx_t *ctx, size_t *out_enc_len);
 
+// Incremental decoding API
+
+typedef struct cobs_decode_inc_ctx {
+  enum cobs_decode_inc_state {
+    COBS_DECODE_READ_CODE,
+    COBS_DECODE_RUN,
+    COBS_DECODE_FINISH_RUN
+  } state;
+  uint8_t block, code;
+} cobs_decode_inc_ctx_t;
+
+typedef struct cobs_decode_inc_args {
+  void const *enc_src;  // pointer to current position of encoded payload
+  void *dec_dst;        // pointer to decoded output buffer.
+  size_t enc_src_max;   // length of the |src| input buffer.
+  size_t dec_dst_max;   // length of the |dst| output buffer.
+} cobs_decode_inc_args_t;
+
+cobs_ret_t cobs_decode_inc_begin(cobs_decode_inc_ctx_t *ctx);
+cobs_ret_t cobs_decode_inc(cobs_decode_inc_ctx_t *ctx,
+                           cobs_decode_inc_args_t const *args,
+                           size_t *out_enc_src_len,  // how many bytes of src were read
+                           size_t *out_dec_dst_len,  // how many bytes written to dst
+                           bool *out_decode_complete);
+
 #ifdef __cplusplus
 }
 #endif

make-win.bat

@@ -1,7 +1,8 @@
-cl.exe /W4 /WX /MP /EHsc ^
+cl.exe /W4 /WX /MP /EHsc /std:c++20 ^
     cobs.c ^
     tests/cobs_encode_max_c.c ^
     tests/test_cobs_decode.cc ^
+    tests/test_cobs_decode_inc.cc ^
     tests/test_cobs_decode_inplace.cc ^
     tests/test_cobs_encode_max.cc ^
     tests/test_cobs_encode.cc ^

sanitize-ignorelist.txt

@@ -0,0 +1,5 @@
+[memory]
+src:*/stl_tree.h
+src:*/ostream
+src:*/iomanip
+src:*/basic_string.h