improvement for the appended 0

ggerganov · Nov 25, 2023 · 9d3ba0b · 9d3ba0b
1 parent a4b7b4c
commit 9d3ba0b
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/llama.cpp b/llama.cpp
@@ -6425,8 +6425,8 @@ static std::pair<std::vector<uint32_t>, llama_partial_utf8> decode_utf8(
     static const int      lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 2, 2, 3, 4 };
     const char          * pos      = src;
     std::vector<uint32_t> code_points;
-    // common english strings have the same number of codepoints and bytes.
-    code_points.reserve(n_src);
+    // common english strings have the same number of codepoints and bytes. `+ 1` for the terminating 0.
+    code_points.reserve(n_src + 1);
     uint32_t              value    = partial_start.value;
     int                   n_remain = partial_start.n_remain;