[update] add llm_camera!

ext_components/StackFlow/stackflow/StackFlowUtil.cpp

@@ -106,33 +106,19 @@ std::string StackFlows::sample_get_work_id(int work_id_num, const std::string &u
 {
     return unit_name + "." + std::to_string(work_id_num);
 }
-
+// clang-format off
 std::string StackFlows::sample_escapeString(const std::string &input)
 {
     std::string escaped;
     for (char c : input) {
         switch (c) {
-            case '\n':
-                escaped += "\\n";
-                break;
-            case '\t':
-                escaped += "\\t";
-                break;
-            case '\\':
-                escaped += "\\\\";
-                break;
-            case '\"':
-                escaped += "\\\"";
-                break;
-            case '\r':
-                escaped += "\\r";
-                break;
-            case '\b':
-                escaped += "\\b";
-                break;
-            default:
-                escaped += c;
-                break;
+            case '\n':escaped += "\\n" ;break;
+            case '\t':escaped += "\\t" ;break;
+            case '\\':escaped += "\\\\";break;
+            case '\"':escaped += "\\\"";break;
+            case '\r':escaped += "\\r" ;break;
+            case '\b':escaped += "\\b" ;break;
+            default  :escaped += c     ;break;
         }
     }
     return escaped;
@@ -144,41 +130,21 @@ std::string StackFlows::sample_unescapeString(const std::string &input)
     for (size_t i = 0; i < input.length(); ++i) {
         if (input[i] == '\\' && i + 1 < input.length()) {
             switch (input[i + 1]) {
-                case 'n':
-                    unescaped += '\n';
-                    ++i;
-                    break;
-                case 't':
-                    unescaped += '\t';
-                    ++i;
-                    break;
-                case '\\':
-                    unescaped += '\\';
-                    ++i;
-                    break;
-                case '\"':
-                    unescaped += '\"';
-                    ++i;
-                    break;
-                case 'r':
-                    unescaped += '\r';
-                    ++i;
-                    break;
-                case 'b':
-                    unescaped += '\b';
-                    ++i;
-                    break;
-                default:
-                    unescaped += input[i];
-                    break;
+                case 'n' :unescaped += '\n';++i;break;
+                case 't' :unescaped += '\t';++i;break;
+                case '\\':unescaped += '\\';++i;break;
+                case '\"':unescaped += '\"';++i;break;
+                case 'r' :unescaped += '\r';++i;break;
+                case 'b' :unescaped += '\b';++i;break;
+                default  :unescaped += input[i];break;
             }
         } else {
             unescaped += input[i];
         }
     }
     return unescaped;
 }
-
+// clang-format on
 bool StackFlows::decode_stream(const std::string &in, std::string &out,
                                std::unordered_map<int, std::string> &stream_buff)
 {
@@ -200,125 +166,23 @@ bool StackFlows::decode_stream(const std::string &in, std::string &out,
 #define BASE64_DECODE_OUT_SIZE(s) (((s)) / 4 * 3)
 #include <stdio.h>
 /* BASE 64 encode table */
-static const char base64en[] = {
-    'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', 'S', 'T', 'U', 'V',
-    'W', 'X', 'Y', 'Z', 'a', 'b', 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r',
-    's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '+', '/',
-};
+static const char base64en[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
 
 #define BASE64_PAD     '='
 #define BASE64DE_FIRST '+'
 #define BASE64DE_LAST  'z'
 /* ASCII order for BASE 64 decode, -1 in unused character */
 static const signed char base64de[] = {
-    /* '+', ',', '-', '.', '/', '0', '1', '2', */
-    62,
-    -1,
-    -1,
-    -1,
-    63,
-    52,
-    53,
-    54,
-
-    /* '3', '4', '5', '6', '7', '8', '9', ':', */
-    55,
-    56,
-    57,
-    58,
-    59,
-    60,
-    61,
-    -1,
-
-    /* ';', '<', '=', '>', '?', '@', 'A', 'B', */
-    -1,
-    -1,
-    -1,
-    -1,
-    -1,
-    -1,
-    0,
-    1,
-
-    /* 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', */
-    2,
-    3,
-    4,
-    5,
-    6,
-    7,
-    8,
-    9,
-
-    /* 'K', 'L', 'M', 'N', 'O', 'P', 'Q', 'R', */
-    10,
-    11,
-    12,
-    13,
-    14,
-    15,
-    16,
-    17,
-
-    /* 'S', 'T', 'U', 'V', 'W', 'X', 'Y', 'Z', */
-    18,
-    19,
-    20,
-    21,
-    22,
-    23,
-    24,
-    25,
-
-    /* '[', '\', ']', '^', '_', '`', 'a', 'b', */
-    -1,
-    -1,
-    -1,
-    -1,
-    -1,
-    -1,
-    26,
-    27,
-
-    /* 'c', 'd', 'e', 'f', 'g', 'h', 'i', 'j', */
-    28,
-    29,
-    30,
-    31,
-    32,
-    33,
-    34,
-    35,
-
-    /* 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', */
-    36,
-    37,
-    38,
-    39,
-    40,
-    41,
-    42,
-    43,
-
-    /* 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', */
-    44,
-    45,
-    46,
-    47,
-    48,
-    49,
-    50,
-    51,
+    62, -1, -1, -1, 63, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -1, -1, -1, -1, 0,  1,  2,  3,  4,
+    5,  6,  7,  8,  9,  10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1, -1,
+    26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51,
 };
 
 static int base64_encode(const unsigned char *in, unsigned int inlen, char *out)
 {
     unsigned int i = 0, j = 0;
-
     for (; i < inlen; i++) {
         int s = i % 3;
-
         switch (s) {
             case 0:
                 out[j++] = base64en[(in[i] >> 2) & 0x3F];
@@ -331,10 +195,8 @@ static int base64_encode(const unsigned char *in, unsigned int inlen, char *out)
                 out[j++] = base64en[in[i] & 0x3F];
         }
     }
-
     /* move back */
     i -= 1;
-
     /* check the last and add padding */
     if ((i % 3) == 0) {
         out[j++] = base64en[(in[i] & 0x3) << 4];
@@ -344,35 +206,28 @@ static int base64_encode(const unsigned char *in, unsigned int inlen, char *out)
         out[j++] = base64en[(in[i] & 0xF) << 2];
         out[j++] = BASE64_PAD;
     }
-
     return j;
 }
 
 static int base64_decode(const char *in, unsigned int inlen, unsigned char *out)
 {
     unsigned int i = 0, j = 0;
-
     for (; i < inlen; i++) {
         int c;
         int s = i % 4;
-
         if (in[i] == '=') return j;
-
         if (in[i] < BASE64DE_FIRST || in[i] > BASE64DE_LAST || (c = base64de[in[i] - BASE64DE_FIRST]) == -1) return -1;
-
         switch (s) {
             case 0:
                 out[j] = ((unsigned int)c << 2) & 0xFF;
                 continue;
             case 1:
                 out[j++] += ((unsigned int)c >> 4) & 0x3;
-
                 /* if not last char with padding */
                 if (i < (inlen - 3) || in[inlen - 2] != '=') out[j] = ((unsigned int)c & 0xF) << 4;
                 continue;
             case 2:
                 out[j++] += ((unsigned int)c >> 2) & 0xF;
-
                 /* if not last char with padding */
                 if (i < (inlen - 2) || in[inlen - 1] != '=') out[j] = ((unsigned int)c & 0x3) << 6;
                 continue;

ext_components/StackFlow/stackflow/pzmq.hpp

@@ -59,6 +59,9 @@ class pzmq {
     pzmq(const std::string &url, int mode, const msg_callback_fun &raw_call = nullptr)
         : zmq_ctx_(NULL), zmq_socket_(NULL), mode_(mode), flage_(true), timeout_(3000)
     {
+        if ((url[0] != 'i') && (url[1] != 'p')) {
+            rpc_url_head_.clear();
+        }
         if (mode_ != ZMQ_RPC_FUN) creat(url, raw_call);
     }
     void set_timeout(int ms)

projects/llm_framework/.gitignore

@@ -8,3 +8,4 @@ static_lib**
 *.tar.gz
 m5stack_*
 debian*
+gitignore_*

projects/llm_framework/config_defaults.mk

@@ -13,7 +13,7 @@ CONFIG_DEVICE_UART_ENABLED=y
 # CONFIG_LHV_WITH_EVPP=y
 CONFIG_UTILITIES_ENABLED=y
 CONFIG_BACKWARD_CPP_ENABLED=y
-CONFIG_TOOLCHAIN_FLAGS="-O2"
+CONFIG_TOOLCHAIN_FLAGS="-O0"
 CONFIG_EVENTPP_ENABLED=y
 CONFIG_LHV_ENABLED=y
 CONFIG_LHV_WITH_EVPP=y

projects/llm_framework/main/src/main.cpp

@@ -229,7 +229,7 @@ class llm_llm : public StackFlow {
             req_body["model"]           = llm_task_obj->model_;
             req_body["response_format"] = llm_task_obj->response_format_;
             req_body["enoutput"]        = llm_task_obj->enoutput_;
-            req_body["inputs_"]         = llm_task_obj->inputs_;
+            req_body["inputs"]         = llm_task_obj->inputs_;
             send("llm.taskinfo", req_body, LLM_NO_ERROR, work_id);
         }
     }

projects/llm_framework/main_asr/src/main.cpp

@@ -179,7 +179,7 @@ class llm_task {
         out_callback_ = out_callback;
     }
 
-    void sys_pcm_on_data(pzmq *_pzmq, const std::string &raw)
+    void sys_pcm_on_data(const std::string &raw)
     {
         static int count = 0;
         if (count < delay_audio_frame_) {
@@ -403,7 +403,7 @@ class llm_asr : public StackFlow {
             }
             next_data = &tmp_msg4;
         }
-        llm_task_obj->sys_pcm_on_data(nullptr, (*next_data));
+        llm_task_obj->sys_pcm_on_data((*next_data));
     }
 
     void _task_pause(const std::string &work_id, const std::string &data)
@@ -435,8 +435,10 @@ class llm_asr : public StackFlow {
         }
         llm_task_obj->kws_awake();
         if ((!audio_url_.empty()) && (llm_task_obj->audio_flage_ == false)) {
-            llm_channel->subscriber(audio_url_, std::bind(&llm_task::sys_pcm_on_data, llm_task_obj.get(),
-                                                          std::placeholders::_1, std::placeholders::_2));
+            std::weak_ptr<llm_task> _llm_task_obj = llm_task_obj;
+            llm_channel->subscriber(audio_url_, [_llm_task_obj](pzmq *_pzmq, const std::string &raw) {
+                _llm_task_obj.lock()->sys_pcm_on_data(raw);
+            });
             llm_task_obj->audio_flage_ = true;
         }
     }
@@ -521,9 +523,11 @@ class llm_asr : public StackFlow {
 
             for (const auto input : llm_task_obj->inputs_) {
                 if (input.find("sys") != std::string::npos) {
-                    audio_url_ = unit_call("audio", "cap", input);
-                    llm_channel->subscriber(audio_url_, std::bind(&llm_task::sys_pcm_on_data, llm_task_obj.get(),
-                                                                  std::placeholders::_1, std::placeholders::_2));
+                    audio_url_                            = unit_call("audio", "cap", input);
+                    std::weak_ptr<llm_task> _llm_task_obj = llm_task_obj;
+                    llm_channel->subscriber(audio_url_, [_llm_task_obj](pzmq *_pzmq, const std::string &raw) {
+                        _llm_task_obj.lock()->sys_pcm_on_data(raw);
+                    });
                     llm_task_obj->audio_flage_ = true;
                 } else if (input.find("asr") != std::string::npos) {
                     llm_channel->subscriber_work_id(
@@ -568,8 +572,10 @@ class llm_asr : public StackFlow {
         auto llm_task_obj = llm_task_[work_id_num];
         if (data.find("sys") != std::string::npos) {
             if (audio_url_.empty()) audio_url_ = unit_call("audio", "cap", data);
-            llm_channel->subscriber(audio_url_, std::bind(&llm_task::sys_pcm_on_data, llm_task_obj.get(),
-                                                          std::placeholders::_1, std::placeholders::_2));
+            std::weak_ptr<llm_task> _llm_task_obj = llm_task_obj;
+            llm_channel->subscriber(audio_url_, [_llm_task_obj](pzmq *_pzmq, const std::string &raw) {
+                _llm_task_obj.lock()->sys_pcm_on_data(raw);
+            });
             llm_task_obj->audio_flage_ = true;
             llm_task_obj->inputs_.push_back(data);
         } else if (data.find("kws") != std::string::npos) {
@@ -638,7 +644,7 @@ class llm_asr : public StackFlow {
             req_body["model"]           = llm_task_obj->model_;
             req_body["response_format"] = llm_task_obj->response_format_;
             req_body["enoutput"]        = llm_task_obj->enoutput_;
-            req_body["inputs_"]         = llm_task_obj->inputs_;
+            req_body["inputs"]         = llm_task_obj->inputs_;
             send("asr.taskinfo", req_body, LLM_NO_ERROR, work_id);
         }
     }