Initial support for ESP32-C6 and ESP32-H2, plus assorted fixes & impr…

…ovements (#3646)

* Proof-of-concept multi-type console support via stdio

* Address crashes on linput's use of printf.

On an empty line input, a C3 with UART console would panic while attempting
to output the new Lua prompt. The backtrace shows a xQueueSemaphoreTake
with uxItemSize==0 as the panic cause, deep inside the uart driver, invoked
via vfs_uart and vfs_console layers, from printf.
Similarly, the printf for outputting a backspace/erase sequence would also
trigger a panic.

This workaround (of not mixing fflush() with printf) is likely merely hiding
a deeper issue, but it appears to be consistent. Plus, printf with no args
and a user-supplied format string is a no-no and should be fixed anyway.

* Work around IDF inconsistency with stdout buffering.

* Increase console task stack size.

Seems on Xtensa it ended up not being enough.

* Switch to single-byte console reads.

* Stop cheating and feed Lua from the right context.

* Work around IDF buffering stdout even when told not to, on ACM consoles.

* Initial build support for esp32c6.

Plus fixup of module selection for a variety of targets.

* Update github actions to node 20 versions.

* Update github build to deal with Lua 5.3 being default.

* Address fatal compiler warning.

Newer IDF toolchain is stricter, and we'd apparently failed to build test
the Lua-5.1 path for some time.

* Initial build support for esp32h2.

* Upgrade IDF to v5.1.3

* Fix left-over incorrect type in uzlib.

* Avoid null pointer crashes when debugging startup.

* Workaround for using wifi module on S2 with USB-CDC console.

---------

Co-authored-by: Jade Mattsson <github@frozenlogic.org>

.github/workflows/build.yml

@@ -10,19 +10,19 @@ jobs:
       matrix:
         lua_ver: ['5.1', '5.3']
         numbers: ['default', 'alternate']
-        target: ['esp32', 'esp32s2', 'esp32s3', 'esp32c3']
+        target: ['esp32', 'esp32s2', 'esp32s3', 'esp32c3', 'esp32c6', 'esp32h2']
 
     runs-on: ubuntu-20.04
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           submodules: recursive
       - name: Prepare cache key
         run: git rev-parse HEAD:sdk/esp32-esp-idf > idf.rev
         shell: bash
       - name: Cache Espressif tools
-        uses: actions/cache@v3
+        uses: actions/cache@v4
         with:
           path: ~/.espressif
           key: ${{ runner.os }}-espressif-tools-${{ hashFiles('idf.rev') }}
@@ -33,6 +33,11 @@ jobs:
         run: |
           cp sdkconfig.defaults sdkconfig
         shell: bash
+      - name: Update config for Lua 5.1
+        if: ${{ matrix.lua_ver == '5.1' }}
+        run: |
+          echo CONFIG_LUA_VERSION_51=y >> sdkconfig
+        shell: bash
       - name: Update config for Lua 5.1, integer-only
         if: ${{ matrix.lua_ver == '5.1' && matrix.numbers == 'alternate' }}
         run: |
@@ -58,7 +63,7 @@ jobs:
           echo lua_build_opts="$(expr "$(./build/luac_cross/luac.cross -v)" : '.*\[\(.*\)\]')" >> $GITHUB_ENV
         shell: bash
       - name: Upload luac.cross
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         if: ${{ success() }}
         with:
           name: luac.cross-${{ env.lua_build_opts }}-${{ matrix.target }}

components/base_nodemcu/CMakeLists.txt

@@ -2,6 +2,6 @@ idf_component_register(
   SRCS "ip_fmt.c" "user_main.c"
   INCLUDE_DIRS "include"
   REQUIRES "lua"
-  PRIV_REQUIRES "nvs_flash" "spiffs" "esp_netif" "driver"
+  PRIV_REQUIRES "nvs_flash" "spiffs" "esp_netif" "driver" "vfs"
   LDFRAGMENTS "nodemcu.lf"
 )

components/base_nodemcu/user_main.c

@@ -13,11 +13,16 @@
 #include "platform.h"
 #include <string.h>
 #include <stdlib.h>
+#include <fcntl.h>
 #include "sdkconfig.h"
 #include "esp_system.h"
 #include "esp_event.h"
 #include "esp_spiffs.h"
 #include "esp_netif.h"
+#include "esp_vfs_dev.h"
+#include "esp_vfs_cdcacm.h"
+#include "esp_vfs_usb_serial_jtag.h"
+#include "driver/usb_serial_jtag.h"
 #include "nvs_flash.h"
 
 #include "task/task.h"
@@ -48,6 +53,8 @@ typedef struct {
 static task_handle_t     relayed_event_task;
 static SemaphoreHandle_t relayed_event_handled;
 
+static task_handle_t lua_feed_task;
+
 
 // This function runs in the context of the system default event loop RTOS task
 static void relay_default_loop_events(
@@ -98,7 +105,7 @@ static void start_lua ()
     lua_main();
 }
 
-void nodemcu_init(void)
+static void nodemcu_init(void)
 {
     NODE_ERR("\n");
     // Initialize platform first for lua modules.
@@ -141,10 +148,111 @@ void nodemcu_init(void)
 }
 
 
+static void console_nodemcu_task(task_param_t param, task_prio_t prio)
+{
+  (void)prio;
+  char c = (char)param;
+  feed_lua_input(&c, 1);
+  // The IDF doesn't seem to honor setvbuf(stdout, NULL, _IONBF, 0) :(
+  fsync(fileno(stdout));
+}
+
+
+static void console_task(void *)
+{
+  for (;;)
+  {
+    /* We can't use a large read buffer here as some console choices
+     * (e.g. usb-serial-jtag) don't support read timeouts/partial reads,
+     * which breaks the echo support and makes for a bad user experience.
+     */
+    char c;
+    ssize_t n = read(fileno(stdin), &c, 1);
+    if (n > 0 && run_input)
+    {
+      if (!task_post_block_high(lua_feed_task, (task_param_t)c))
+      {
+        NODE_ERR("Lost console input data?!\n");
+      }
+    }
+  }
+}
+
+
+static void console_init(void)
+{
+  fflush(stdout);
+  fsync(fileno(stdout));
+
+  /* Disable buffering */
+  setvbuf(stdin, NULL, _IONBF, 0);
+  setvbuf(stdout, NULL, _IONBF, 0);
+
+  /* Disable non-blocking mode */
+  fcntl(fileno(stdin), F_SETFL, 0);
+  fcntl(fileno(stdout), F_SETFL, 0);
+
+#if CONFIG_ESP_CONSOLE_UART_DEFAULT || CONFIG_ESP_CONSOLE_UART_CUSTOM
+  /* Based on console/advanced example */
+
+  esp_vfs_dev_uart_port_set_rx_line_endings(
+    CONFIG_ESP_CONSOLE_UART_NUM, ESP_LINE_ENDINGS_CR);
+  esp_vfs_dev_uart_port_set_tx_line_endings(
+    CONFIG_ESP_CONSOLE_UART_NUM, ESP_LINE_ENDINGS_CRLF);
+
+  /* Configure UART. Note that REF_TICK is used so that the baud rate remains
+   * correct while APB frequency is changing in light sleep mode.
+   */
+  const uart_config_t uart_config = {
+    .baud_rate = CONFIG_ESP_CONSOLE_UART_BAUDRATE,
+    .data_bits = UART_DATA_8_BITS,
+    .parity = UART_PARITY_DISABLE,
+    .stop_bits = UART_STOP_BITS_1,
+#if SOC_UART_SUPPORT_REF_TICK
+    .source_clk = UART_SCLK_REF_TICK,
+#elif SOC_UART_SUPPORT_XTAL_CLK
+    .source_clk = UART_SCLK_XTAL,
+#endif
+  };
+  /* Install UART driver for interrupt-driven reads and writes */
+  uart_driver_install(CONFIG_ESP_CONSOLE_UART_NUM, 256, 0, 0, NULL, 0);
+  uart_param_config(CONFIG_ESP_CONSOLE_UART_NUM, &uart_config);
+
+  /* Tell VFS to use UART driver */
+  esp_vfs_dev_uart_use_driver(CONFIG_ESP_CONSOLE_UART_NUM);
+
+#elif CONFIG_ESP_CONSOLE_USB_SERIAL_JTAG
+  /* Based on @pjsg's work */
+
+  esp_vfs_dev_usb_serial_jtag_set_rx_line_endings(ESP_LINE_ENDINGS_CR);
+  esp_vfs_dev_usb_serial_jtag_set_tx_line_endings(ESP_LINE_ENDINGS_CRLF);
+
+  usb_serial_jtag_driver_config_t usb_serial_jtag_config =
+    USB_SERIAL_JTAG_DRIVER_CONFIG_DEFAULT();
+  /* Install USB-SERIAL-JTAG driver for interrupt-driven reads and write */
+  usb_serial_jtag_driver_install(&usb_serial_jtag_config);
+
+  esp_vfs_usb_serial_jtag_use_driver();
+#elif CONFIG_ESP_CONSOLE_USB_CDC
+  /* Based on console/advanced_usb_cdc */
+
+  esp_vfs_dev_cdcacm_set_rx_line_endings(ESP_LINE_ENDINGS_CR);
+  esp_vfs_dev_cdcacm_set_tx_line_endings(ESP_LINE_ENDINGS_CRLF);
+#else
+# error "Unsupported console type"
+#endif
+
+  xTaskCreate(
+    console_task, "console", 1024, NULL, ESP_TASK_MAIN_PRIO+1, NULL);
+}
+
+
 void __attribute__((noreturn)) app_main(void)
 {
   task_init();
 
+  lua_feed_task = task_get_id(console_nodemcu_task);
+
   relayed_event_handled = xSemaphoreCreateBinary();
   relayed_event_task = task_get_id(handle_default_loop_event);
 
@@ -155,14 +263,13 @@ void __attribute__((noreturn)) app_main(void)
     relay_default_loop_events,
     NULL);
 
-  platform_uart_start(CONFIG_ESP_CONSOLE_UART_NUM);
-  setvbuf(stdout, NULL, _IONBF, 0);
-
   nodemcu_init ();
 
   nvs_flash_init ();
   esp_netif_init ();
 
+  console_init();
+
   start_lua ();
   task_pump_messages ();
   __builtin_unreachable ();

components/lua/common/linput.c

@@ -3,14 +3,17 @@
 #include "lua.h"
 #include "lauxlib.h"
 #include <stdio.h>
+#include <string.h>
 
 static struct input_state {
   char       *data;
   int         line_pos;
   size_t      len;
   const char *prompt;
   char last_nl_char;
-} ins = {0};
+} ins = {
+  .prompt = "? ", // prompt should never be allowed to be null
+};
 
 #define NUL '\0'
 #define BS  '\010'
@@ -22,6 +25,15 @@ static struct input_state {
 bool input_echo = true;
 bool run_input = true;
 
+void input_setprompt (const char *prompt) {
+  if (prompt == NULL)
+  {
+    fprintf(stderr, "Error: attempted to set a null prompt?!");
+    return;
+  }
+  ins.prompt = prompt;
+}
+
 /*
 ** The input state (ins) is private, so input_setup() exposes the necessary
 ** access to public properties and is called in user_init() before the Lua
@@ -31,11 +43,8 @@ void input_setup(int bufsize, const char *prompt) {
   // Initialise non-zero elements
   ins.data      = malloc(bufsize);
   ins.len       = bufsize;
-  ins.prompt    = prompt;
-}
-
-void input_setprompt (const char *prompt) {
-  ins.prompt = prompt;
+  // Call to get the prompt error checking
+  input_setprompt(prompt);
 }
 
 
@@ -59,7 +68,7 @@ size_t feed_lua_input(const char *buf, size_t n)
     /* backspace key */
     if (ch == DEL || ch == BS) {
       if (ins.line_pos > 0) {
-        if(input_echo) printf(BS_OVER);
+        if(input_echo) fwrite(BS_OVER, strlen(BS_OVER), 1, stdout);
         ins.line_pos--;
       }
       ins.data[ins.line_pos] = 0;
@@ -73,7 +82,7 @@ size_t feed_lua_input(const char *buf, size_t n)
       if (input_echo) putchar(LF);
       if (ins.line_pos == 0) {
         /* Get a empty line, then go to get a new line */
-        printf(ins.prompt);
+        fwrite(ins.prompt, strlen(ins.prompt), 1, stdout);
         fflush(stdout);
       } else {
         ins.data[ins.line_pos++] = LF;

components/lua/lua-5.1/ldump.c

@@ -178,7 +178,7 @@ static void DumpCode(const Proto *f, DumpState* D)
 
 static void DumpString(const TString* s, DumpState* D)
 {
- if (s==NULL || getstr(s)==NULL)
+ if (s==NULL)
  {
   strsize_t size=0;
   DumpSize(size,D);

components/lua/lua-5.3/lua.c

@@ -6,6 +6,7 @@
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
+#include <unistd.h>
 #include "user_version.h"
 #include "linput.h"
 
@@ -211,6 +212,7 @@ static void dojob (lua_State *L) {
   prompt = get_prompt(L, MLref!= LUA_NOREF ? 0 : 1);
   input_setprompt(prompt);
   lua_writestring(prompt,strlen(prompt));
+  fsync(fileno(stdout)); /* work around IDF issue on ACM consoles */
 }
 
 

components/modules/CMakeLists.txt

@@ -1,4 +1,11 @@
 # Modules common to all chips
+set(wifi_modules
+  "espnow.c"
+  "wifi.c"
+  "wifi_ap.c"
+  "wifi_common.c"
+  "wifi_sta.c"
+)
 set(module_srcs
   "adc.c"
   "bit.c"
@@ -8,7 +15,6 @@ set(module_srcs
   "dht.c"
   "encoder.c"
   "eromfs.c"
-  "espnow.c"
   "file.c"
   "gpio.c"
   "heaptrace.c"
@@ -24,6 +30,7 @@ set(module_srcs
   "otaupgrade.c"
   "ow.c"
   "pipe.c"
+  "rmt.c"
   "rtcmem.c"
   "qrcodegen.c"
   "sigma_delta.c"
@@ -37,10 +44,6 @@ set(module_srcs
   "u8g2.c"
   "uart.c"
   "ucg.c"
-  "wifi.c"
-  "wifi_ap.c"
-  "wifi_common.c"
-  "wifi_sta.c"
   "ws2812.c"
 )
 
@@ -53,25 +56,34 @@ if(IDF_TARGET STREQUAL "esp32")
     "eth.c"
     "i2s.c"
     "pulsecnt.c"
-    "rmt.c"
     "sdmmc.c"
     "touch.c"
+    ${wifi_modules}
   )
 elseif(IDF_TARGET STREQUAL "esp32s2")
   list(APPEND module_srcs
     "dac.c"
     "pulsecnt.c"
-    "rmt.c"
+    ${wifi_modules}
   )
 elseif(IDF_TARGET STREQUAL "esp32s3")
   list(APPEND module_srcs
+    "dac.c"
     "pulsecnt.c"
-    "rmt.c"
     "sdmmc.c"
+    ${wifi_modules}
   )
 elseif(IDF_TARGET STREQUAL "esp32c3")
   list(APPEND module_srcs
-    "rmt.c"
+  )
+elseif(IDF_TARGET STREQUAL "esp32c6")
+  list(APPEND module_srcs
+    "dac.c"
+    "pulsecnt.c"
+    ${wifi_modules}
+  )
+elseif(IDF_TARGET STREQUAL "esp32h2")
+  list(APPEND module_srcs
   )
 endif()