LFS support for ESP32 NodeMCU

components/base_nodemcu/Makefile.projbuild

@@ -1,8 +1,14 @@
 BASE_NODEMCU_DIR:=$(dir $(lastword $(MAKEFILE_LIST)))
 BASE_NODEMCU_BUILD_DIR:=$(BUILD_DIR_BASE)/base_nodemcu
 
+ifeq ($(CONFIG_LUA_EMBED_LFS),y)
+ NODEMCU_LD_SCRIPT:= nodemcu_rodata_lfs.ld
+else
+ NODEMCU_LD_SCRIPT:= nodemcu_rodata.ld
+endif
+
 $(BUILD_DIR_BASE)/$(PROJECT_NAME).elf: $(BASE_NODEMCU_BUILD_DIR)/ld_patched
 
 $(BASE_NODEMCU_BUILD_DIR)/ld_patched: $(BUILD_DIR_BASE)/esp32/esp32.project.ld
-	"$(BASE_NODEMCU_DIR)/add_rodata_ld.sh" "$<" "$(BASE_NODEMCU_DIR)/ld/nodemcu_rodata.ld"
+	"$(BASE_NODEMCU_DIR)/add_rodata_ld.sh" "$<" "$(BASE_NODEMCU_DIR)/ld/$(NODEMCU_LD_SCRIPT)"
 	touch $@

components/base_nodemcu/include/module.h

@@ -1,8 +1,8 @@
 #ifndef __MODULE_H__
 #define __MODULE_H__
 
-#include "lrodefs.h"
 #include "sdkconfig.h"
+#include "lrotable.h"
 
 /* Registering a module within NodeMCU is really easy these days!
  *
@@ -36,37 +36,37 @@
 #define MODULE_PASTE_(x,y) x##y
 #define MODULE_EXPAND_PASTE_(x,y) MODULE_PASTE_(x,y)
 
-#define LOCK_IN_SECTION(s) __attribute__((used,unused,section(s)))
-
+#ifdef LUA_CROSS_COMPILER
+#ifdef _MSC_VER
+//on msvc it is necessary to go through more pre-processor hoops to get the
+//section name built; string merging does not happen in the _declspecs.
+//NOTE: linker magic is invoked via the magical '$' character. Caveat editor.
+#define __TOKIFY(s) .rodata1$##s
+#define __TOTOK(s) __TOKIFY(s)
+#define __STRINGIFY(x) #x
+#define __TOSTRING(x) __STRINGIFY(x)
+#define __ROSECNAME(s) __TOSTRING(__TOTOK(s))
+#define LOCK_IN_SECTION(s) __declspec ( allocate( __ROSECNAME(s) ) )
+#else
+#define LOCK_IN_SECTION(s) __attribute__((used,unused,section(".rodata1." #s)))
+#endif
+#else
+#define LOCK_IN_SECTION(s) __attribute__((used,unused,section(".lua_" #s)))
+#endif
 /* For the ROM table, we name the variable according to ( | denotes concat):
  *   cfgname | _module_selected | CONFIG_LUA_MODULE_##cfgname
  * where the CONFIG_LUA_MODULE_XYZ macro is first expanded to yield either
  * an empty string (or 1) if the module has been enabled, or the literal
  * CONFIG_LUA_MODULE_XYZ in the case it hasn't. Thus, the name of the variable
  * ends up looking either like XYZ_module_enabled, or if not enabled,
- * XYZ_module_enabledLUA_USE_MODULES_XYZ.  This forms the basis for
+ * XYZ_module_enabledCONFIG_LUA_MODULE_XYZ.  This forms the basis for
  * letting the build system detect automatically (via nm) which modules need
  * to be linked in.
  */
 #define NODEMCU_MODULE(cfgname, luaname, map, initfunc) \
-  const LOCK_IN_SECTION(".lua_libs") \
-    luaL_Reg MODULE_PASTE_(lua_lib_,cfgname) = { luaname, initfunc }; \
-  const LOCK_IN_SECTION(".lua_rotable") \
-    luaR_table MODULE_EXPAND_PASTE_(cfgname,MODULE_EXPAND_PASTE_(_module_selected,MODULE_PASTE_(CONFIG_LUA_MODULE_,cfgname))) \
-    = { luaname, map }
-
-
-/* System module registration support, not using CONFIG_LUA_MODULE_XYZ. */
-#define BUILTIN_LIB_INIT(name, luaname, initfunc) \
-  const LOCK_IN_SECTION(".lua_libs") \
-    luaL_Reg MODULE_PASTE_(lua_lib_,name) = { luaname, initfunc }
-
-#define BUILTIN_LIB(name, luaname, map) \
-  const LOCK_IN_SECTION(".lua_rotable") \
-    luaR_table MODULE_PASTE_(lua_rotable_,name) = { luaname, map }
-
-#if !defined(LUA_CROSS_COMPILER) && !(MIN_OPT_LEVEL==2 && LUA_OPTIMIZE_MEMORY==2)
-# error "NodeMCU modules must be built with LTR enabled (MIN_OPT_LEVEL=2 and LUA_OPTIMIZE_MEMORY=2)"
-#endif
-
+  const LOCK_IN_SECTION(libs) \
+    luaR_entry MODULE_PASTE_(lua_lib_,cfgname) = { luaname, LRO_FUNCVAL(initfunc) }; \
+  const LOCK_IN_SECTION(rotable) \
+    luaR_entry MODULE_EXPAND_PASTE_(cfgname,MODULE_EXPAND_PASTE_(_module_selected,MODULE_PASTE_(CONFIG_LUA_MODULE_,cfgname))) \
+    = {luaname, LRO_ROVAL(map ## _map)}
 #endif

components/base_nodemcu/include/nodemcu_esp_event.h

@@ -1,5 +1,5 @@
 /*
- * Copyright 2016 Dius Computing Pty Ltd. All rights reserved.
+ * Copyright 2016-2019 Dius Computing Pty Ltd. All rights reserved.
  *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
@@ -36,6 +36,8 @@
 
 #include "esp_event.h"
 #include "module.h"
+#include <assert.h>
+#include <stdalign.h>
 
 /**
  * Similarly to how a NodeMCU module is registered, a module can register
@@ -66,7 +68,10 @@ typedef struct {
 extern nodemcu_esp_event_reg_t esp_event_cb_table;
 
 #define NODEMCU_ESP_EVENT(evcode, func) \
-  static const LOCK_IN_SECTION(".esp_event_cb_table") \
+  static const LOCK_IN_SECTION(esp_event_cb_table) \
     nodemcu_esp_event_reg_t MODULE_PASTE_(func,evcode) = { evcode, func };
 
+_Static_assert(_Alignof(nodemcu_esp_event_reg_t) == 4, "Unexpected alignment of event registration - update linker script snippets to match!");
+_Static_assert(sizeof(nodemcu_esp_event_reg_t) == 8, "Unexpected size of array member - update the linker script snippets to match!");
+
 #endif

components/base_nodemcu/ld/nodemcu_rodata.ld

@@ -1,17 +1,29 @@
 
     /* ----- Begin NodeMCU link-time arrays ------- */
 
-    . = ALIGN(4);
+    /* Don't change the alignment here without also updating the _Static_assert
+     * over in linit.c! */
+    . = ALIGN(8);
     /* Link-time arrays containing the defs for the included modules */
-    lua_libs = ABSOLUTE(.);
+    lua_libs_map = ABSOLUTE(.);
     KEEP(*(.lua_libs))
-    LONG(0) LONG(0) /* Null-terminate the array */
-    lua_rotable = ABSOLUTE(.);
+    /* Null-terminate the array, 24 bytes */
+    LONG(0) LONG(0) LONG(0) LONG(0) LONG(0) LONG(0)
+
+    /* Don't change the alignment here without also updating the _Static_assert
+     * over in linit.c! */
+    . = ALIGN(8);
+    lua_rotables_map = ABSOLUTE(.);
     KEEP(*(.lua_rotable))
-    LONG(0) LONG(0) /* Null-terminate the array */
+    /* Null-terminate the array, 24 bytes */
+    LONG(0) LONG(0) LONG(0) LONG(0) LONG(0) LONG(0)
+
+    /* Don't change the alignment here without also updating the _Static_assert
+     * over in nodemcu_esp_event.h! */
+    . = ALIGN(4);
     esp_event_cb_table = ABSOLUTE(.);
-    KEEP(*(.esp_event_cb_table))
-    LONG(0) LONG(0) /* Null-terminate the array */
+    KEEP(*(.lua_esp_event_cb_table))
+    LONG(0) LONG(0) /* Null-terminate the array, 8 bytes */
 
     /* ----- End NodeMCU link-time arrays ------- */
 

components/base_nodemcu/ld/nodemcu_rodata_lfs.ld

@@ -0,0 +1,37 @@
+
+    /* ----- Begin NodeMCU link-time arrays ------- */
+
+    /* Don't change the alignment here without also updating the _Static_assert
+     * over in linit.c! */
+    . = ALIGN(8);
+    /* Link-time arrays containing the defs for the included modules */
+    lua_libs_map = ABSOLUTE(.);
+    KEEP(*(.lua_libs))
+    /* Null-terminate the array, 24 bytes */
+    LONG(0) LONG(0) LONG(0) LONG(0) LONG(0) LONG(0)
+
+    /* Don't change the alignment here without also updating the _Static_assert
+     * over in linit.c! */
+    . = ALIGN(8);
+    lua_rotables_map = ABSOLUTE(.);
+    KEEP(*(.lua_rotable))
+    /* Null-terminate the array, 24 bytes */
+    LONG(0) LONG(0) LONG(0) LONG(0) LONG(0) LONG(0)
+
+    /* Don't change the alignment here without also updating the _Static_assert
+     * over in nodemcu_esp_event.h! */
+    . = ALIGN(4);
+    esp_event_cb_table = ABSOLUTE(.);
+    KEEP(*(.lua_esp_event_cb_table))
+    LONG(0) LONG(0) /* Null-terminate the array, 8 bytes */
+
+    /* ----- End NodeMCU link-time arrays ------- */
+
+    /* ----- NodeMCU embedded LFS reserved area --------- */
+
+    . = ALIGN(4096); /* flash page alignment needed */
+    lua_flash_store_reserved = ABSOLUTE(.);
+    KEEP(*(.lfs.reserved))
+
+    /* ----- End NodeMCU embedded LFS reserved area ----- */
+

components/base_nodemcu/linit.c

@@ -15,64 +15,111 @@
 #include "lauxlib.h"
 #include "luaconf.h"
 #include "module.h"
-#include "sdkconfig.h"
+#include "lstate.h"
+#include <assert.h>
+#include <stdalign.h>
 
-BUILTIN_LIB_INIT( BASE,      "",                 luaopen_base);
-BUILTIN_LIB_INIT( LOADLIB,   LUA_LOADLIBNAME,    luaopen_package);
-
-#if defined(LUA_USE_BUILTIN_IO)
-BUILTIN_LIB_INIT( IO,        LUA_IOLIBNAME,      luaopen_io);
-#endif
-
-#if defined (CONFIG_LUA_BUILTIN_STRING)
-extern const luaR_entry strlib[];
-BUILTIN_LIB_INIT( STRING,    LUA_STRLIBNAME,     luaopen_string);
-BUILTIN_LIB(      STRING,    LUA_STRLIBNAME,     strlib);
+LROT_EXTERN(strlib);
+LROT_EXTERN(tab_funcs);
+LROT_EXTERN(dblib);
+LROT_EXTERN(co_funcs);
+LROT_EXTERN(math);
+#if defined(LUA_CROSS_COMPILER)
+LROT_EXTERN(oslib);
+//LROT_EXTERN(iolib);
 #endif
+/*
+ * The NodeMCU Lua initalisation has been adapted to use linker-based module
+ * registration.  This uses a PSECT naming convention to allow the ROTable and
+ * initialisation function entries to be collected by the linker into two
+ * consoliated ROTables.  This significantly simplifies adding new modules and
+ * configuring builds with a small subset of the total modules.
+ *
+ * This linker-based approach is not practical for cross compiler builds which
+ * must link on a range of platforms, and where we don't have control of PSECT
+ * placement.   However unlike the target builds, the  luac.cross builds only
+ * used a small and fixed list of libraries and so in this case the table can
+ * be defined in this source file, so in this case all library ROTables are
+ * defined here, avoiding the need for linker magic is avoided on host builds.
+ *
+ * Note that a separate ROTable is defined in lbaselib.c for the base functions
+ * and there is a metatable index cascade from _G to this base function table to
+ * the master rotables table.  In the target build, the linker marshals the
+ * table, hence the LROT_BREAK() macros which don't 0 terminate the lists.
+ */
 
-#if defined(CONFIG_LUA_BUILTIN_TABLE)
-extern const luaR_entry tab_funcs[];
-BUILTIN_LIB_INIT( TABLE,     LUA_TABLIBNAME,     luaopen_table);
-BUILTIN_LIB(      TABLE,     LUA_TABLIBNAME,     tab_funcs);
+#ifdef _MSC_VER
+//MSVC requires us to declare these sections before we refer to them
+#pragma section(__ROSECNAME(A), read)
+#pragma section(__ROSECNAME(zzzzzzzz), read)
+#pragma section(__ROSECNAME(libs), read)
+#pragma section(__ROSECNAME(rotable), read)
+//These help us to find the beginning and ending of the RO data.  NOTE:  linker
+//magic is used; the section names are lexically sorted, so 'a' and 'z' are
+//important to keep the other sections lexically between these two dummy
+//variables.  Check your mapfile output if you need to fiddle with this stuff.
+const LOCK_IN_SECTION(A) char _ro_start[1] = {0};
+const LOCK_IN_SECTION(zzzzzzzz) char _ro_end[1] = {0};
 #endif
 
-#if defined(CONFIG_LUA_BUILTIN_DEBUG)
-extern const luaR_entry dblib[];
-BUILTIN_LIB_INIT( DBG,       LUA_DBLIBNAME,      luaopen_debug);
-BUILTIN_LIB(      DBG,       LUA_DBLIBNAME,      dblib);
+LROT_PUBLIC_BEGIN(LOCK_IN_SECTION(rotable) lua_rotables)
+  LROT_TABENTRY( string, strlib )
+  LROT_TABENTRY( table, tab_funcs )
+  LROT_TABENTRY( debug, dblib)
+  LROT_TABENTRY( coroutine, co_funcs )
+  LROT_TABENTRY( math, math )
+  LROT_TABENTRY( ROM, lua_rotables )
+#ifdef LUA_CROSS_COMPILER
+  LROT_TABENTRY( os, oslib )
+  //LROT_TABENTRY( io, iolib )
+LROT_END(lua_rotables, NULL, 0)
+#else
+LROT_BREAK(lua_rotables)
 #endif
 
-#if defined(LUA_USE_BUILTIN_OS)
-extern const luaR_entry syslib[];
-BUILTIN_LIB(      OS,        LUA_OSLIBNAME,     syslib);
+LROT_PUBLIC_BEGIN(LOCK_IN_SECTION(libs) lua_libs)
+  LROT_FUNCENTRY( _, luaopen_base )
+  LROT_FUNCENTRY( package, luaopen_package )
+  LROT_FUNCENTRY( string, luaopen_string )
+  LROT_FUNCENTRY( table, luaopen_table )
+  LROT_FUNCENTRY( debug, luaopen_debug )
+#ifndef LUA_CROSS_COMPILER
+LROT_BREAK(lua_rotables)
+#else
+  LROT_FUNCENTRY( io, luaopen_io )
+LROT_END( lua_libs, NULL, 0)
 #endif
 
-#if defined(CONFIG_LUA_BUILTIN_COROUTINE)
-extern const luaR_entry co_funcs[];
-BUILTIN_LIB(      CO,        LUA_COLIBNAME,     co_funcs);
+#ifndef LUA_CROSS_COMPILER
+extern void luaL_dbgbreak(void);
 #endif
 
-#if defined(CONFIG_LUA_BUILTIN_MATH)
-extern const luaR_entry math_map[];
-BUILTIN_LIB(      MATH,      LUA_MATHLIBNAME,   math_map);
-#endif
+void luaL_openlibs (lua_State *L) {
 
-#ifdef LUA_CROSS_COMPILER
-const luaL_Reg lua_libs[] = {{NULL, NULL}};
-const luaR_table lua_rotable[] = {{NULL, NULL}};
-#else 
-extern const luaL_Reg lua_libs[];
+  lua_pushrotable(L, LROT_TABLEREF(lua_libs));
+  lua_pushnil(L);  /* first key */
+  /* loop round and open libraries */
+#ifndef LUA_CROSS_COMPILER
+// luaL_dbgbreak();  // This is a test point for debugging library start ups
 #endif
-
-void luaL_openlibs (lua_State *L) {
-  const luaL_Reg *lib = lua_libs;
-  for (; lib->name; lib++) {
-    if (lib->func)
-    {
-      lua_pushcfunction(L, lib->func);
-      lua_pushstring(L, lib->name);
-      lua_call(L, 1, 0);
+  while (lua_next(L, -2) != 0) {
+    if (lua_islightfunction(L,-1) &&
+        fvalue(L->top-1)) { // only process function entries
+      lua_pushvalue(L, -2);
+      lua_call(L, 1, 0);  // call luaopen_XXX(libname)
+    } else {
+      lua_pop(L, 1);
     }
   }
+  lua_pop(L, 1);  //cleanup stack
 }
 
+#ifndef LUA_CROSS_COMPILER
+# ifdef LUA_NUMBER_INTEGRAL
+#  define COMPARE <=
+# else
+#  define COMPARE ==
+# endif
+_Static_assert(_Alignof(luaR_entry) COMPARE 8, "Unexpected alignment of module registration - update the linker script snippets to match!");
+_Static_assert(sizeof(luaR_entry) COMPARE 24, "Unexpect size of array member - update the linker script snippets to match!");
+#endif