Auto detect flash size and adjust FS (#349) (#19215)

pio-tools/get_flash_size.py

@@ -0,0 +1,43 @@
+from os.path import join
+import subprocess
+from SCons.Script import COMMAND_LINE_TARGETS
+
+Import("env")
+env = DefaultEnvironment()
+platform = env.PioPlatform()
+
+if "upload" in COMMAND_LINE_TARGETS:
+
+    def esp32_detect_flashsize():
+        esptoolpy = join(platform.get_package_dir("tool-esptoolpy") or "", "esptool.py")
+        esptoolpy_flags = ["flash_id"]
+        esptoolpy_cmd = [env["PYTHONEXE"], esptoolpy] + esptoolpy_flags
+        try:
+            output = subprocess.run(esptoolpy_cmd, capture_output=True).stdout.splitlines()
+            for l in output:
+                if l.decode().startswith("Detected flash size: "):
+                    size = (l.decode().split(": ")[1])
+                    print("Did get flash size:",size)
+                    return size, True
+            return "4MB",False
+        except subprocess.CalledProcessError as exc:
+            print("Did get chip info failed with " + str(exc))
+            return "4MB",False
+
+    size, overridden = esp32_detect_flashsize()
+    old_flash_size = env.BoardConfig().get("upload.flash_size")
+    old_maximum_size = env.BoardConfig().get("upload.maximum_size")
+    new_maximum_size = int(size.split("MB")[0]) * 0x100000
+
+    extra_flags = env.BoardConfig().get("build.extra_flags").split(" ")
+    new_flags = ""
+    for flag in extra_flags:
+        if flag.startswith("-DESP32"):
+            flag = f"-DESP32_{size}"
+        new_flags += flag + " "
+    env.BoardConfig().update("build.extra_flags",new_flags)
+    env.BoardConfig().update("upload.flash_size",size)
+
+    env.BoardConfig().update("upload.maximum_size", new_maximum_size)
+
+    env.Replace(TASMOTA_flash_size=size)

pio-tools/post_esp32.py

@@ -1,6 +1,8 @@
 # From: https://github.com/letscontrolit/ESPEasy/blob/mega/tools/pio/post_esp32.py
 # Thanks TD-er :)
 
+# Thanks @staars for safeboot and auto resizing LittleFS code and enhancements
+
 # Combines separate bin files with their respective offsets into a single file
 # This single file must then be flashed to an ESP32 node with 0 offset.
 #
@@ -29,6 +31,8 @@
 import requests
 import shutil
 import subprocess
+import codecs
+from SCons.Script import COMMAND_LINE_TARGETS
 
 sys.path.append(join(platform.get_package_dir("tool-esptoolpy")))
 import esptool
@@ -52,6 +56,29 @@
 
 variants_dir = join(FRAMEWORK_DIR, "variants", "tasmota")
 
+def patch_partitions_bin(size_string):
+    partition_bin_path = join(env.subst("$BUILD_DIR"),"partitions.bin")
+    with open(partition_bin_path, 'r+b') as file:
+        binary_data = file.read(0xb0)
+        import hashlib
+        bin_list = list(binary_data)
+        size = codecs.decode(size_string[2:], 'hex_codec') # 0xc50000 -> [c5,00,00]
+        bin_list[0x8a] = size[0]
+        bin_list[0x8b] = size[1]
+        result = hashlib.md5(bytes(bin_list[0:0xa0]))
+        partition_data = bytes(bin_list) + result.digest()
+        file.seek(0)
+        file.write(partition_data)
+        print("New partition hash:",result.digest().hex())
+
+def esp32_detect_flashsize():
+    if not "upload" in COMMAND_LINE_TARGETS:
+        return "4MB",False
+    try:
+        return env.get("TASMOTA_flash_size"),True
+    except:
+        return "4MB",False
+
 def esp32_create_chip_string(chip):
     tasmota_platform = env.subst("$BUILD_DIR").split(os.path.sep)[-1]
     tasmota_platform = tasmota_platform.split('-')[0]
@@ -120,6 +147,8 @@ def esp32_create_combined_bin(source, target, env):
     # factory_offset = -1      # error code value - currently unused
     app_offset = 0x10000     # default value for "old" scheme
     fs_offset = -1           # error code value
+    flash_size_from_esp, flash_size_was_overridden = esp32_detect_flashsize()
+
     with open(env.BoardConfig().get("build.partitions")) as csv_file:
         print("Read partitions from ",env.BoardConfig().get("build.partitions"))
         csv_reader = csv.reader(csv_file, delimiter=',')
@@ -136,7 +165,13 @@ def esp32_create_combined_bin(source, target, env):
                 # elif(row[0] == 'factory'):
                 #     factory_offset = int(row[3],base=16)
                 elif(row[0] == 'spiffs'):
-                    if esp32_build_filesystem(row[4]):
+                    partition_size = row[4]
+                    if flash_size_was_overridden:
+                        print(f"Will override fixed FS partition size from {env.BoardConfig().get('build.partitions')}: {partition_size} ...")
+                        partition_size =  hex(int(flash_size_from_esp.split("MB")[0]) * 0x100000 - int(row[3],base=16))
+                        print(f"... with computed maximum size from connected {env.get('BOARD_MCU')}: {partition_size}")
+                        patch_partitions_bin(partition_size)
+                    if esp32_build_filesystem(partition_size):
                         fs_offset = int(row[3],base=16)
 
 
@@ -147,16 +182,19 @@ def esp32_create_combined_bin(source, target, env):
     tasmota_platform = esp32_create_chip_string(chip)
 
     if "-DUSE_USB_CDC_CONSOLE" in env.BoardConfig().get("build.extra_flags") and "cdc" not in tasmota_platform:
-        tasmota_platform += "cdc" 
+        tasmota_platform += "cdc"
         print("WARNING: board definition uses CDC configuration, but environment name does not -> changing tasmota safeboot binary to:", tasmota_platform + "-safeboot.bin")
- 
+
     if not os.path.exists(variants_dir):
         os.makedirs(variants_dir)
     if("safeboot" in firmware_name):
         esp32_copy_new_safeboot_bin(tasmota_platform,firmware_name)
     else:
         esp32_fetch_safeboot_bin(tasmota_platform)
+
     flash_size = env.BoardConfig().get("upload.flash_size", "4MB")
+    if flash_size_was_overridden:
+        flash_size = flash_size_from_esp
     flash_freq = env.BoardConfig().get("build.f_flash", "40000000L")
     flash_freq = str(flash_freq).replace("L", "")
     flash_freq = str(int(int(flash_freq) / 1000000)) + "m"
@@ -225,4 +263,4 @@ def esp32_create_combined_bin(source, target, env):
     esptool.main(cmd)
 
 
-env.AddPostAction("$BUILD_DIR/${PROGNAME}.bin", esp32_create_combined_bin)
+env.AddPostAction("$BUILD_DIR/${PROGNAME}.bin", esp32_create_combined_bin)

platformio_tasmota32.ini

@@ -37,6 +37,7 @@ build_flags                 = ${esp_defaults.build_flags}
                               -Wl,--wrap=_Z11analogWritehi  ; `analogWrite(unsigned char, int)` use the Tasmota version of analogWrite for deeper integration and phase control
                               -Wl,--wrap=ledcReadFreq  ; `uint32_t ledcReadFreq(uint8_t chan)`
 extra_scripts               = pre:pio-tools/add_c_flags.py
+                              pre:pio-tools/get_flash_size.py
                               pre:pio-tools/gen-berry-structures.py
                               post:pio-tools/post_esp32.py
                               ${esp_defaults.extra_scripts}

platformio_tasmota_cenv_sample.ini

@@ -22,13 +22,6 @@ board_build.f_flash         = 80000000L
 build_flags                 = ${env:tasmota32_base.build_flags}
                               -DFIRMWARE_TASMOTA32
                               -DOTA_URL='"http://ota.tasmota.com/tasmota32/release/tasmota32s3.bin"'
-; !!! Real flash size needed, avoid autoresize since it is formating FS !!!
-board_upload.flash_size     = 8MB
-board_upload.maximum_size   = 8388608
-; Without autoresize a partition scheme is needed which does fit to flash size
-board_build.partitions      = partitions/esp32_partition_app2944k_fs2M.csv
-; Dont use safeboot, not used in this partition scheme -> an empty entry needed to overwrite the default setting
-board_upload.arduino.flash_extra_images =
 ; Example for custom file upload in Tasmota Filesystem
 custom_files_upload         = ${env:tasmota32_base.custom_files_upload}
                               tasmota/berry/modules/Partition_Wizard.tapp