Merge branch 'bugfix/fatfs_cannot_work_after_multiple_operations' int…

…o 'master' fix(ESPAT-1655): Fixed that AT+FS might return wrong data after multiple write See merge request application/esp-at!1537
espressif · Mar 20, 2024 · 195fa22 · 195fa22
2 parents 4a6ee01 + ebdb5b5
commit 195fa22
Show file tree

Hide file tree

Showing 23 changed files with 1,395 additions and 1 deletion.
diff --git a/components/customized_partitions/generation_tools/fatfs.py b/components/customized_partitions/generation_tools/fatfs.py
@@ -82,7 +82,7 @@ def main():
                     fat_type = 16
                     continue
 
-    fatfs_param = '--sector_size {} {} --sectors_per_cluster {} --fat_type {}'.format(
+    fatfs_param = '--sector_size {} {} --sectors_per_cluster {} --fat_type {} --wl_mode "safe"'.format(
             sector_size, long_name_support, sectors_per_cluster, fat_type)
 
     if sys.platform == 'win32':

diff --git a/module_config/module_esp32-d2wd/patch/fatfs_generation.patch b/module_config/module_esp32-d2wd/patch/fatfs_generation.patch
@@ -0,0 +1,123 @@
+diff --git a/components/fatfs/fatfs_utils/utils.py b/components/fatfs/fatfs_utils/utils.py
+index e7d2e25a7f..ddc68c9225 100644
+--- a/components/fatfs/fatfs_utils/utils.py
++++ b/components/fatfs/fatfs_utils/utils.py
+@@ -1,14 +1,17 @@
+ # SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+ # SPDX-License-Identifier: Apache-2.0
+-
+ import argparse
+ import binascii
+ import os
+ import uuid
+ from datetime import datetime
+-from typing import List, Optional, Tuple
++from typing import List
++from typing import Optional
++from typing import Tuple
+
+-from construct import BitsInteger, BitStruct, Int16ul
++from construct import BitsInteger
++from construct import BitStruct
++from construct import Int16ul
+
+ FAT12_MAX_CLUSTERS: int = 4085
+ FAT16_MAX_CLUSTERS: int = 65525
+@@ -200,6 +203,11 @@ def get_args_for_partition_generator(desc: str) -> argparse.Namespace:
+                         Type of fat. Select 12 for fat12, 16 for fat16. Don't set, or set to 0 for automatic
+                         calculation using cluster size and partition size.
+                         """)
++    parser.add_argument('--wl_mode',
++                        default=None,
++                        type=str,
++                        choices=['safe', 'perf'],
++                        help='Wear levelling mode to use. Safe or performance. Only for sector size of 512')
+
+     args = parser.parse_args()
+     if args.fat_type == 0:
+@@ -207,6 +215,9 @@ def get_args_for_partition_generator(desc: str) -> argparse.Namespace:
+     args.partition_size = int(str(args.partition_size), 0)
+     if not os.path.isdir(args.input_directory):
+         raise NotADirectoryError(f'The target directory `{args.input_directory}` does not exist!')
++    if args.wl_mode is not None:
++        if args.sector_size != 512:
++            raise ValueError('Wear levelling mode can be set only for sector size 512')
+     return args
+
+
+diff --git a/components/fatfs/wl_fatfsgen.py b/components/fatfs/wl_fatfsgen.py
+index b6c5dd4fe7..8881938f03 100755
+--- a/components/fatfs/wl_fatfsgen.py
++++ b/components/fatfs/wl_fatfsgen.py
+@@ -1,13 +1,19 @@
+ #!/usr/bin/env python
+-# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
++# SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD
+ # SPDX-License-Identifier: Apache-2.0
++from typing import List
++from typing import Optional
+
+-from typing import List, Optional
+-
+-from construct import Const, Int32ul, Struct
++from construct import Const
++from construct import Int32ul
++from construct import Struct
+ from fatfs_utils.exceptions import WLNotInitialized
+-from fatfs_utils.utils import (FULL_BYTE, UINT32_MAX, FATDefaults, crc32, generate_4bytes_random,
+-                               get_args_for_partition_generator)
++from fatfs_utils.utils import crc32
++from fatfs_utils.utils import FATDefaults
++from fatfs_utils.utils import FULL_BYTE
++from fatfs_utils.utils import generate_4bytes_random
++from fatfs_utils.utils import get_args_for_partition_generator
++from fatfs_utils.utils import UINT32_MAX
+ from fatfsgen import FATFS
+
+
+@@ -55,6 +61,7 @@ class WLFATFS:
+     WL_STATE_HEADER_SIZE = 64
+     WL_STATE_COPY_COUNT = 2  # always 2 copies for power failure safety
+     WL_SECTOR_SIZE = 0x1000
++    WL_SAFE_MODE_DUMP_SECTORS = 2
+
+     WL_STATE_T_DATA = Struct(
+         'pos' / Int32ul,
+@@ -99,7 +106,8 @@ class WLFATFS:
+                  temp_buff_size: int = FATDefaults.TEMP_BUFFER_SIZE,
+                  device_id: int = None,
+                  root_entry_count: int = FATDefaults.ROOT_ENTRIES_COUNT,
+-                 media_type: int = FATDefaults.MEDIA_TYPE) -> None:
++                 media_type: int = FATDefaults.MEDIA_TYPE,
++                 wl_mode: Optional[str] = None) -> None:
+         self._initialized = False
+         self._version = version
+         self._temp_buff_size = temp_buff_size
+@@ -107,6 +115,7 @@ class WLFATFS:
+         self.partition_size = size
+         self.total_sectors = self.partition_size // FATDefaults.WL_SECTOR_SIZE
+         self.wl_state_size = WLFATFS.WL_STATE_HEADER_SIZE + WLFATFS.WL_STATE_RECORD_SIZE * self.total_sectors
++        self.wl_mode = wl_mode
+
+         # determine the number of required sectors (roundup to sector size)
+         self.wl_state_sectors = (self.wl_state_size + FATDefaults.WL_SECTOR_SIZE - 1) // FATDefaults.WL_SECTOR_SIZE
+@@ -116,6 +125,9 @@ class WLFATFS:
+
+         wl_sectors = (WLFATFS.WL_DUMMY_SECTORS_COUNT + WLFATFS.WL_CFG_SECTORS_COUNT +
+                       self.wl_state_sectors * WLFATFS.WL_STATE_COPY_COUNT)
++        if self.wl_mode is not None and self.wl_mode == 'safe':
++            wl_sectors += WLFATFS.WL_SAFE_MODE_DUMP_SECTORS
++
+         self.plain_fat_sectors = self.total_sectors - wl_sectors
+         self.plain_fatfs = FATFS(
+             explicit_fat_type=explicit_fat_type,
+@@ -226,7 +238,8 @@ if __name__ == '__main__':
+                        root_entry_count=args.root_entry_count,
+                        explicit_fat_type=args.fat_type,
+                        long_names_enabled=args.long_name_support,
+-                       use_default_datetime=args.use_default_datetime)
++                       use_default_datetime=args.use_default_datetime,
++                       wl_mode=args.wl_mode)
+
+     wl_fatfs.wl_generate(args.input_directory)
+     wl_fatfs.init_wl()
diff --git a/module_config/module_esp32-d2wd/patch/patch_list.ini b/module_config/module_esp32-d2wd/patch/patch_list.ini
@@ -3,3 +3,7 @@
 [esp_tls_parse_ecc_key.patch]
     path = esp-idf
     note = "Fixed an issue where esp-tls is unable to parse ECC keys"
+
+[fatfs_generation.patch]
+    path = esp-idf
+    note = "Fixed an issue where data read after writing fatfs partition more than 63 times might be incorrect"
diff --git a/module_config/module_esp32-sdio/patch/fatfs_generation.patch b/module_config/module_esp32-sdio/patch/fatfs_generation.patch
@@ -0,0 +1,123 @@
+diff --git a/components/fatfs/fatfs_utils/utils.py b/components/fatfs/fatfs_utils/utils.py
+index e7d2e25a7f..ddc68c9225 100644
+--- a/components/fatfs/fatfs_utils/utils.py
++++ b/components/fatfs/fatfs_utils/utils.py
+@@ -1,14 +1,17 @@
+ # SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+ # SPDX-License-Identifier: Apache-2.0
+-
+ import argparse
+ import binascii
+ import os
+ import uuid
+ from datetime import datetime
+-from typing import List, Optional, Tuple
++from typing import List
++from typing import Optional
++from typing import Tuple
+
+-from construct import BitsInteger, BitStruct, Int16ul
++from construct import BitsInteger
++from construct import BitStruct
++from construct import Int16ul
+
+ FAT12_MAX_CLUSTERS: int = 4085
+ FAT16_MAX_CLUSTERS: int = 65525
+@@ -200,6 +203,11 @@ def get_args_for_partition_generator(desc: str) -> argparse.Namespace:
+                         Type of fat. Select 12 for fat12, 16 for fat16. Don't set, or set to 0 for automatic
+                         calculation using cluster size and partition size.
+                         """)
++    parser.add_argument('--wl_mode',
++                        default=None,
++                        type=str,
++                        choices=['safe', 'perf'],
++                        help='Wear levelling mode to use. Safe or performance. Only for sector size of 512')
+
+     args = parser.parse_args()
+     if args.fat_type == 0:
+@@ -207,6 +215,9 @@ def get_args_for_partition_generator(desc: str) -> argparse.Namespace:
+     args.partition_size = int(str(args.partition_size), 0)
+     if not os.path.isdir(args.input_directory):
+         raise NotADirectoryError(f'The target directory `{args.input_directory}` does not exist!')
++    if args.wl_mode is not None:
++        if args.sector_size != 512:
++            raise ValueError('Wear levelling mode can be set only for sector size 512')
+     return args
+
+
+diff --git a/components/fatfs/wl_fatfsgen.py b/components/fatfs/wl_fatfsgen.py
+index b6c5dd4fe7..8881938f03 100755
+--- a/components/fatfs/wl_fatfsgen.py
++++ b/components/fatfs/wl_fatfsgen.py
+@@ -1,13 +1,19 @@
+ #!/usr/bin/env python
+-# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
++# SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD
+ # SPDX-License-Identifier: Apache-2.0
++from typing import List
++from typing import Optional
+
+-from typing import List, Optional
+-
+-from construct import Const, Int32ul, Struct
++from construct import Const
++from construct import Int32ul
++from construct import Struct
+ from fatfs_utils.exceptions import WLNotInitialized
+-from fatfs_utils.utils import (FULL_BYTE, UINT32_MAX, FATDefaults, crc32, generate_4bytes_random,
+-                               get_args_for_partition_generator)
++from fatfs_utils.utils import crc32
++from fatfs_utils.utils import FATDefaults
++from fatfs_utils.utils import FULL_BYTE
++from fatfs_utils.utils import generate_4bytes_random
++from fatfs_utils.utils import get_args_for_partition_generator
++from fatfs_utils.utils import UINT32_MAX
+ from fatfsgen import FATFS
+
+
+@@ -55,6 +61,7 @@ class WLFATFS:
+     WL_STATE_HEADER_SIZE = 64
+     WL_STATE_COPY_COUNT = 2  # always 2 copies for power failure safety
+     WL_SECTOR_SIZE = 0x1000
++    WL_SAFE_MODE_DUMP_SECTORS = 2
+
+     WL_STATE_T_DATA = Struct(
+         'pos' / Int32ul,
+@@ -99,7 +106,8 @@ class WLFATFS:
+                  temp_buff_size: int = FATDefaults.TEMP_BUFFER_SIZE,
+                  device_id: int = None,
+                  root_entry_count: int = FATDefaults.ROOT_ENTRIES_COUNT,
+-                 media_type: int = FATDefaults.MEDIA_TYPE) -> None:
++                 media_type: int = FATDefaults.MEDIA_TYPE,
++                 wl_mode: Optional[str] = None) -> None:
+         self._initialized = False
+         self._version = version
+         self._temp_buff_size = temp_buff_size
+@@ -107,6 +115,7 @@ class WLFATFS:
+         self.partition_size = size
+         self.total_sectors = self.partition_size // FATDefaults.WL_SECTOR_SIZE
+         self.wl_state_size = WLFATFS.WL_STATE_HEADER_SIZE + WLFATFS.WL_STATE_RECORD_SIZE * self.total_sectors
++        self.wl_mode = wl_mode
+
+         # determine the number of required sectors (roundup to sector size)
+         self.wl_state_sectors = (self.wl_state_size + FATDefaults.WL_SECTOR_SIZE - 1) // FATDefaults.WL_SECTOR_SIZE
+@@ -116,6 +125,9 @@ class WLFATFS:
+
+         wl_sectors = (WLFATFS.WL_DUMMY_SECTORS_COUNT + WLFATFS.WL_CFG_SECTORS_COUNT +
+                       self.wl_state_sectors * WLFATFS.WL_STATE_COPY_COUNT)
++        if self.wl_mode is not None and self.wl_mode == 'safe':
++            wl_sectors += WLFATFS.WL_SAFE_MODE_DUMP_SECTORS
++
+         self.plain_fat_sectors = self.total_sectors - wl_sectors
+         self.plain_fatfs = FATFS(
+             explicit_fat_type=explicit_fat_type,
+@@ -226,7 +238,8 @@ if __name__ == '__main__':
+                        root_entry_count=args.root_entry_count,
+                        explicit_fat_type=args.fat_type,
+                        long_names_enabled=args.long_name_support,
+-                       use_default_datetime=args.use_default_datetime)
++                       use_default_datetime=args.use_default_datetime,
++                       wl_mode=args.wl_mode)
+
+     wl_fatfs.wl_generate(args.input_directory)
+     wl_fatfs.init_wl()
diff --git a/module_config/module_esp32-sdio/patch/patch_list.ini b/module_config/module_esp32-sdio/patch/patch_list.ini
@@ -3,3 +3,7 @@
 [esp_tls_parse_ecc_key.patch]
     path = esp-idf
     note = "Fixed an issue where esp-tls is unable to parse ECC keys"
+
+[fatfs_generation.patch]
+    path = esp-idf
+    note = "Fixed an issue where data read after writing fatfs partition more than 63 times might be incorrect"
diff --git a/module_config/module_esp32_default/patch/fatfs_generation.patch b/module_config/module_esp32_default/patch/fatfs_generation.patch
@@ -0,0 +1,123 @@
+diff --git a/components/fatfs/fatfs_utils/utils.py b/components/fatfs/fatfs_utils/utils.py
+index e7d2e25a7f..ddc68c9225 100644
+--- a/components/fatfs/fatfs_utils/utils.py
++++ b/components/fatfs/fatfs_utils/utils.py
+@@ -1,14 +1,17 @@
+ # SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
+ # SPDX-License-Identifier: Apache-2.0
+-
+ import argparse
+ import binascii
+ import os
+ import uuid
+ from datetime import datetime
+-from typing import List, Optional, Tuple
++from typing import List
++from typing import Optional
++from typing import Tuple
+
+-from construct import BitsInteger, BitStruct, Int16ul
++from construct import BitsInteger
++from construct import BitStruct
++from construct import Int16ul
+
+ FAT12_MAX_CLUSTERS: int = 4085
+ FAT16_MAX_CLUSTERS: int = 65525
+@@ -200,6 +203,11 @@ def get_args_for_partition_generator(desc: str) -> argparse.Namespace:
+                         Type of fat. Select 12 for fat12, 16 for fat16. Don't set, or set to 0 for automatic
+                         calculation using cluster size and partition size.
+                         """)
++    parser.add_argument('--wl_mode',
++                        default=None,
++                        type=str,
++                        choices=['safe', 'perf'],
++                        help='Wear levelling mode to use. Safe or performance. Only for sector size of 512')
+
+     args = parser.parse_args()
+     if args.fat_type == 0:
+@@ -207,6 +215,9 @@ def get_args_for_partition_generator(desc: str) -> argparse.Namespace:
+     args.partition_size = int(str(args.partition_size), 0)
+     if not os.path.isdir(args.input_directory):
+         raise NotADirectoryError(f'The target directory `{args.input_directory}` does not exist!')
++    if args.wl_mode is not None:
++        if args.sector_size != 512:
++            raise ValueError('Wear levelling mode can be set only for sector size 512')
+     return args
+
+
+diff --git a/components/fatfs/wl_fatfsgen.py b/components/fatfs/wl_fatfsgen.py
+index b6c5dd4fe7..8881938f03 100755
+--- a/components/fatfs/wl_fatfsgen.py
++++ b/components/fatfs/wl_fatfsgen.py
+@@ -1,13 +1,19 @@
+ #!/usr/bin/env python
+-# SPDX-FileCopyrightText: 2021-2022 Espressif Systems (Shanghai) CO LTD
++# SPDX-FileCopyrightText: 2021-2024 Espressif Systems (Shanghai) CO LTD
+ # SPDX-License-Identifier: Apache-2.0
++from typing import List
++from typing import Optional
+
+-from typing import List, Optional
+-
+-from construct import Const, Int32ul, Struct
++from construct import Const
++from construct import Int32ul
++from construct import Struct
+ from fatfs_utils.exceptions import WLNotInitialized
+-from fatfs_utils.utils import (FULL_BYTE, UINT32_MAX, FATDefaults, crc32, generate_4bytes_random,
+-                               get_args_for_partition_generator)
++from fatfs_utils.utils import crc32
++from fatfs_utils.utils import FATDefaults
++from fatfs_utils.utils import FULL_BYTE
++from fatfs_utils.utils import generate_4bytes_random
++from fatfs_utils.utils import get_args_for_partition_generator
++from fatfs_utils.utils import UINT32_MAX
+ from fatfsgen import FATFS
+
+
+@@ -55,6 +61,7 @@ class WLFATFS:
+     WL_STATE_HEADER_SIZE = 64
+     WL_STATE_COPY_COUNT = 2  # always 2 copies for power failure safety
+     WL_SECTOR_SIZE = 0x1000
++    WL_SAFE_MODE_DUMP_SECTORS = 2
+
+     WL_STATE_T_DATA = Struct(
+         'pos' / Int32ul,
+@@ -99,7 +106,8 @@ class WLFATFS:
+                  temp_buff_size: int = FATDefaults.TEMP_BUFFER_SIZE,
+                  device_id: int = None,
+                  root_entry_count: int = FATDefaults.ROOT_ENTRIES_COUNT,
+-                 media_type: int = FATDefaults.MEDIA_TYPE) -> None:
++                 media_type: int = FATDefaults.MEDIA_TYPE,
++                 wl_mode: Optional[str] = None) -> None:
+         self._initialized = False
+         self._version = version
+         self._temp_buff_size = temp_buff_size
+@@ -107,6 +115,7 @@ class WLFATFS:
+         self.partition_size = size
+         self.total_sectors = self.partition_size // FATDefaults.WL_SECTOR_SIZE
+         self.wl_state_size = WLFATFS.WL_STATE_HEADER_SIZE + WLFATFS.WL_STATE_RECORD_SIZE * self.total_sectors
++        self.wl_mode = wl_mode
+
+         # determine the number of required sectors (roundup to sector size)
+         self.wl_state_sectors = (self.wl_state_size + FATDefaults.WL_SECTOR_SIZE - 1) // FATDefaults.WL_SECTOR_SIZE
+@@ -116,6 +125,9 @@ class WLFATFS:
+
+         wl_sectors = (WLFATFS.WL_DUMMY_SECTORS_COUNT + WLFATFS.WL_CFG_SECTORS_COUNT +
+                       self.wl_state_sectors * WLFATFS.WL_STATE_COPY_COUNT)
++        if self.wl_mode is not None and self.wl_mode == 'safe':
++            wl_sectors += WLFATFS.WL_SAFE_MODE_DUMP_SECTORS
++
+         self.plain_fat_sectors = self.total_sectors - wl_sectors
+         self.plain_fatfs = FATFS(
+             explicit_fat_type=explicit_fat_type,
+@@ -226,7 +238,8 @@ if __name__ == '__main__':
+                        root_entry_count=args.root_entry_count,
+                        explicit_fat_type=args.fat_type,
+                        long_names_enabled=args.long_name_support,
+-                       use_default_datetime=args.use_default_datetime)
++                       use_default_datetime=args.use_default_datetime,
++                       wl_mode=args.wl_mode)
+
+     wl_fatfs.wl_generate(args.input_directory)
+     wl_fatfs.init_wl()