From 017a8d94d64526b69faa09b1c78bbb07e89bf719 Mon Sep 17 00:00:00 2001 From: Sergey Polyakov Date: Wed, 7 Feb 2024 17:02:04 +0200 Subject: [PATCH 01/10] Minor renamings --- hal/src/gcc/core_hal.cpp | 2 +- hal/src/gcc/device_config.cpp | 2 +- hal/src/gcc/device_config.h | 1 - hal/src/gcc/deviceid_hal.cpp | 2 +- hal/src/gcc/eeprom_hal.cpp | 2 +- hal/src/gcc/{filesystem.cpp => filesystem_util.cpp} | 2 +- hal/src/gcc/{filesystem.h => filesystem_util.h} | 0 hal/src/gcc/ota_flash_hal.cpp | 2 +- 8 files changed, 6 insertions(+), 7 deletions(-) rename hal/src/gcc/{filesystem.cpp => filesystem_util.cpp} (98%) rename hal/src/gcc/{filesystem.h => filesystem_util.h} (100%) diff --git a/hal/src/gcc/core_hal.cpp b/hal/src/gcc/core_hal.cpp index f7c6fc0ec2..341125ce6f 100644 --- a/hal/src/gcc/core_hal.cpp +++ b/hal/src/gcc/core_hal.cpp @@ -31,7 +31,7 @@ #include #include #include -#include "filesystem.h" +#include "filesystem_util.h" #include "service_debug.h" #include "device_config.h" #include "hal_platform.h" diff --git a/hal/src/gcc/device_config.cpp b/hal/src/gcc/device_config.cpp index 875a93834d..8f22527e8c 100644 --- a/hal/src/gcc/device_config.cpp +++ b/hal/src/gcc/device_config.cpp @@ -18,7 +18,7 @@ #include "device_config.h" #include "core_msg.h" -#include "filesystem.h" +#include "filesystem_util.h" #include "ota_flash_hal.h" #include "../../../system/inc/system_info.h" // FIXME diff --git a/hal/src/gcc/device_config.h b/hal/src/gcc/device_config.h index dafed93a95..0016f5c0cb 100644 --- a/hal/src/gcc/device_config.h +++ b/hal/src/gcc/device_config.h @@ -25,7 +25,6 @@ #include #include -#include "filesystem.h" #include "spark_protocol_functions.h" #include "module_info.h" diff --git a/hal/src/gcc/deviceid_hal.cpp b/hal/src/gcc/deviceid_hal.cpp index 3d6282e9c1..ca5544901c 100644 --- a/hal/src/gcc/deviceid_hal.cpp +++ b/hal/src/gcc/deviceid_hal.cpp @@ -26,7 +26,7 @@ #include "deviceid_hal.h" #include "device_config.h" -#include "filesystem.h" +#include "filesystem_util.h" #include #include diff --git a/hal/src/gcc/eeprom_hal.cpp b/hal/src/gcc/eeprom_hal.cpp index fdb66f2cfb..e5b73101a1 100644 --- a/hal/src/gcc/eeprom_hal.cpp +++ b/hal/src/gcc/eeprom_hal.cpp @@ -19,7 +19,7 @@ #include "eeprom_hal.h" #include "eeprom_file.h" -#include "filesystem.h" +#include "filesystem_util.h" #include #include diff --git a/hal/src/gcc/filesystem.cpp b/hal/src/gcc/filesystem_util.cpp similarity index 98% rename from hal/src/gcc/filesystem.cpp rename to hal/src/gcc/filesystem_util.cpp index 5c1df77197..0c3b8ae16d 100644 --- a/hal/src/gcc/filesystem.cpp +++ b/hal/src/gcc/filesystem_util.cpp @@ -7,7 +7,7 @@ #include #include "service_debug.h" -#include "filesystem.h" +#include "filesystem_util.h" namespace fs = std::filesystem; diff --git a/hal/src/gcc/filesystem.h b/hal/src/gcc/filesystem_util.h similarity index 100% rename from hal/src/gcc/filesystem.h rename to hal/src/gcc/filesystem_util.h diff --git a/hal/src/gcc/ota_flash_hal.cpp b/hal/src/gcc/ota_flash_hal.cpp index 8d2f0b994c..c7161b2edf 100644 --- a/hal/src/gcc/ota_flash_hal.cpp +++ b/hal/src/gcc/ota_flash_hal.cpp @@ -10,7 +10,7 @@ #include "device_config.h" #include "service_debug.h" #include "core_hal.h" -#include "filesystem.h" +#include "filesystem_util.h" #include "bytes2hexbuf.h" #include "module_info.h" #include "../../../system/inc/system_info.h" // FIXME From ad4deaac5b49a10234f7f0b306b53ccb99db4b70 Mon Sep 17 00:00:00 2001 From: Sergey Polyakov Date: Thu, 8 Feb 2024 17:09:29 +0200 Subject: [PATCH 02/10] Implement strlcpy for compatibility with glibc --- services/inc/str_compat.h | 36 ++++++++++++++++++++++++++++ services/src/str_compat.cpp | 33 +++++++++++++++++++++++++ system/src/ledger/ledger.cpp | 2 ++ system/src/ledger/ledger_manager.cpp | 1 + 4 files changed, 72 insertions(+) create mode 100644 services/inc/str_compat.h create mode 100644 services/src/str_compat.cpp diff --git a/services/inc/str_compat.h b/services/inc/str_compat.h new file mode 100644 index 0000000000..59bf7d72dd --- /dev/null +++ b/services/inc/str_compat.h @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2024 Particle Industries, Inc. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation, either + * version 3 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see . + */ + +#pragma once + +#include + +#include "hal_platform.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#if PLATFORM_ID == PLATFORM_GCC && defined(__GLIBC__) + +size_t strlcpy(char* dest, const char* src, size_t size); + +#endif // PLATFORM_ID == PLATFORM_GCC && defined(__GLIBC__) + +#ifdef __cplusplus +} // extern "C" +#endif diff --git a/services/src/str_compat.cpp b/services/src/str_compat.cpp new file mode 100644 index 0000000000..9b1a3738ae --- /dev/null +++ b/services/src/str_compat.cpp @@ -0,0 +1,33 @@ +/* + * Copyright (c) 2024 Particle Industries, Inc. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation, either + * version 3 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see . + */ + +#include "str_compat.h" + +#if PLATFORM_ID == PLATFORM_GCC && defined(__GLIBC__) + +size_t strlcpy(char* dest, const char* src, size_t size) { + auto srcLen = strlen(src); + if (srcLen + 1 <= size) { + memcpy(dest, src, srcLen + 1); + } else if (size > 0) { + memcpy(dest, src, size - 1); + dest[size - 1] = '\0'; + } + return srcLen; +} + +#endif // PLATFORM_ID == PLATFORM_GCC && defined(__GLIBC__) diff --git a/system/src/ledger/ledger.cpp b/system/src/ledger/ledger.cpp index b1eca0a047..e068bd3955 100644 --- a/system/src/ledger/ledger.cpp +++ b/system/src/ledger/ledger.cpp @@ -24,6 +24,7 @@ #if HAL_PLATFORM_LEDGER #include +#include #include #include "ledger.h" @@ -33,6 +34,7 @@ #include "file_util.h" #include "time_util.h" #include "endian_util.h" +#include "str_compat.h" #include "scope_guard.h" #include "check.h" diff --git a/system/src/ledger/ledger_manager.cpp b/system/src/ledger/ledger_manager.cpp index 31ca871bc2..706aac3a3e 100644 --- a/system/src/ledger/ledger_manager.cpp +++ b/system/src/ledger/ledger_manager.cpp @@ -43,6 +43,7 @@ #include "file_util.h" #include "time_util.h" #include "endian_util.h" +#include "str_compat.h" #include "scope_guard.h" #include "check.h" From 42dbfbac1933c87693d24a93c0ddcb71fbdfa177 Mon Sep 17 00:00:00 2001 From: Sergey Polyakov Date: Fri, 9 Feb 2024 12:57:03 +0200 Subject: [PATCH 03/10] Build the filesystem API on the GCC platform --- build/platform-id.mk | 2 - hal/shared/filesystem.h | 8 +- hal/src/gcc/include.mk | 1 + hal/src/gcc/{ => littlefs}/filesystem_impl.h | 12 +- hal/src/gcc/littlefs/lfs_config.h | 198 ++++++++++++++++++ hal/src/gcc/littlefs/lfs_utils.cpp | 55 +++++ hal/src/gcc/sources.mk | 1 + ...form_config.h => static_recursive_mutex.h} | 33 ++- platform/MCU/gcc/inc/flash_mal.h | 11 + platform/MCU/gcc/inc/platform_config.h | 40 ++++ platform/MCU/gcc/include.mk | 7 + platform/MCU/gcc/tlm.mk | 3 + 12 files changed, 355 insertions(+), 16 deletions(-) rename hal/src/gcc/{ => littlefs}/filesystem_impl.h (64%) create mode 100644 hal/src/gcc/littlefs/lfs_config.h create mode 100644 hal/src/gcc/littlefs/lfs_utils.cpp rename hal/src/gcc/{platform_config.h => static_recursive_mutex.h} (57%) create mode 100644 platform/MCU/gcc/include.mk create mode 100644 platform/MCU/gcc/tlm.mk diff --git a/build/platform-id.mk b/build/platform-id.mk index f11fc1546c..aabecf4422 100644 --- a/build/platform-id.mk +++ b/build/platform-id.mk @@ -90,8 +90,6 @@ PLATFORM_MCU=gcc PLATFORM_NET=gcc ARCH=gcc PRODUCT_DESC=GCC xcompile -# explicitly exclude platform headers -SPARK_NO_PLATFORM=1 DEFAULT_PRODUCT_ID=3 endif diff --git a/hal/shared/filesystem.h b/hal/shared/filesystem.h index 0cf0d008bd..60f2b7c73b 100644 --- a/hal/shared/filesystem.h +++ b/hal/shared/filesystem.h @@ -22,13 +22,13 @@ #include "filesystem_impl.h" +#include +#include + #ifdef __cplusplus extern "C" { #endif /* __cplusplus */ -#include -#include - typedef enum filesystem_instance_t { FILESYSTEM_INSTANCE_DEFAULT = 0, FILESYSTEM_INSTANCE_ASSET_STORAGE = 1 @@ -67,7 +67,7 @@ int filesystem_unlock(filesystem_t* fs); int filesystem_to_system_error(int error); #ifdef __cplusplus -} +} // extern "C" #define CHECK_FS(expr) \ ({ \ diff --git a/hal/src/gcc/include.mk b/hal/src/gcc/include.mk index 3726d7ba81..3fbbf56b5d 100644 --- a/hal/src/gcc/include.mk +++ b/hal/src/gcc/include.mk @@ -6,6 +6,7 @@ HAL_INCL_NETWORK_PATH = $(TARGET_HAL_PATH)/network HAL_INCL_NETWORK_UTIL_PATH = $(TARGET_HAL_PATH)/network/util INCLUDE_DIRS += $(HAL_SRC_GCC_PATH) +INCLUDE_DIRS += $(HAL_SRC_GCC_PATH)/littlefs INCLUDE_DIRS += $(HAL_INCL_NETWORK_UTIL_PATH) INCLUDE_DIRS += $(HAL_INCL_NETWORK_PATH) diff --git a/hal/src/gcc/filesystem_impl.h b/hal/src/gcc/littlefs/filesystem_impl.h similarity index 64% rename from hal/src/gcc/filesystem_impl.h rename to hal/src/gcc/littlefs/filesystem_impl.h index bdf699d1bf..2072fc35c0 100644 --- a/hal/src/gcc/filesystem_impl.h +++ b/hal/src/gcc/littlefs/filesystem_impl.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2023 Particle Industries, Inc. All rights reserved. + * Copyright (c) 2024 Particle Industries, Inc. All rights reserved. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -16,3 +16,13 @@ */ #pragma once + +#include "platform_config.h" + +#define FILESYSTEM_PROG_SIZE (256) +#define FILESYSTEM_READ_SIZE (256) + +#define FILESYSTEM_BLOCK_SIZE (sFLASH_PAGESIZE) +#define FILESYSTEM_BLOCK_COUNT (sFLASH_FILESYSTEM_PAGE_COUNT) +#define FILESYSTEM_FIRST_BLOCK (sFLASH_FILESYSTEM_FIRST_PAGE) +#define FILESYSTEM_LOOKAHEAD (128) diff --git a/hal/src/gcc/littlefs/lfs_config.h b/hal/src/gcc/littlefs/lfs_config.h new file mode 100644 index 0000000000..c275192adf --- /dev/null +++ b/hal/src/gcc/littlefs/lfs_config.h @@ -0,0 +1,198 @@ +/* + * Copyright (c) 2018 Particle Industries, Inc. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation, either + * version 3 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see . + */ +/* + * lfs utility functions + * + * Copyright (c) 2017 ARM Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef LFS_CONFIG_H +#define LFS_CONFIG_H + +#include +#include +#include + +#define NO_STATIC_ASSERT +#include "module_info.h" + +// #define LFS_NO_DEBUG +// #define LFS_NO_WARN +// #define LFS_NO_ERROR +// #define LFS_NO_ASSERT + +#if MODULE_FUNCTION == MOD_FUNC_BOOTLOADER +#define LFS_NO_MALLOC +#else +#include +#endif /* MODULE_FUNCTION == MOD_FUNC_BOOTLOADER */ + +#if !defined(LFS_NO_DEBUG) || !defined(LFS_NO_WARN) || !defined(LFS_NO_ERROR) +#include "logging.h" +#endif /* !defined(LFS_NO_DEBUG) && !defined(LFS_NO_WARN) && !defined(LFS_NO_ERROR) */ + +#ifndef LFS_NO_DEBUG +#define LFS_DEBUG(fmt, ...) LOG_DEBUG(TRACE, fmt, __VA_ARGS__) +#else +#define LFS_DEBUG(fmt, ...) +#endif /* LFS_NO_DEBUG */ + +#ifndef LFS_NO_WARN +#define LFS_WARN(fmt, ...) LOG_DEBUG(WARN, fmt, __VA_ARGS__) +#else +#define LFS_WARN(fmt, ...) +#endif /* LFS_NO_WARN */ + +#ifndef LFS_NO_ERROR +#define LFS_ERROR(fmt, ...) LOG_DEBUG(ERROR, fmt, __VA_ARGS__) +#else +#define LFS_ERROR(fmt, ...) +#endif /* LFS_NO_ERROR */ + +#ifndef LFS_NO_ASSERT +#include "service_debug.h" +#define LFS_ASSERT(test) SPARK_ASSERT(test) +#else +#define LFS_ASSERT(test) +#endif /* LFS_NO_ASSERT */ + +#ifdef __cplusplus +extern "C" +{ +#endif + +// Builtin functions, these may be replaced by more efficient +// toolchain-specific implementations. LFS_NO_INTRINSICS falls back to a more +// expensive basic C implementation for debugging purposes + +// Min/max functions for unsigned 32-bit numbers +static inline uint32_t lfs_max(uint32_t a, uint32_t b) { + return (a > b) ? a : b; +} + +static inline uint32_t lfs_min(uint32_t a, uint32_t b) { + return (a < b) ? a : b; +} + +// Find the next smallest power of 2 less than or equal to a +static inline uint32_t lfs_npw2(uint32_t a) { +#if !defined(LFS_NO_INTRINSICS) && (defined(__GNUC__) || defined(__CC_ARM)) + return 32 - __builtin_clz(a-1); +#else + uint32_t r = 0; + uint32_t s; + a -= 1; + s = (a > 0xffff) << 4; a >>= s; r |= s; + s = (a > 0xff ) << 3; a >>= s; r |= s; + s = (a > 0xf ) << 2; a >>= s; r |= s; + s = (a > 0x3 ) << 1; a >>= s; r |= s; + return (r | (a >> 1)) + 1; +#endif +} + +// Count the number of trailing binary zeros in a +// lfs_ctz(0) may be undefined +static inline uint32_t lfs_ctz(uint32_t a) { +#if !defined(LFS_NO_INTRINSICS) && defined(__GNUC__) + return __builtin_ctz(a); +#else + return lfs_npw2((a & -a) + 1) - 1; +#endif +} + +// Count the number of binary ones in a +static inline uint32_t lfs_popc(uint32_t a) { +#if !defined(LFS_NO_INTRINSICS) && (defined(__GNUC__) || defined(__CC_ARM)) + return __builtin_popcount(a); +#else + a = a - ((a >> 1) & 0x55555555); + a = (a & 0x33333333) + ((a >> 2) & 0x33333333); + return (((a + (a >> 4)) & 0xf0f0f0f) * 0x1010101) >> 24; +#endif +} + +// Find the sequence comparison of a and b, this is the distance +// between a and b ignoring overflow +static inline int lfs_scmp(uint32_t a, uint32_t b) { + return (int)(unsigned)(a - b); +} + +// Convert between 32-bit little-endian and native order +static inline uint32_t lfs_fromle32(uint32_t a) { +#if !defined(LFS_NO_INTRINSICS) && ( \ + (defined( BYTE_ORDER ) && defined( ORDER_LITTLE_ENDIAN ) && BYTE_ORDER == ORDER_LITTLE_ENDIAN ) || \ + (defined(__BYTE_ORDER ) && defined(__ORDER_LITTLE_ENDIAN ) && __BYTE_ORDER == __ORDER_LITTLE_ENDIAN ) || \ + (defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)) + return a; +#elif !defined(LFS_NO_INTRINSICS) && ( \ + (defined( BYTE_ORDER ) && defined( ORDER_BIG_ENDIAN ) && BYTE_ORDER == ORDER_BIG_ENDIAN ) || \ + (defined(__BYTE_ORDER ) && defined(__ORDER_BIG_ENDIAN ) && __BYTE_ORDER == __ORDER_BIG_ENDIAN ) || \ + (defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)) + return __builtin_bswap32(a); +#else + return (((uint8_t*)&a)[0] << 0) | + (((uint8_t*)&a)[1] << 8) | + (((uint8_t*)&a)[2] << 16) | + (((uint8_t*)&a)[3] << 24); +#endif +} + +// Convert to 32-bit little-endian from native order +static inline uint32_t lfs_tole32(uint32_t a) { + return lfs_fromle32(a); +} + +// Calculate CRC-32 with polynomial = 0x04c11db7 +void lfs_crc(uint32_t *crc, const void *buffer, size_t size); + +// Allocate memory, only used if buffers are not provided to littlefs +static inline void *lfs_malloc(size_t size) { +#ifndef LFS_NO_MALLOC + return malloc(size); +#else + (void)size; + return NULL; +#endif +} + +// Deallocate memory, only used if buffers are not provided to littlefs +static inline void lfs_free(void *p) { +#ifndef LFS_NO_MALLOC + free(p); +#else + (void)p; +#endif +} + + +#ifdef __cplusplus +} /* extern "C" */ +#endif + +#endif /* LFS_CONFIG_H */ \ No newline at end of file diff --git a/hal/src/gcc/littlefs/lfs_utils.cpp b/hal/src/gcc/littlefs/lfs_utils.cpp new file mode 100644 index 0000000000..c51242c63b --- /dev/null +++ b/hal/src/gcc/littlefs/lfs_utils.cpp @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2018 Particle Industries, Inc. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation, either + * version 3 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see . + */ +/* + * lfs util functions + * + * Copyright (c) 2017 ARM Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "lfs_util.h" +#include "hw_config.h" + +// Software CRC implementation with small lookup table +void lfs_crc(uint32_t* __restrict__ crc, const void* buffer, size_t size) { + /* Doesn't work */ + // *crc = Compute_CRC32((const uint8_t*)buffer, size, crc); + static const uint32_t rtable[16] = { + 0x00000000, 0x1db71064, 0x3b6e20c8, 0x26d930ac, + 0x76dc4190, 0x6b6b51f4, 0x4db26158, 0x5005713c, + 0xedb88320, 0xf00f9344, 0xd6d6a3e8, 0xcb61b38c, + 0x9b64c2b0, 0x86d3d2d4, 0xa00ae278, 0xbdbdf21c, + }; + + const uint8_t *data = (const uint8_t*)buffer; + + for (size_t i = 0; i < size; i++) { + *crc = (*crc >> 4) ^ rtable[(*crc ^ (data[i] >> 0)) & 0xf]; + *crc = (*crc >> 4) ^ rtable[(*crc ^ (data[i] >> 4)) & 0xf]; + } +} diff --git a/hal/src/gcc/sources.mk b/hal/src/gcc/sources.mk index 7cbd066659..01a317bb75 100644 --- a/hal/src/gcc/sources.mk +++ b/hal/src/gcc/sources.mk @@ -36,6 +36,7 @@ CSRC += $(call target_files,$(overridedir)/,*.c) CPPSRC += $(call target_files,$(overridedir)/,*.cpp) CPPSRC += $(call target_files,$(HAL_MODULE_PATH)/network/util/,*.cpp) +CPPSRC += $(HAL_MODULE_PATH)/shared/filesystem.cpp # ASM source files included in this build. ASRC += diff --git a/hal/src/gcc/platform_config.h b/hal/src/gcc/static_recursive_mutex.h similarity index 57% rename from hal/src/gcc/platform_config.h rename to hal/src/gcc/static_recursive_mutex.h index 4a4e2dc7de..1e0288ea28 100644 --- a/hal/src/gcc/platform_config.h +++ b/hal/src/gcc/static_recursive_mutex.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2020 Particle Industries, Inc. All rights reserved. + * Copyright (c) 2024 Particle Industries, Inc. All rights reserved. * * This library is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public @@ -15,13 +15,28 @@ * License along with this library; if not, see . */ -#ifndef __PLATFORM_CONFIG_H -#define __PLATFORM_CONFIG_H +#pragma once -// Need to define some LED constants that are normally provided by platform_config.h, -// used by sparking_wiring_rgb.cpp -#define PARTICLE_LED_RED PARTICLE_LED2 -#define PARTICLE_LED_GREEN PARTICLE_LED3 -#define PARTICLE_LED_BLUE PARTICLE_LED4 +#include +#include -#endif /* __PLATFORM_CONFIG_H */ +class StaticRecursiveMutex { +public: + StaticRecursiveMutex() = default; + + bool lock(unsigned timeout = 0) { + if (timeout > 0) { + return mutex_.try_lock_for(std::chrono::milliseconds(timeout)); + } + mutex_.lock(); + return true; + } + + bool unlock() { + mutex_.unlock(); + return true; + } + +private: + std::recursive_timed_mutex mutex_; +}; diff --git a/platform/MCU/gcc/inc/flash_mal.h b/platform/MCU/gcc/inc/flash_mal.h index bdf699d1bf..32f5030162 100644 --- a/platform/MCU/gcc/inc/flash_mal.h +++ b/platform/MCU/gcc/inc/flash_mal.h @@ -16,3 +16,14 @@ */ #pragma once + +#include "platform_config.h" + +#ifdef USE_SERIAL_FLASH + +#define EXTERNAL_FLASH_SIZE (sFLASH_PAGESIZE * sFLASH_PAGECOUNT) + +#define EXTERNAL_FLASH_ASSET_STORAGE_FIRST_PAGE (sFLASH_ASSET_STORAGE_FIRST_PAGE) +#define EXTERNAL_FLASH_ASSET_STORAGE_PAGE_COUNT (sFLASH_ASSET_STORAGE_PAGE_COUNT) + +#endif // defined(USE_SERIAL_FLASH) diff --git a/platform/MCU/gcc/inc/platform_config.h b/platform/MCU/gcc/inc/platform_config.h index e69de29bb2..a97809c601 100644 --- a/platform/MCU/gcc/inc/platform_config.h +++ b/platform/MCU/gcc/inc/platform_config.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2024 Particle Industries, Inc. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation, either + * version 3 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see . + */ + +#ifndef __PLATFORM_CONFIG_H +#define __PLATFORM_CONFIG_H + +// Need to define some LED constants that are normally provided by platform_config.h, +// used by sparking_wiring_rgb.cpp +#define PARTICLE_LED_RED PARTICLE_LED2 +#define PARTICLE_LED_GREEN PARTICLE_LED3 +#define PARTICLE_LED_BLUE PARTICLE_LED4 + +#define USE_SERIAL_FLASH + +#ifdef USE_SERIAL_FLASH + +#define sFLASH_PAGESIZE 0x1000 /* 4096 bytes sector size that needs to be erased */ +#define sFLASH_PAGECOUNT 2048 /* 8MByte storage */ +#define sFLASH_FILESYSTEM_PAGE_COUNT 512 /* 2MB */ +#define sFLASH_FILESYSTEM_FIRST_PAGE 0 +#define sFLASH_ASSET_STORAGE_PAGE_COUNT 512 /* 2MB */ +#define sFLASH_ASSET_STORAGE_FIRST_PAGE (sFLASH_FILESYSTEM_FIRST_PAGE + sFLASH_FILESYSTEM_PAGE_COUNT) + +#endif // defined(USE_SERIAL_FLASH) + +#endif /* __PLATFORM_CONFIG_H */ diff --git a/platform/MCU/gcc/include.mk b/platform/MCU/gcc/include.mk new file mode 100644 index 0000000000..0ab6f659ed --- /dev/null +++ b/platform/MCU/gcc/include.mk @@ -0,0 +1,7 @@ +PLATFORM_DEPS = third_party/littlefs +PLATFORM_DEPS_INCLUDE_SCRIPTS =$(foreach module,$(PLATFORM_DEPS),$(PROJECT_ROOT)/$(module)/import.mk) +include $(PLATFORM_DEPS_INCLUDE_SCRIPTS) + +PLATFORM_LIB_DEP += $(LITTLEFS_LIB_DEP) +LIBS += $(notdir $(PLATFORM_DEPS)) +LIB_DIRS += $(LITTLEFS_LIB_DIR) diff --git a/platform/MCU/gcc/tlm.mk b/platform/MCU/gcc/tlm.mk new file mode 100644 index 0000000000..e5bce30cda --- /dev/null +++ b/platform/MCU/gcc/tlm.mk @@ -0,0 +1,3 @@ +# Inject dependencies +DEPENDENCIES += third_party/littlefs +MAKE_DEPENDENCIES += third_party/littlefs From 7bc5d870137245bb16befd1ff3ae024ad91f9e20 Mon Sep 17 00:00:00 2001 From: Sergey Polyakov Date: Fri, 9 Feb 2024 16:50:51 +0200 Subject: [PATCH 04/10] Initial implementation of exflash_hal for the GCC platform --- hal/src/gcc/device_config.cpp | 3 + hal/src/gcc/device_config.h | 2 + hal/src/gcc/exflash_hal.cpp | 258 ++++++++++++++++++++++++++++++++++ 3 files changed, 263 insertions(+) create mode 100644 hal/src/gcc/exflash_hal.cpp diff --git a/hal/src/gcc/device_config.cpp b/hal/src/gcc/device_config.cpp index 8f22527e8c..e9bcb5413e 100644 --- a/hal/src/gcc/device_config.cpp +++ b/hal/src/gcc/device_config.cpp @@ -108,6 +108,7 @@ class ConfigParser ("product_version", po::value(&config.product_version)->default_value(0xffff), "the product version") ("describe", po::value(&config.describe), "the filename containing the device description") ("protocol,p", po::value(&config.protocol)->default_value(PROTOCOL_NONE), "the cloud communication protocol to use") + ("flash_file", po::value(&config.ext_flash_file), "the filename to use to store the contents of the external flash") ; command_line_options.add(program_options).add(device_options); @@ -369,5 +370,7 @@ void DeviceConfig::read(Configuration& config) } } + this->ext_flash_file = config.ext_flash_file; + setLoggerLevel((LoggerOutputLevel)(NO_LOG_LEVEL - config.log_level)); } diff --git a/hal/src/gcc/device_config.h b/hal/src/gcc/device_config.h index 0016f5c0cb..16621b17ff 100644 --- a/hal/src/gcc/device_config.h +++ b/hal/src/gcc/device_config.h @@ -255,6 +255,7 @@ struct Configuration std::string device_key; std::string server_key; std::string describe; + std::string ext_flash_file; uint16_t log_level; ProtocolFactory protocol; uint16_t platform_id; @@ -268,6 +269,7 @@ struct DeviceConfig { std::vector argv; particle::config::Describe describe; + std::string ext_flash_file; uint8_t device_id[12]; uint8_t device_key[1024]; uint8_t server_key[1024]; diff --git a/hal/src/gcc/exflash_hal.cpp b/hal/src/gcc/exflash_hal.cpp new file mode 100644 index 0000000000..b0fa2175cd --- /dev/null +++ b/hal/src/gcc/exflash_hal.cpp @@ -0,0 +1,258 @@ +/* + * Copyright (c) 2024 Particle Industries, Inc. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation, either + * version 3 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see . + */ + +#include +#include +#include +#include +#include +#include + +#include "device_config.h" +#include "filesystem_util.h" + +#include "exflash_hal.h" +#include "flash_mal.h" + +#include "logging.h" +#include "system_error.h" + +using namespace particle; + +namespace fs = std::filesystem; + +namespace { + +class ExternalFlash { +public: + void read(uintptr_t addr, uint8_t* data, size_t size) { + if (!size) { + return; + } + if (addr + size > EXTERNAL_FLASH_SIZE) { + throw std::runtime_error("Invalid address"); + } + std::lock_guard lock(mutex_); + auto strm = openFile(); + strm.seekg(0, std::ios::end); + size_t fileSize = strm.tellg(); + std::unique_ptr buf(new char[size]); + std::memset(buf.get(), 0xff, size); + if (addr < fileSize) { + strm.seekg(addr); + size_t n = std::min(size, fileSize - addr); + strm.read(buf.get(), n); + } + std::memcpy(data, buf.get(), size); + } + + void write(uintptr_t addr, const uint8_t* data, size_t size) { + if (!size) { + return; + } + if (addr + size > EXTERNAL_FLASH_SIZE) { + throw std::runtime_error("Invalid address"); + } + std::lock_guard lock(mutex_); + auto strm = openFile(); + strm.seekg(0, std::ios::end); + size_t fileSize = strm.tellg(); + if (addr > fileSize) { + // Fill the file with 0xff up to the destination offset + strm.seekp(fileSize); + fillBytes(strm, addr - fileSize); + fileSize = addr; + } + // Read the current contents of the affected region + std::unique_ptr buf(new uint8_t[size]); + std::memset(buf.get(), 0xff, size); + strm.seekg(addr); + size_t n = std::min(size, fileSize - addr); + strm.read((char*)buf.get(), n); + // Mimic the flash memory behavior + for (size_t i = 0; i < size; ++i) { + buf[i] &= data[i]; + } + // Write the resulting data + strm.seekp(addr); + strm.write((const char*)buf.get(), size); + } + + void erase(uintptr_t addr, size_t blockCount, size_t blockSize) { + if (!blockCount || !blockSize) { + return; + } + addr = addr / blockSize * blockSize; + size_t size = blockSize * blockCount; + if (addr + size > EXTERNAL_FLASH_SIZE) { + throw std::runtime_error("Invalid address"); + } + std::lock_guard lock(mutex_); + auto strm = openFile(); + strm.seekg(0, std::ios::end); + size_t fileSize = strm.tellg(); + // Do not write past the current end of file + if (addr < fileSize) { + strm.seekp(addr); + fillBytes(strm, fileSize - addr); + } + } + + void lock() { + mutex_.lock(); + } + + void unlock() { + mutex_.lock(); + } + + static ExternalFlash* instance() { + static ExternalFlash f; + return &f; + } + +private: + std::string fileName_; + bool usingTempFile_; + + mutable std::recursive_mutex mutex_; + + ExternalFlash() : + usingTempFile_(false) { + if (!deviceConfig.ext_flash_file.empty()) { + fileName_ = fs::absolute(deviceConfig.ext_flash_file); + } else { + fileName_ = temp_file_name("ext_flash_", ".bin"); + usingTempFile_ = true; + // TODO: Delete the file on abnormal program termination + } + } + + ~ExternalFlash() { + if (usingTempFile_) { + deleteFile(); + } + } + + std::fstream openFile() { + std::fstream strm; + strm.exceptions(std::ios::badbit | std::ios::failbit); + auto flags = std::ios::in | std::ios::out | std::ios::binary; + if (!fs::exists(fileName_)) { + flags |= std::ios::trunc; + } + strm.open(fileName_, flags); + return strm; + } + + void deleteFile() { + std::error_code ec; + fs::remove(fileName_, ec); + if (ec) { + LOG(ERROR, "Failed to remove file: %s", fileName_.c_str()); + } + } + + static void fillBytes(std::fstream& strm, size_t count) { + char buf[4096]; + std::memset(buf, 0xff, sizeof(buf)); + size_t offs = 0; + while (offs < count) { + size_t n = std::min(count - offs, sizeof(buf)); + strm.write(buf, n); + offs += n; + } + } +}; + +} // namespace + +int hal_exflash_init(void) { + return 0; +} + +int hal_exflash_uninit(void) { + return 0; +} + +int hal_exflash_write(uintptr_t addr, const uint8_t* data_buf, size_t data_size) { + try { + ExternalFlash::instance()->write(addr, data_buf, data_size); + return 0; + } catch (const std::exception& e) { + LOG(ERROR, "hal_exflash_write() failed: %s", e.what()); + return SYSTEM_ERROR_IO; + } +} + +int hal_exflash_erase_sector(uintptr_t addr, size_t num_sectors) { + try { + ExternalFlash::instance()->erase(addr, num_sectors, sFLASH_PAGESIZE); + return 0; + } catch (const std::exception& e) { + LOG(ERROR, "hal_exflash_erase_sector() failed: %s", e.what()); + return SYSTEM_ERROR_IO; + } +} + +int hal_exflash_erase_block(uintptr_t addr, size_t num_blocks) { + return SYSTEM_ERROR_NOT_SUPPORTED; +} + +int hal_exflash_read(uintptr_t addr, uint8_t* data_buf, size_t data_size) { + try { + ExternalFlash::instance()->read(addr, data_buf, data_size); + return 0; + } catch (const std::exception& e) { + LOG(ERROR, "hal_exflash_read() failed: %s", e.what()); + return SYSTEM_ERROR_IO; + } +} + +int hal_exflash_copy_sector(uintptr_t src_addr, size_t dest_addr, size_t data_size) { + return SYSTEM_ERROR_NOT_SUPPORTED; +} + +int hal_exflash_lock(void) { + ExternalFlash::instance()->lock(); + return 0; +} + +int hal_exflash_unlock(void) { + ExternalFlash::instance()->unlock(); + return 0; +} + +int hal_exflash_read_special(hal_exflash_special_sector_t sp, uintptr_t addr, uint8_t* data_buf, size_t data_size) { + return SYSTEM_ERROR_NOT_SUPPORTED; +} + +int hal_exflash_write_special(hal_exflash_special_sector_t sp, uintptr_t addr, const uint8_t* data_buf, size_t data_size) { + return SYSTEM_ERROR_NOT_SUPPORTED; +} + +int hal_exflash_erase_special(hal_exflash_special_sector_t sp, uintptr_t addr, size_t size) { + return SYSTEM_ERROR_NOT_SUPPORTED; +} + +int hal_exflash_special_command(hal_exflash_special_sector_t sp, hal_exflash_command_t cmd, const uint8_t* data, uint8_t* result, size_t size) { + return SYSTEM_ERROR_NOT_SUPPORTED; +} + +int hal_exflash_sleep(bool sleep, void* reserved) { + return SYSTEM_ERROR_NOT_SUPPORTED; +} From 486cd5ed829e4382a9463aa9fe35bb4452a31d24 Mon Sep 17 00:00:00 2001 From: Sergey Polyakov Date: Mon, 12 Feb 2024 12:54:29 +0200 Subject: [PATCH 05/10] Use a sparse buffer to store the contents of the external flash --- hal/src/gcc/device_config.cpp | 6 +- hal/src/gcc/device_config.h | 4 +- hal/src/gcc/exflash_hal.cpp | 194 +++++++++++++----------------- hal/src/gcc/hal_platform_config.h | 2 +- hal/src/gcc/sparse_buffer.h | 128 ++++++++++++++++++++ services/src/file_util.cpp | 1 + 6 files changed, 221 insertions(+), 114 deletions(-) create mode 100644 hal/src/gcc/sparse_buffer.h diff --git a/hal/src/gcc/device_config.cpp b/hal/src/gcc/device_config.cpp index e9bcb5413e..a783b810f5 100644 --- a/hal/src/gcc/device_config.cpp +++ b/hal/src/gcc/device_config.cpp @@ -22,6 +22,7 @@ #include "ota_flash_hal.h" #include "../../../system/inc/system_info.h" // FIXME +#include #include #include #include @@ -36,6 +37,7 @@ using namespace particle; using namespace particle::config; namespace po = boost::program_options; +namespace fs = std::filesystem; DeviceConfig deviceConfig; @@ -108,7 +110,7 @@ class ConfigParser ("product_version", po::value(&config.product_version)->default_value(0xffff), "the product version") ("describe", po::value(&config.describe), "the filename containing the device description") ("protocol,p", po::value(&config.protocol)->default_value(PROTOCOL_NONE), "the cloud communication protocol to use") - ("flash_file", po::value(&config.ext_flash_file), "the filename to use to store the contents of the external flash") + ("flash_file", po::value(&config.flash_file), "the filename to use to store the contents of the external flash") ; command_line_options.add(program_options).add(device_options); @@ -370,7 +372,7 @@ void DeviceConfig::read(Configuration& config) } } - this->ext_flash_file = config.ext_flash_file; + this->flash_file = fs::absolute(config.flash_file); setLoggerLevel((LoggerOutputLevel)(NO_LOG_LEVEL - config.log_level)); } diff --git a/hal/src/gcc/device_config.h b/hal/src/gcc/device_config.h index 16621b17ff..fc60a598c5 100644 --- a/hal/src/gcc/device_config.h +++ b/hal/src/gcc/device_config.h @@ -255,7 +255,7 @@ struct Configuration std::string device_key; std::string server_key; std::string describe; - std::string ext_flash_file; + std::string flash_file; uint16_t log_level; ProtocolFactory protocol; uint16_t platform_id; @@ -269,7 +269,7 @@ struct DeviceConfig { std::vector argv; particle::config::Describe describe; - std::string ext_flash_file; + std::string flash_file; uint8_t device_id[12]; uint8_t device_key[1024]; uint8_t server_key[1024]; diff --git a/hal/src/gcc/exflash_hal.cpp b/hal/src/gcc/exflash_hal.cpp index b0fa2175cd..d52f30add1 100644 --- a/hal/src/gcc/exflash_hal.cpp +++ b/hal/src/gcc/exflash_hal.cpp @@ -23,6 +23,7 @@ #include #include "device_config.h" +#include "sparse_buffer.h" #include "filesystem_util.h" #include "exflash_hal.h" @@ -47,17 +48,8 @@ class ExternalFlash { throw std::runtime_error("Invalid address"); } std::lock_guard lock(mutex_); - auto strm = openFile(); - strm.seekg(0, std::ios::end); - size_t fileSize = strm.tellg(); - std::unique_ptr buf(new char[size]); - std::memset(buf.get(), 0xff, size); - if (addr < fileSize) { - strm.seekg(addr); - size_t n = std::min(size, fileSize - addr); - strm.read(buf.get(), n); - } - std::memcpy(data, buf.get(), size); + auto s = buf_.read(addr, size); + std::memcpy(data, s.data(), size); } void write(uintptr_t addr, const uint8_t* data, size_t size) { @@ -68,28 +60,15 @@ class ExternalFlash { throw std::runtime_error("Invalid address"); } std::lock_guard lock(mutex_); - auto strm = openFile(); - strm.seekg(0, std::ios::end); - size_t fileSize = strm.tellg(); - if (addr > fileSize) { - // Fill the file with 0xff up to the destination offset - strm.seekp(fileSize); - fillBytes(strm, addr - fileSize); - fileSize = addr; - } - // Read the current contents of the affected region - std::unique_ptr buf(new uint8_t[size]); - std::memset(buf.get(), 0xff, size); - strm.seekg(addr); - size_t n = std::min(size, fileSize - addr); - strm.read((char*)buf.get(), n); - // Mimic the flash memory behavior + // Read the contents of the affected region + std::string s = buf_.read(addr, size); + uint8_t* d = (uint8_t*)s.data(); for (size_t i = 0; i < size; ++i) { - buf[i] &= data[i]; + d[i] &= data[i]; // Mimic the flash memory behavior } - // Write the resulting data - strm.seekp(addr); - strm.write((const char*)buf.get(), size); + // Write the changes + buf_.write(addr, s); + saveBuffer(); } void erase(uintptr_t addr, size_t blockCount, size_t blockSize) { @@ -102,14 +81,8 @@ class ExternalFlash { throw std::runtime_error("Invalid address"); } std::lock_guard lock(mutex_); - auto strm = openFile(); - strm.seekg(0, std::ios::end); - size_t fileSize = strm.tellg(); - // Do not write past the current end of file - if (addr < fileSize) { - strm.seekp(addr); - fillBytes(strm, fileSize - addr); - } + buf_.erase(addr, size); + saveBuffer(); } void lock() { @@ -126,56 +99,87 @@ class ExternalFlash { } private: - std::string fileName_; - bool usingTempFile_; + SparseBuffer buf_; + std::string tempFile_; + std::string persistFile_; mutable std::recursive_mutex mutex_; ExternalFlash() : - usingTempFile_(false) { - if (!deviceConfig.ext_flash_file.empty()) { - fileName_ = fs::absolute(deviceConfig.ext_flash_file); - } else { - fileName_ = temp_file_name("ext_flash_", ".bin"); - usingTempFile_ = true; - // TODO: Delete the file on abnormal program termination + buf_(0xff /* fill */) { + if (!deviceConfig.flash_file.empty()) { + persistFile_ = deviceConfig.flash_file; + if (fs::exists(persistFile_)) { + loadBuffer(buf_, persistFile_); + } + tempFile_ = temp_file_name("flash_", ".bin"); + } + } + + void saveBuffer() { + if (!persistFile_.empty()) { + saveBuffer(buf_, tempFile_); + fs::rename(tempFile_, persistFile_); } } - ~ExternalFlash() { - if (usingTempFile_) { - deleteFile(); + static void loadBuffer(SparseBuffer& buf, const std::string& file) { + std::ifstream f; + f.exceptions(std::ios::badbit | std::ios::failbit); + f.open(file, std::ios::binary); + size_t maxOffs = 0; + while (!f.eof()) { + size_t offs = readVarint(f) + maxOffs; + size_t size = readVarint(f); + std::string s; + s.resize(size); + f.read(s.data(), size); + buf.write(offs, s); + maxOffs = offs + size; } } - std::fstream openFile() { - std::fstream strm; - strm.exceptions(std::ios::badbit | std::ios::failbit); - auto flags = std::ios::in | std::ios::out | std::ios::binary; - if (!fs::exists(fileName_)) { - flags |= std::ios::trunc; + static void saveBuffer(const SparseBuffer& buf, const std::string& file) { + std::ofstream f; + f.exceptions(std::ios::badbit | std::ios::failbit); + f.open(file, std::ios::binary | std::ios::trunc); + size_t maxOffs = 0; + auto& seg = buf.segments(); + for (auto it = seg.begin(); it != seg.end(); ++it) { + writeVarint(f, it->first - maxOffs); // Use delta encoding + writeVarint(f, it->second.size()); + f.write(it->second.data(), it->second.size()); + maxOffs = it->first + it->second.size(); } - strm.open(fileName_, flags); - return strm; + f.close(); } - void deleteFile() { - std::error_code ec; - fs::remove(fileName_, ec); - if (ec) { - LOG(ERROR, "Failed to remove file: %s", fileName_.c_str()); + static uint32_t readVarint(std::ifstream& f) { + char buf[maxUnsignedVarintSize()]; + size_t n = 0; + uint8_t b = 0; + do { + f.read((char*)&b, 1); + if (n >= sizeof(buf)) { + throw std::runtime_error("Decoding error"); + } + buf[n++] = b; + } while (b & 0x80); + uint32_t v = 0; + int r = decodeUnsignedVarint(buf, n, &v); + if (r != (int)n) { + throw std::runtime_error("Decoding error"); } + return v; } - static void fillBytes(std::fstream& strm, size_t count) { - char buf[4096]; - std::memset(buf, 0xff, sizeof(buf)); - size_t offs = 0; - while (offs < count) { - size_t n = std::min(count - offs, sizeof(buf)); - strm.write(buf, n); - offs += n; + static void writeVarint(std::ofstream& f, uint32_t val) { + char buf[maxUnsignedVarintSize()]; + int r = encodeUnsignedVarint(buf, sizeof(buf), val); + if (r <= 0 || r > (int)sizeof(buf)) { + throw std::runtime_error("Encoding error"); } + f.write(buf, r); } }; @@ -189,44 +193,36 @@ int hal_exflash_uninit(void) { return 0; } -int hal_exflash_write(uintptr_t addr, const uint8_t* data_buf, size_t data_size) { +int hal_exflash_read(uintptr_t addr, uint8_t* data_buf, size_t data_size) { try { - ExternalFlash::instance()->write(addr, data_buf, data_size); + ExternalFlash::instance()->read(addr, data_buf, data_size); return 0; } catch (const std::exception& e) { - LOG(ERROR, "hal_exflash_write() failed: %s", e.what()); + LOG(ERROR, "hal_exflash_read() failed: %s", e.what()); return SYSTEM_ERROR_IO; } } -int hal_exflash_erase_sector(uintptr_t addr, size_t num_sectors) { +int hal_exflash_write(uintptr_t addr, const uint8_t* data_buf, size_t data_size) { try { - ExternalFlash::instance()->erase(addr, num_sectors, sFLASH_PAGESIZE); + ExternalFlash::instance()->write(addr, data_buf, data_size); return 0; } catch (const std::exception& e) { - LOG(ERROR, "hal_exflash_erase_sector() failed: %s", e.what()); + LOG(ERROR, "hal_exflash_write() failed: %s", e.what()); return SYSTEM_ERROR_IO; } } -int hal_exflash_erase_block(uintptr_t addr, size_t num_blocks) { - return SYSTEM_ERROR_NOT_SUPPORTED; -} - -int hal_exflash_read(uintptr_t addr, uint8_t* data_buf, size_t data_size) { +int hal_exflash_erase_sector(uintptr_t addr, size_t num_sectors) { try { - ExternalFlash::instance()->read(addr, data_buf, data_size); + ExternalFlash::instance()->erase(addr, num_sectors, sFLASH_PAGESIZE); return 0; } catch (const std::exception& e) { - LOG(ERROR, "hal_exflash_read() failed: %s", e.what()); + LOG(ERROR, "hal_exflash_erase_sector() failed: %s", e.what()); return SYSTEM_ERROR_IO; } } -int hal_exflash_copy_sector(uintptr_t src_addr, size_t dest_addr, size_t data_size) { - return SYSTEM_ERROR_NOT_SUPPORTED; -} - int hal_exflash_lock(void) { ExternalFlash::instance()->lock(); return 0; @@ -236,23 +232,3 @@ int hal_exflash_unlock(void) { ExternalFlash::instance()->unlock(); return 0; } - -int hal_exflash_read_special(hal_exflash_special_sector_t sp, uintptr_t addr, uint8_t* data_buf, size_t data_size) { - return SYSTEM_ERROR_NOT_SUPPORTED; -} - -int hal_exflash_write_special(hal_exflash_special_sector_t sp, uintptr_t addr, const uint8_t* data_buf, size_t data_size) { - return SYSTEM_ERROR_NOT_SUPPORTED; -} - -int hal_exflash_erase_special(hal_exflash_special_sector_t sp, uintptr_t addr, size_t size) { - return SYSTEM_ERROR_NOT_SUPPORTED; -} - -int hal_exflash_special_command(hal_exflash_special_sector_t sp, hal_exflash_command_t cmd, const uint8_t* data, uint8_t* result, size_t size) { - return SYSTEM_ERROR_NOT_SUPPORTED; -} - -int hal_exflash_sleep(bool sleep, void* reserved) { - return SYSTEM_ERROR_NOT_SUPPORTED; -} diff --git a/hal/src/gcc/hal_platform_config.h b/hal/src/gcc/hal_platform_config.h index 509a9939f1..2fb1320659 100644 --- a/hal/src/gcc/hal_platform_config.h +++ b/hal/src/gcc/hal_platform_config.h @@ -66,4 +66,4 @@ #define HAL_PLATFORM_FREERTOS (0) -#define HAL_PLATFORM_LEDGER (0) +#define HAL_PLATFORM_FILESYSTEM (1) diff --git a/hal/src/gcc/sparse_buffer.h b/hal/src/gcc/sparse_buffer.h new file mode 100644 index 0000000000..eea82d5ea6 --- /dev/null +++ b/hal/src/gcc/sparse_buffer.h @@ -0,0 +1,128 @@ +#pragma once + +#include +#include +#include + +namespace particle { + +// Infinite buffer for sparse data +class SparseBuffer { +public: + typedef std::map Segments; + + explicit SparseBuffer(int fill = 0) : + fill_(fill) { + } + + std::string read(size_t offs, size_t size) { + std::string dest; + dest.reserve(size); + // Check if there's a segment at a lower offset that overlaps with the requested range + auto it = seg_.lower_bound(offs); + if (it != seg_.begin()) { + auto prev = std::prev(it); + if (offs < prev->first + prev->second.size()) { + auto s = std::string_view(prev->second).substr(offs - prev->first); + append(dest, offs, s, offs); + } + } + // Read the data of the subsequent segments that overlap with the requested range + auto end = offs + size; + while (it != seg_.end() && end > it->first) { + if (end < it->first + it->second.size()) { + auto s = std::string_view(it->second).substr(0, end - it->first); + append(dest, offs, s, it->first); + break; + } + append(dest, offs, it->second, it->first); + ++it; + } + // Add trailing filler bytes if necessary + if (dest.size() < size) { + append(dest, offs, std::string_view(), end); + } + return dest; + } + + void write(size_t offs, std::string_view data) { + if (data.empty()) { + return; + } + erase(offs, data.size()); + auto it = seg_.insert({ offs, std::string(data) }).first; + // Check if the new segment can be merged with the adjacent segments + if (it != seg_.begin()) { + auto prev = std::prev(it); + if (prev->first + prev->second.size() == it->first) { + prev->second.append(it->second); + seg_.erase(it); + it = prev; + } + } + auto next = std::next(it); + if (next != seg_.end() && it->first + it->second.size() == next->first) { + it->second.append(next->second); + seg_.erase(next); + } + } + + void erase(size_t offs, size_t size) { + if (!size) { + return; + } + auto end = offs + size; + // Check if there's a segment at a lower offset that overlaps with the requested range + auto it = seg_.lower_bound(offs); + if (it != seg_.begin()) { + auto prev = std::prev(it); + if (offs < prev->first + prev->second.size()) { + auto s = std::move(prev->second); + prev->second = s.substr(0, offs - prev->first); + if (end < prev->first + s.size()) { + // Split the segment into two + seg_.insert({ end, s.substr(end - prev->first) }); + } + } + } + // Update the subsequent segments that overlap with the requested range + while (it != seg_.end() && end > it->first) { + if (end < it->first + it->second.size()) { + auto s = it->second.substr(end - it->first); + seg_.erase(it); + seg_.insert({ end, std::move(s) }); + break; + } + auto next = std::next(it); + seg_.erase(it); + it = next; + } + } + + size_t size() const { + if (seg_.empty()) { + return 0; + } + auto it = --seg_.end(); + return it->first + it->second.size(); + } + + const Segments& segments() const { + return seg_; + } + +private: + std::map seg_; + int fill_; + + void append(std::string& dest, size_t destOffs, std::string_view src, size_t srcOffs) const { + size_t n = dest.size(); + if (srcOffs - destOffs > n) { + n += srcOffs - destOffs; // Add space for filler bytes + } + dest.resize(src.size() + n, fill_); + std::memcpy(dest.data() + n, src.data(), src.size()); + } +}; + +} // namespace particle diff --git a/services/src/file_util.cpp b/services/src/file_util.cpp index 7d7a6c76cf..9cf16b76b2 100644 --- a/services/src/file_util.cpp +++ b/services/src/file_util.cpp @@ -23,6 +23,7 @@ #include "nanopb_misc.h" #include "bytes2hexbuf.h" +#include "str_compat.h" #include "scope_guard.h" #include "check.h" From fd6e74b4cd079646ea9e8811abb3915a32cf6b5c Mon Sep 17 00:00:00 2001 From: Sergey Polyakov Date: Mon, 12 Feb 2024 13:22:54 +0200 Subject: [PATCH 06/10] Minor fixes --- hal/src/gcc/device_config.cpp | 4 ++- hal/src/gcc/exflash_hal.cpp | 52 +++++++++------------------- hal/src/gcc/sparse_buffer.h | 2 +- system/src/ledger/ledger_manager.cpp | 12 ++++--- 4 files changed, 28 insertions(+), 42 deletions(-) diff --git a/hal/src/gcc/device_config.cpp b/hal/src/gcc/device_config.cpp index a783b810f5..5ef6ea4e12 100644 --- a/hal/src/gcc/device_config.cpp +++ b/hal/src/gcc/device_config.cpp @@ -372,7 +372,9 @@ void DeviceConfig::read(Configuration& config) } } - this->flash_file = fs::absolute(config.flash_file); + if (!config.flash_file.empty()) { + this->flash_file = fs::absolute(config.flash_file); + } setLoggerLevel((LoggerOutputLevel)(NO_LOG_LEVEL - config.log_level)); } diff --git a/hal/src/gcc/exflash_hal.cpp b/hal/src/gcc/exflash_hal.cpp index d52f30add1..0890861d3d 100644 --- a/hal/src/gcc/exflash_hal.cpp +++ b/hal/src/gcc/exflash_hal.cpp @@ -24,13 +24,13 @@ #include "device_config.h" #include "sparse_buffer.h" -#include "filesystem_util.h" #include "exflash_hal.h" #include "flash_mal.h" -#include "logging.h" +#include "endian_util.h" #include "system_error.h" +#include "logging.h" using namespace particle; @@ -112,7 +112,8 @@ class ExternalFlash { if (fs::exists(persistFile_)) { loadBuffer(buf_, persistFile_); } - tempFile_ = temp_file_name("flash_", ".bin"); + fs::path p(persistFile_); + tempFile_ = p.parent_path().append('~' + p.filename().string()); } } @@ -127,15 +128,14 @@ class ExternalFlash { std::ifstream f; f.exceptions(std::ios::badbit | std::ios::failbit); f.open(file, std::ios::binary); - size_t maxOffs = 0; - while (!f.eof()) { - size_t offs = readVarint(f) + maxOffs; - size_t size = readVarint(f); + size_t segCount = readUint32(f); + for (size_t i = 0; i < segCount; ++i) { + size_t offs = readUint32(f); + size_t size = readUint32(f); std::string s; s.resize(size); f.read(s.data(), size); buf.write(offs, s); - maxOffs = offs + size; } } @@ -143,43 +143,25 @@ class ExternalFlash { std::ofstream f; f.exceptions(std::ios::badbit | std::ios::failbit); f.open(file, std::ios::binary | std::ios::trunc); - size_t maxOffs = 0; auto& seg = buf.segments(); + writeUint32(f, seg.size()); for (auto it = seg.begin(); it != seg.end(); ++it) { - writeVarint(f, it->first - maxOffs); // Use delta encoding - writeVarint(f, it->second.size()); + writeUint32(f, it->first); + writeUint32(f, it->second.size()); f.write(it->second.data(), it->second.size()); - maxOffs = it->first + it->second.size(); } f.close(); } - static uint32_t readVarint(std::ifstream& f) { - char buf[maxUnsignedVarintSize()]; - size_t n = 0; - uint8_t b = 0; - do { - f.read((char*)&b, 1); - if (n >= sizeof(buf)) { - throw std::runtime_error("Decoding error"); - } - buf[n++] = b; - } while (b & 0x80); + static uint32_t readUint32(std::ifstream& f) { uint32_t v = 0; - int r = decodeUnsignedVarint(buf, n, &v); - if (r != (int)n) { - throw std::runtime_error("Decoding error"); - } - return v; + f.read((char*)&v, sizeof(v)); + return littleEndianToNative(v); } - static void writeVarint(std::ofstream& f, uint32_t val) { - char buf[maxUnsignedVarintSize()]; - int r = encodeUnsignedVarint(buf, sizeof(buf), val); - if (r <= 0 || r > (int)sizeof(buf)) { - throw std::runtime_error("Encoding error"); - } - f.write(buf, r); + static void writeUint32(std::ofstream& f, uint32_t val) { + val = nativeToLittleEndian(val); + f.write((const char*)&val, sizeof(val)); } }; diff --git a/hal/src/gcc/sparse_buffer.h b/hal/src/gcc/sparse_buffer.h index eea82d5ea6..948b721b4c 100644 --- a/hal/src/gcc/sparse_buffer.h +++ b/hal/src/gcc/sparse_buffer.h @@ -51,7 +51,7 @@ class SparseBuffer { } erase(offs, data.size()); auto it = seg_.insert({ offs, std::string(data) }).first; - // Check if the new segment can be merged with the adjacent segments + // Check if the new segment has adjacent segments it can be merged with if (it != seg_.begin()) { auto prev = std::prev(it); if (prev->first + prev->second.size() == it->first) { diff --git a/system/src/ledger/ledger_manager.cpp b/system/src/ledger/ledger_manager.cpp index 706aac3a3e..6d80afd86b 100644 --- a/system/src/ledger/ledger_manager.cpp +++ b/system/src/ledger/ledger_manager.cpp @@ -260,9 +260,12 @@ int LedgerManager::init() { if (!buf) { return SYSTEM_ERROR_NO_MEMORY; } + auto fsInstance = filesystem_get_instance(FILESYSTEM_INSTANCE_DEFAULT, nullptr); + assert(fsInstance); + FsLock fs(fsInstance); + CHECK(filesystem_mount(fsInstance)); // Enumerate local ledgers LedgerSyncContexts contexts; - FsLock fs; lfs_dir_t dir = {}; int r = lfs_dir_open(fs.instance(), &dir, LEDGER_ROOT_DIR); if (r == 0) { @@ -1587,14 +1590,13 @@ LedgerManager* LedgerManager::instance() { // XXX: Lazy initialization is used so that ledger instances can be requested in the global // scope by the application. It's dangerous because the application's global constructors are // called before the system is fully initialized but seems to work in this case - static volatile bool initCalled = false; - if (!initCalled) { + static std::once_flag onceFlag; + std::call_once(onceFlag, []() { int r = mgr.init(); - initCalled = true; if (r < 0) { LOG(ERROR, "Failed to initialize ledger manager: %d", r); } - } + }); return &mgr; } From 12bb6cb10f5c8f1d1564e5ff66cddc6c37b03e5e Mon Sep 17 00:00:00 2001 From: Sergey Polyakov Date: Tue, 13 Feb 2024 14:06:51 +0200 Subject: [PATCH 07/10] Add unit tests; bugfixes --- hal/src/gcc/sparse_buffer.h | 45 ++++- test/unit_tests/hal/CMakeLists.txt | 2 + test/unit_tests/hal/sparse_buffer.cpp | 233 ++++++++++++++++++++++++ test/unit_tests/services/CMakeLists.txt | 2 +- test/unit_tests/wiring/CMakeLists.txt | 1 + wiring/src/spark_wiring_stream.cpp | 2 +- 6 files changed, 274 insertions(+), 11 deletions(-) create mode 100644 test/unit_tests/hal/sparse_buffer.cpp diff --git a/hal/src/gcc/sparse_buffer.h b/hal/src/gcc/sparse_buffer.h index 948b721b4c..e4332bd9df 100644 --- a/hal/src/gcc/sparse_buffer.h +++ b/hal/src/gcc/sparse_buffer.h @@ -1,7 +1,25 @@ +/* + * Copyright (c) 2024 Particle Industries, Inc. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation, either + * version 3 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see . + */ + #pragma once #include #include +#include #include namespace particle { @@ -11,11 +29,11 @@ class SparseBuffer { public: typedef std::map Segments; - explicit SparseBuffer(int fill = 0) : + explicit SparseBuffer(char fill = 0) : fill_(fill) { } - std::string read(size_t offs, size_t size) { + std::string read(size_t offs, size_t size) const { std::string dest; dest.reserve(size); // Check if there's a segment at a lower offset that overlaps with the requested range @@ -23,7 +41,7 @@ class SparseBuffer { if (it != seg_.begin()) { auto prev = std::prev(it); if (offs < prev->first + prev->second.size()) { - auto s = std::string_view(prev->second).substr(offs - prev->first); + auto s = std::string_view(prev->second).substr(offs - prev->first, size); append(dest, offs, s, offs); } } @@ -51,7 +69,7 @@ class SparseBuffer { } erase(offs, data.size()); auto it = seg_.insert({ offs, std::string(data) }).first; - // Check if the new segment has adjacent segments it can be merged with + // Check if the new segment has adjacent segments that it can be merged with if (it != seg_.begin()) { auto prev = std::prev(it); if (prev->first + prev->second.size() == it->first) { @@ -99,6 +117,10 @@ class SparseBuffer { } } + void clear() { + seg_.clear(); + } + size_t size() const { if (seg_.empty()) { return 0; @@ -107,19 +129,24 @@ class SparseBuffer { return it->first + it->second.size(); } + bool isEmpty() const { + return seg_.empty(); + } + const Segments& segments() const { return seg_; } + char fill() const { + return fill_; + } + private: std::map seg_; - int fill_; + char fill_; void append(std::string& dest, size_t destOffs, std::string_view src, size_t srcOffs) const { - size_t n = dest.size(); - if (srcOffs - destOffs > n) { - n += srcOffs - destOffs; // Add space for filler bytes - } + size_t n = srcOffs - destOffs; dest.resize(src.size() + n, fill_); std::memcpy(dest.data() + n, src.data(), src.size()); } diff --git a/test/unit_tests/hal/CMakeLists.txt b/test/unit_tests/hal/CMakeLists.txt index 0b3a2a71fe..a7942159cf 100644 --- a/test/unit_tests/hal/CMakeLists.txt +++ b/test/unit_tests/hal/CMakeLists.txt @@ -3,6 +3,7 @@ set(target_name hal) # Create test executable add_executable( ${target_name} inflate.cpp + sparse_buffer.cpp ${DEVICE_OS_DIR}/hal/shared/inflate.cpp ${DEVICE_OS_DIR}/hal/shared/inflate_impl.cpp ${DEVICE_OS_DIR}/third_party/miniz/miniz/miniz_tinfl.c @@ -16,6 +17,7 @@ target_compile_definitions( ${target_name} # Set include path specific to target target_include_directories( ${target_name} + PRIVATE ${TEST_DIR} PRIVATE ${DEVICE_OS_DIR}/hal/inc PRIVATE ${DEVICE_OS_DIR}/hal/shared PRIVATE ${DEVICE_OS_DIR}/hal/src/nRF52840 diff --git a/test/unit_tests/hal/sparse_buffer.cpp b/test/unit_tests/hal/sparse_buffer.cpp new file mode 100644 index 0000000000..04fe335105 --- /dev/null +++ b/test/unit_tests/hal/sparse_buffer.cpp @@ -0,0 +1,233 @@ +#include + +#include "sparse_buffer.h" + +#include "util/catch.h" + +using namespace particle; + +namespace { + +void checkBuffer(const SparseBuffer& buf, const std::string& data) { + std::map seg; // Expected segments + size_t size = 0; // Expected size + size_t i = 0; + while (i < data.size()) { + if (data[i] != buf.fill()) { + size_t j = i + 1; + while (j < data.size() && data[j] != buf.fill()) { + ++j; + } + seg.insert({ { i, data.substr(i, j - i) } }); + size = j; + i = j; + } else { + ++i; + } + } + CHECK(buf.segments() == seg); + CHECK(buf.size() == size); + CHECK(buf.isEmpty() == !size); + CHECK(buf.read(0, size) == data); +} + +} // namespace + +TEST_CASE("SparseBuffer") { + SparseBuffer buf('.'); + + SECTION("initial state") { + checkBuffer(buf, ""); + } + + SECTION("writing non-adjacent segments") { + SECTION("") { + buf.write(0, "a"); + checkBuffer(buf, "a"); + } + SECTION("") { + buf.write(3, "ab"); + checkBuffer(buf, "...ab"); + } + SECTION("") { + buf.write(1, "ab"); + checkBuffer(buf, ".ab"); + buf.write(6, "cde"); + checkBuffer(buf, ".ab...cde"); + buf.write(4, "f"); + checkBuffer(buf, ".ab.f.cde"); + } + } + + SECTION("writing adjacent segments") { + SECTION("") { + buf.write(0, "a"); + checkBuffer(buf, "a"); + buf.write(1, "b"); + checkBuffer(buf, "ab"); + } + SECTION("") { + buf.write(1, "ab"); + checkBuffer(buf, ".ab"); + buf.write(0, "c"); + checkBuffer(buf, "cab"); + } + SECTION("") { + buf.write(4, "abc"); + checkBuffer(buf, "....abc"); + buf.write(1, "d"); + checkBuffer(buf, ".d..abc"); + buf.write(2, "ef"); + checkBuffer(buf, ".defabc"); + } + } + + SECTION("writing overlapping segments") { + SECTION("") { + buf.write(0, "a"); + checkBuffer(buf, "a"); + buf.write(0, "b"); + checkBuffer(buf, "b"); + } + SECTION("") { + buf.write(1, "abc"); + checkBuffer(buf, ".abc"); + buf.write(0, "de"); + checkBuffer(buf, "debc"); + } + SECTION("") { + buf.write(1, "abc"); + checkBuffer(buf, ".abc"); + buf.write(3, "de"); + checkBuffer(buf, ".abde"); + } + SECTION("") { + buf.write(1, "a"); + checkBuffer(buf, ".a"); + buf.write(4, "bc"); + checkBuffer(buf, ".a..bc"); + buf.write(7, "d"); + checkBuffer(buf, ".a..bc.d"); + buf.write(9, "efg"); + checkBuffer(buf, ".a..bc.d.efg"); + buf.write(3, "hijkl"); + checkBuffer(buf, ".a.hijkl.efg"); + } + } + + SECTION("reading empty buffer") { + CHECK(buf.read(0, 3) == "..."); + } + + SECTION("reading entire buffer") { + buf.write(0, "abc"); + checkBuffer(buf, "abc"); + CHECK(buf.read(0, 3) == "abc"); + } + + SECTION("reading past the last segment") { + buf.write(1, "a"); + checkBuffer(buf, ".a"); + buf.write(3, "bc"); + checkBuffer(buf, ".a.bc"); + CHECK(buf.read(3, 5) == "bc..."); + } + + SECTION("reading part of a segment") { + SECTION("") { + buf.write(1, "abc"); + checkBuffer(buf, ".abc"); + CHECK(buf.read(0, 3) == ".ab"); + } + SECTION("") { + buf.write(1, "abc"); + checkBuffer(buf, ".abc"); + CHECK(buf.read(2, 2) == "bc"); + } + SECTION("") { + buf.write(1, "abcde"); + checkBuffer(buf, ".abcde"); + CHECK(buf.read(2, 2) == "bc"); + } + } + + SECTION("reading multiple segments") { + buf.write(1, "a"); + checkBuffer(buf, ".a"); + buf.write(3, "bcd"); + checkBuffer(buf, ".a.bcd"); + buf.write(8, "e"); + checkBuffer(buf, ".a.bcd..e"); + buf.write(10, "fg"); + checkBuffer(buf, ".a.bcd..e.fg"); + buf.write(13, "h"); + checkBuffer(buf, ".a.bcd..e.fg.h"); + CHECK(buf.read(5, 7) == "d..e.fg"); + } + + SECTION("erasing empty buffer") { + buf.erase(0, 3); + checkBuffer(buf, ""); + } + + SECTION("erasing entire buffer") { + buf.write(0, "abc"); + checkBuffer(buf, "abc"); + buf.erase(0, 3); + checkBuffer(buf, ""); + } + + SECTION("erasing past the last segment") { + buf.write(1, "a"); + checkBuffer(buf, ".a"); + buf.write(3, "bc"); + checkBuffer(buf, ".a.bc"); + buf.erase(3, 5); + checkBuffer(buf, ".a"); + } + + SECTION("erasing part of a segment") { + SECTION("") { + buf.write(1, "abc"); + checkBuffer(buf, ".abc"); + buf.erase(0, 3); + checkBuffer(buf, "...c"); + } + SECTION("") { + buf.write(1, "abc"); + checkBuffer(buf, ".abc"); + buf.erase(2, 2); + checkBuffer(buf, ".a"); + } + SECTION("") { + buf.write(1, "abcde"); + checkBuffer(buf, ".abcde"); + buf.erase(2, 2); + checkBuffer(buf, ".a..de"); + } + } + + SECTION("erasing multiple segments") { + buf.write(1, "a"); + checkBuffer(buf, ".a"); + buf.write(3, "bcd"); + checkBuffer(buf, ".a.bcd"); + buf.write(8, "e"); + checkBuffer(buf, ".a.bcd..e"); + buf.write(10, "fg"); + checkBuffer(buf, ".a.bcd..e.fg"); + buf.write(13, "h"); + checkBuffer(buf, ".a.bcd..e.fg.h"); + buf.erase(5, 7); + checkBuffer(buf, ".a.bc........h"); + } + + SECTION("clearing buffer") { + buf.write(1, "a"); + checkBuffer(buf, ".a"); + buf.write(3, "bc"); + checkBuffer(buf, ".a.bc"); + buf.clear(); + checkBuffer(buf, ""); + } +} diff --git a/test/unit_tests/services/CMakeLists.txt b/test/unit_tests/services/CMakeLists.txt index ac36ce176e..a60f25e689 100644 --- a/test/unit_tests/services/CMakeLists.txt +++ b/test/unit_tests/services/CMakeLists.txt @@ -33,7 +33,6 @@ get_filename_component(CURRENT_TEST_DIRECTORY_FULL "${CMAKE_CURRENT_SOURCE_DIR}" # Set defines specific to target target_compile_definitions( ${target_name} PRIVATE PLATFORM_ID=3 - PRIVATE HAL_PLATFORM_FILESYSTEM=1 PRIVATE FIXTURES_DIRECTORY="${CURRENT_TEST_DIRECTORY_FULL}/fixtures" ) @@ -51,6 +50,7 @@ target_include_directories( ${target_name} PRIVATE ${DEVICE_OS_DIR}/dynalib/inc PRIVATE ${DEVICE_OS_DIR}/hal/shared PRIVATE ${DEVICE_OS_DIR}/hal/src/gcc + PRIVATE ${DEVICE_OS_DIR}/platform/MCU/gcc/inc PRIVATE ${DEVICE_OS_DIR}/system/inc PRIVATE ${DEVICE_OS_DIR}/wiring/inc PRIVATE ${THIRD_PARTY_DIR}/hippomocks diff --git a/test/unit_tests/wiring/CMakeLists.txt b/test/unit_tests/wiring/CMakeLists.txt index d439a865cf..e8bf8b883d 100644 --- a/test/unit_tests/wiring/CMakeLists.txt +++ b/test/unit_tests/wiring/CMakeLists.txt @@ -65,6 +65,7 @@ target_include_directories( ${target_name} PRIVATE ${DEVICE_OS_DIR}/hal/inc/ PRIVATE ${DEVICE_OS_DIR}/hal/src/gcc/ PRIVATE ${DEVICE_OS_DIR}/hal/shared/ + PRIVATE ${DEVICE_OS_DIR}/platform/MCU/gcc/inc PRIVATE ${DEVICE_OS_DIR}/dynalib/inc/ PRIVATE ${DEVICE_OS_DIR}/services/inc/ PRIVATE ${DEVICE_OS_DIR}/system/inc/ diff --git a/wiring/src/spark_wiring_stream.cpp b/wiring/src/spark_wiring_stream.cpp index 677479d919..68c9582137 100644 --- a/wiring/src/spark_wiring_stream.cpp +++ b/wiring/src/spark_wiring_stream.cpp @@ -25,7 +25,7 @@ */ #include "spark_wiring_stream.h" -#include "spark_wiring.h" // for millis()) +#include "spark_wiring_ticks.h" #include #define PARSE_TIMEOUT 1000 // default number of milli-seconds to wait From 7acdb9e38ec7f53b7a03367440ae3f51aabf2482 Mon Sep 17 00:00:00 2001 From: Sergey Polyakov Date: Tue, 13 Feb 2024 17:38:17 +0200 Subject: [PATCH 08/10] Minimal concurrent_hal implementation for GCC --- hal/src/gcc/concurrent_hal.cpp | 269 +++++++++++++++++++++++++++++ hal/src/gcc/exflash_hal.cpp | 16 +- wiring/src/spark_wiring_stream.cpp | 2 +- 3 files changed, 275 insertions(+), 12 deletions(-) create mode 100644 hal/src/gcc/concurrent_hal.cpp diff --git a/hal/src/gcc/concurrent_hal.cpp b/hal/src/gcc/concurrent_hal.cpp new file mode 100644 index 0000000000..250d7a7c92 --- /dev/null +++ b/hal/src/gcc/concurrent_hal.cpp @@ -0,0 +1,269 @@ +/* + * Copyright (c) 2024 Particle Industries, Inc. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation, either + * version 3 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see . + */ + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "concurrent_hal.h" + +#include "system_error.h" +#include "logging.h" + +namespace { + +class Timer; + +class IoService { +public: + void stop() { + work_.reset(); // Break the event loop + if (thread_.joinable()) { + thread_.join(); + } + } + + static IoService* instance() { + static IoService s; + return &s; + } + +protected: + boost::asio::io_context& context() { + return ctx_; + } + +private: + boost::asio::io_context ctx_; + boost::asio::executor_work_guard work_; + std::thread thread_; + + IoService() : + work_(ctx_.get_executor()), + thread_([this]() { this->ctx_.run(); }) { + } + + ~IoService() { + stop(); + } + + friend class Timer; +}; + +class Timer { +public: + typedef void (*Callback)(os_timer_t timer); + + Timer(unsigned period, Callback callback, bool oneShot = true, void* id = nullptr) : + d_(std::make_shared(this, period, callback, oneShot, id)) { + if (!callback) { + throw std::runtime_error("Invalid argument"); + } + } + + Timer(const Timer&) = delete; + + ~Timer() { + stop(); + } + + void start() { + std::lock_guard lock(d_->mutex); + reset(); + startTimer(d_); + d_->started = true; + } + + void stop() { + std::lock_guard lock(d_->mutex); + reset(); + } + + void setPeriod(unsigned period) { + std::lock_guard lock(d_->mutex); + reset(); + d_->period = period; + // This mimics the behavior of FreeRTOS' xTimerChangePeriod(): + // https://www.freertos.org/FreeRTOS-timers-xTimerChangePeriod.html + startTimer(d_); + } + + void setId(void* id) { + std::lock_guard lock(d_->mutex); + d_->timerId = id; + } + + void* id() const { + std::lock_guard lock(d_->mutex); + return d_->timerId; + } + + bool isStarted() { + std::lock_guard lock(d_->mutex); + return d_->started; + } + + Timer& operator=(const Timer&) = delete; + +private: + struct Data { + boost::asio::steady_timer timer; + std::recursive_mutex mutex; + Callback callback; + os_timer_t halInstance; + void* timerId; + unsigned period; + int stateId; + bool oneShot; + bool started; + + Data(os_timer_t halInstance, unsigned period, Callback callback, bool oneShot, void* id) : + timer(IoService::instance()->context()), + callback(callback), + halInstance(halInstance), + timerId(id), + period(period), + stateId(0), + oneShot(oneShot), + started(false) { + } + }; + + std::shared_ptr d_; + + void reset() { + d_->timer.cancel(); + d_->started = false; + // "Invalidate" any references to the timer's internal state + d_->stateId += 1; + } + + static void startTimer(std::shared_ptr d) { + auto t = std::chrono::steady_clock::now() + std::chrono::milliseconds(d->period); + d->timer.expires_at(t); + d->timer.async_wait([d, stateId = d->stateId](const boost::system::error_code& err) { + std::lock_guard lock(d->mutex); + if (err || d->stateId != stateId) { + return; + } + if (d->oneShot) { + d->started = false; + } + assert(d->callback); + d->callback(d->halInstance); + if (d->stateId != stateId) { + return; + } + if (!d->oneShot) { + startTimer(d); + } + }); + } +}; + +} // namespace + +int os_timer_create(os_timer_t* timer, unsigned period, void (*callback)(os_timer_t timer), void* timer_id, bool one_shot, + void* /* reserved */) { + try { + assert(timer); + *timer = new Timer(period, callback, one_shot, timer_id); + return 0; + } catch (const std::exception& e) { + LOG(ERROR, "os_timer_create() failed: %s", e.what()); + return SYSTEM_ERROR_UNKNOWN; + } +} + +int os_timer_destroy(os_timer_t timer, void* /* reserved */) { + auto t = static_cast(timer); + delete t; + return 0; +} + +int os_timer_change(os_timer_t timer, os_timer_change_t change, bool /* fromISR */, unsigned period, unsigned /* block */, + void* /* reserved */) { + try { + auto t = static_cast(timer); + assert(t); + switch (change) { + // xTimerStart() and xTimerReset() seem to behave exactly the same: + // https://www.freertos.org/FreeRTOS-timers-xTimerStart.html + // https://www.freertos.org/FreeRTOS-timers-xTimerReset.html + case OS_TIMER_CHANGE_START: + case OS_TIMER_CHANGE_RESET: { + t->start(); + break; + } + case OS_TIMER_CHANGE_STOP: { + t->stop(); + break; + } + case OS_TIMER_CHANGE_PERIOD: { + t->setPeriod(period); + break; + } + default: + return SYSTEM_ERROR_INVALID_ARGUMENT; + } + return 0; + } catch (const std::exception& e) { + LOG(ERROR, "os_timer_change() failed: %s", e.what()); + return SYSTEM_ERROR_UNKNOWN; + } +} + +int os_timer_set_id(os_timer_t timer, void* timer_id) { + try { + auto t = static_cast(timer); + assert(t); + t->setId(timer_id); + return 0; + } catch (const std::exception& e) { + LOG(ERROR, "os_timer_set_id() failed: %s", e.what()); + return SYSTEM_ERROR_UNKNOWN; + } +} + +int os_timer_get_id(os_timer_t timer, void** timer_id) { + try { + auto t = static_cast(timer); + assert(t && timer_id); + *timer_id = t->id(); + return 0; + } catch (const std::exception& e) { + LOG(ERROR, "os_timer_get_id() failed: %s", e.what()); + return SYSTEM_ERROR_UNKNOWN; + } +} + +int os_timer_is_active(os_timer_t timer, void* /* reserved */) { + try { + auto t = static_cast(timer); + assert(t); + return t->isStarted(); + } catch (const std::exception& e) { + LOG(ERROR, "os_timer_is_active() failed: %s", e.what()); + return SYSTEM_ERROR_UNKNOWN; + } +} diff --git a/hal/src/gcc/exflash_hal.cpp b/hal/src/gcc/exflash_hal.cpp index 0890861d3d..286958cbad 100644 --- a/hal/src/gcc/exflash_hal.cpp +++ b/hal/src/gcc/exflash_hal.cpp @@ -40,10 +40,7 @@ namespace { class ExternalFlash { public: - void read(uintptr_t addr, uint8_t* data, size_t size) { - if (!size) { - return; - } + void read(uintptr_t addr, uint8_t* data, size_t size) const { if (addr + size > EXTERNAL_FLASH_SIZE) { throw std::runtime_error("Invalid address"); } @@ -53,9 +50,6 @@ class ExternalFlash { } void write(uintptr_t addr, const uint8_t* data, size_t size) { - if (!size) { - return; - } if (addr + size > EXTERNAL_FLASH_SIZE) { throw std::runtime_error("Invalid address"); } @@ -68,11 +62,11 @@ class ExternalFlash { } // Write the changes buf_.write(addr, s); - saveBuffer(); + bufferChanged(); } void erase(uintptr_t addr, size_t blockCount, size_t blockSize) { - if (!blockCount || !blockSize) { + if (!blockSize) { return; } addr = addr / blockSize * blockSize; @@ -82,7 +76,7 @@ class ExternalFlash { } std::lock_guard lock(mutex_); buf_.erase(addr, size); - saveBuffer(); + bufferChanged(); } void lock() { @@ -117,7 +111,7 @@ class ExternalFlash { } } - void saveBuffer() { + void bufferChanged() { if (!persistFile_.empty()) { saveBuffer(buf_, tempFile_); fs::rename(tempFile_, persistFile_); diff --git a/wiring/src/spark_wiring_stream.cpp b/wiring/src/spark_wiring_stream.cpp index 68c9582137..7dd2eb2746 100644 --- a/wiring/src/spark_wiring_stream.cpp +++ b/wiring/src/spark_wiring_stream.cpp @@ -25,7 +25,7 @@ */ #include "spark_wiring_stream.h" -#include "spark_wiring_ticks.h" +#include "spark_wiring.h" #include #define PARSE_TIMEOUT 1000 // default number of milli-seconds to wait From e95f6145145f53538eb60c9d96be8a4a8f825c1d Mon Sep 17 00:00:00 2001 From: Sergey Polyakov Date: Tue, 13 Feb 2024 21:44:25 +0200 Subject: [PATCH 09/10] Minor fixes --- hal/shared/filesystem.h | 1 + hal/src/gcc/concurrent_hal.cpp | 38 ++++++++++++++---------------- hal/src/gcc/exflash_hal.cpp | 2 +- hal/src/gcc/littlefs/lfs_config.h | 2 +- system/src/util/system_timer.cpp | 9 +++---- wiring/src/spark_wiring_stream.cpp | 2 +- 6 files changed, 27 insertions(+), 27 deletions(-) diff --git a/hal/shared/filesystem.h b/hal/shared/filesystem.h index 60f2b7c73b..e58b6d4690 100644 --- a/hal/shared/filesystem.h +++ b/hal/shared/filesystem.h @@ -98,6 +98,7 @@ struct FsLock { filesystem_unlock(fs_); } + // TODO: Rename this method to avoid confusion with filesystem_get_instance() lfs_t* instance() const { return &fs_->instance; } diff --git a/hal/src/gcc/concurrent_hal.cpp b/hal/src/gcc/concurrent_hal.cpp index 250d7a7c92..430c6e1e91 100644 --- a/hal/src/gcc/concurrent_hal.cpp +++ b/hal/src/gcc/concurrent_hal.cpp @@ -15,12 +15,9 @@ * License along with this library; if not, see . */ -#include -#include #include #include #include -#include #include #include @@ -32,8 +29,6 @@ namespace { -class Timer; - class IoService { public: void stop() { @@ -43,16 +38,15 @@ class IoService { } } + boost::asio::io_context& context() { + return ctx_; + } + static IoService* instance() { static IoService s; return &s; } -protected: - boost::asio::io_context& context() { - return ctx_; - } - private: boost::asio::io_context ctx_; boost::asio::executor_work_guard work_; @@ -66,8 +60,6 @@ class IoService { ~IoService() { stop(); } - - friend class Timer; }; class Timer { @@ -76,7 +68,9 @@ class Timer { Timer(unsigned period, Callback callback, bool oneShot = true, void* id = nullptr) : d_(std::make_shared(this, period, callback, oneShot, id)) { - if (!callback) { + // Requirement for the period to be greater than 0 mimics the behavior of FreeRTOS' xTimerCreate(): + // https://www.freertos.org/FreeRTOS-timers-xTimerCreate.html + if (!period || !callback) { throw std::runtime_error("Invalid argument"); } } @@ -89,22 +83,26 @@ class Timer { void start() { std::lock_guard lock(d_->mutex); - reset(); + cancel(); startTimer(d_); d_->started = true; } void stop() { std::lock_guard lock(d_->mutex); - reset(); + cancel(); } void setPeriod(unsigned period) { + // Requirement for the period to be greater than 0 mimics the behavior of FreeRTOS' xTimerChangePeriod(): + // https://www.freertos.org/FreeRTOS-timers-xTimerChangePeriod.html + if (!period) { + throw std::runtime_error("Invalid argument"); + } std::lock_guard lock(d_->mutex); - reset(); + cancel(); d_->period = period; - // This mimics the behavior of FreeRTOS' xTimerChangePeriod(): - // https://www.freertos.org/FreeRTOS-timers-xTimerChangePeriod.html + // xTimerChangePeriod() also starts the timer startTimer(d_); } @@ -151,7 +149,7 @@ class Timer { std::shared_ptr d_; - void reset() { + void cancel() { d_->timer.cancel(); d_->started = false; // "Invalidate" any references to the timer's internal state @@ -207,7 +205,7 @@ int os_timer_change(os_timer_t timer, os_timer_change_t change, bool /* fromISR auto t = static_cast(timer); assert(t); switch (change) { - // xTimerStart() and xTimerReset() seem to behave exactly the same: + // xTimerStart() and xTimerReset() seem to behave exactly the same :thinking_face: // https://www.freertos.org/FreeRTOS-timers-xTimerStart.html // https://www.freertos.org/FreeRTOS-timers-xTimerReset.html case OS_TIMER_CHANGE_START: diff --git a/hal/src/gcc/exflash_hal.cpp b/hal/src/gcc/exflash_hal.cpp index 286958cbad..856c06f18f 100644 --- a/hal/src/gcc/exflash_hal.cpp +++ b/hal/src/gcc/exflash_hal.cpp @@ -58,7 +58,7 @@ class ExternalFlash { std::string s = buf_.read(addr, size); uint8_t* d = (uint8_t*)s.data(); for (size_t i = 0; i < size; ++i) { - d[i] &= data[i]; // Mimic the flash memory behavior + d[i] &= data[i]; // Pretend this is flash memory } // Write the changes buf_.write(addr, s); diff --git a/hal/src/gcc/littlefs/lfs_config.h b/hal/src/gcc/littlefs/lfs_config.h index c275192adf..21b3a082f4 100644 --- a/hal/src/gcc/littlefs/lfs_config.h +++ b/hal/src/gcc/littlefs/lfs_config.h @@ -195,4 +195,4 @@ static inline void lfs_free(void *p) { } /* extern "C" */ #endif -#endif /* LFS_CONFIG_H */ \ No newline at end of file +#endif /* LFS_CONFIG_H */ diff --git a/system/src/util/system_timer.cpp b/system/src/util/system_timer.cpp index 7e2cc242fa..0153bbbb19 100644 --- a/system/src/util/system_timer.cpp +++ b/system/src/util/system_timer.cpp @@ -45,16 +45,17 @@ int SystemTimer::start(unsigned timeout) { if (r != 0) { return SYSTEM_ERROR_NO_MEMORY; } + r = os_timer_change(timer_, OS_TIMER_CHANGE_START, false /* fromISR */, 0 /* period */, 0xffffffffu /* block */, nullptr /* reserved */); + if (r != 0) { + return SYSTEM_ERROR_INTERNAL; // Should not happen + } } else { + // Changing the period also starts the timer int r = os_timer_change(timer_, OS_TIMER_CHANGE_PERIOD, false /* fromISR */, timeout, 0xffffffffu /* block */, nullptr /* reserved */); if (r != 0) { return SYSTEM_ERROR_INTERNAL; // Should not happen } } - int r = os_timer_change(timer_, OS_TIMER_CHANGE_START, false /* fromISR */, 0 /* period */, 0xffffffffu /* block */, nullptr /* reserved */); - if (r != 0) { - return SYSTEM_ERROR_INTERNAL; // Should not happen - } } else { // Schedule a call in the system thread SystemISRTaskQueue.enqueue(this); diff --git a/wiring/src/spark_wiring_stream.cpp b/wiring/src/spark_wiring_stream.cpp index 7dd2eb2746..677479d919 100644 --- a/wiring/src/spark_wiring_stream.cpp +++ b/wiring/src/spark_wiring_stream.cpp @@ -25,7 +25,7 @@ */ #include "spark_wiring_stream.h" -#include "spark_wiring.h" +#include "spark_wiring.h" // for millis()) #include #define PARSE_TIMEOUT 1000 // default number of milli-seconds to wait From 290a8ec9e37c1422f2d0873f4b0a4da3c334650f Mon Sep 17 00:00:00 2001 From: Sergey Polyakov Date: Tue, 13 Feb 2024 22:50:03 +0200 Subject: [PATCH 10/10] Increase the size of the mbedTLS message buffer --- crypto/inc/mbedtls_config.h | 4 ++-- hal/src/gcc/include.mk | 1 + hal/src/gcc/mbedtls/mbedtls_config_platform.h | 23 +++++++++++++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) create mode 100644 hal/src/gcc/mbedtls/mbedtls_config_platform.h diff --git a/crypto/inc/mbedtls_config.h b/crypto/inc/mbedtls_config.h index daca55cb66..22a97716a6 100644 --- a/crypto/inc/mbedtls_config.h +++ b/crypto/inc/mbedtls_config.h @@ -27,11 +27,11 @@ #include "platforms.h" /* For new platforms the configuration file resides in hal */ -#if PLATFORM_ID == PLATFORM_GCC || PLATFORM_ID == PLATFORM_NEWHAL +#if PLATFORM_ID == PLATFORM_NEWHAL #include "mbedtls_config_default.h" #else #include "mbedtls_config_platform.h" -#endif /* PLATFORM_ID == PLATFORM_GCC || PLATFORM_ID == PLATFORM_NEWHAL */ +#endif /* PLATFORM_ID == PLATFORM_NEWHAL */ #if defined(MODULAR_FIRMWARE) && MODULAR_FIRMWARE #endif /* defined(MODULAR_FIRMWARE) && MODULAR_FIRMWARE */ diff --git a/hal/src/gcc/include.mk b/hal/src/gcc/include.mk index 3fbbf56b5d..c9905fa7f7 100644 --- a/hal/src/gcc/include.mk +++ b/hal/src/gcc/include.mk @@ -6,6 +6,7 @@ HAL_INCL_NETWORK_PATH = $(TARGET_HAL_PATH)/network HAL_INCL_NETWORK_UTIL_PATH = $(TARGET_HAL_PATH)/network/util INCLUDE_DIRS += $(HAL_SRC_GCC_PATH) +INCLUDE_DIRS += $(HAL_SRC_GCC_PATH)/mbedtls INCLUDE_DIRS += $(HAL_SRC_GCC_PATH)/littlefs INCLUDE_DIRS += $(HAL_INCL_NETWORK_UTIL_PATH) INCLUDE_DIRS += $(HAL_INCL_NETWORK_PATH) diff --git a/hal/src/gcc/mbedtls/mbedtls_config_platform.h b/hal/src/gcc/mbedtls/mbedtls_config_platform.h new file mode 100644 index 0000000000..0c3df891bb --- /dev/null +++ b/hal/src/gcc/mbedtls/mbedtls_config_platform.h @@ -0,0 +1,23 @@ +/* + * Copyright (c) 2024 Particle Industries, Inc. All rights reserved. + * + * This library is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation, either + * version 3 of the License, or (at your option) any later version. + * + * This library is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with this library; if not, see . + */ + +#pragma once + +#include "mbedtls_config_default.h" + +#undef MBEDTLS_SSL_MAX_CONTENT_LEN +#define MBEDTLS_SSL_MAX_CONTENT_LEN 1152