Feature: support JSON payload in MQTT battery provider

this changeset adds support for parsing the MQTT battery provider's SoC and voltage topics' payloads as JSON to extract a numeric value at a configurable path.
hoylabs · Jul 31, 2024 · 1a19f88 · 1a19f88
1 parent accc70d
commit 1a19f88
Show file tree

Hide file tree

Showing 13 changed files with 138 additions and 93 deletions.
diff --git a/include/Configuration.h b/include/Configuration.h
@@ -41,6 +41,7 @@
 #define POWERMETER_MQTT_MAX_VALUES 3
 #define POWERMETER_HTTP_JSON_MAX_VALUES 3
 #define POWERMETER_HTTP_JSON_MAX_PATH_STRLEN 256
+#define BATTERY_JSON_MAX_PATH_STRLEN 128
 
 struct CHANNEL_CONFIG_T {
     uint16_t MaxChannelPower;
@@ -281,7 +282,9 @@ struct CONFIG_T {
         uint8_t JkBmsInterface;
         uint8_t JkBmsPollingInterval;
         char MqttSocTopic[MQTT_MAX_TOPIC_STRLEN + 1];
+        char MqttSocJsonPath[BATTERY_JSON_MAX_PATH_STRLEN + 1];
         char MqttVoltageTopic[MQTT_MAX_TOPIC_STRLEN + 1];
+        char MqttVoltageJsonPath[BATTERY_JSON_MAX_PATH_STRLEN + 1];
     } Battery;
 
     struct {

diff --git a/include/MqttBattery.h b/include/MqttBattery.h
@@ -19,9 +19,10 @@ class MqttBattery : public BatteryProvider {
     String _voltageTopic;
     std::shared_ptr<MqttBatteryStats> _stats = std::make_shared<MqttBatteryStats>();
 
-    std::optional<float> getFloat(std::string const& src, char const* topic);
     void onMqttMessageSoC(espMqttClientTypes::MessageProperties const& properties,
-                          char const* topic, uint8_t const* payload, size_t len, size_t index, size_t total);
+            char const* topic, uint8_t const* payload, size_t len, size_t index, size_t total,
+            char const* jsonPath);
     void onMqttMessageVoltage(espMqttClientTypes::MessageProperties const& properties,
-                              char const* topic, uint8_t const* payload, size_t len, size_t index, size_t total);
+            char const* topic, uint8_t const* payload, size_t len, size_t index, size_t total,
+            char const* jsonPath);
 };
diff --git a/include/Utils.h b/include/Utils.h
@@ -17,4 +17,8 @@ class Utils {
     /* OpenDTU-OnBatter-specific utils go here: */
     template<typename T>
     static std::pair<T, String> getJsonValueByPath(JsonDocument const& root, String const& path);
+
+    template <typename T>
+    static std::optional<T> getNumericValueFromMqttPayload(char const* client,
+            std::string const& src, char const* topic, char const* jsonPath);
 };
diff --git a/src/Configuration.cpp b/src/Configuration.cpp
@@ -257,7 +257,9 @@ bool ConfigurationClass::write()
     battery["jkbms_interface"] = config.Battery.JkBmsInterface;
     battery["jkbms_polling_interval"] = config.Battery.JkBmsPollingInterval;
     battery["mqtt_topic"] = config.Battery.MqttSocTopic;
+    battery["mqtt_json_path"] = config.Battery.MqttSocJsonPath;
     battery["mqtt_voltage_topic"] = config.Battery.MqttVoltageTopic;
+    battery["mqtt_voltage_json_path"] = config.Battery.MqttVoltageJsonPath;
 
     JsonObject huawei = doc["huawei"].to<JsonObject>();
     huawei["enabled"] = config.Huawei.Enabled;
@@ -604,7 +606,9 @@ bool ConfigurationClass::read()
     config.Battery.JkBmsInterface = battery["jkbms_interface"] | BATTERY_JKBMS_INTERFACE;
     config.Battery.JkBmsPollingInterval = battery["jkbms_polling_interval"] | BATTERY_JKBMS_POLLING_INTERVAL;
     strlcpy(config.Battery.MqttSocTopic, battery["mqtt_topic"] | "", sizeof(config.Battery.MqttSocTopic));
+    strlcpy(config.Battery.MqttSocJsonPath, battery["mqtt_json_path"] | "", sizeof(config.Battery.MqttSocJsonPath));
     strlcpy(config.Battery.MqttVoltageTopic, battery["mqtt_voltage_topic"] | "", sizeof(config.Battery.MqttVoltageTopic));
+    strlcpy(config.Battery.MqttVoltageJsonPath, battery["mqtt_voltage_json_path"] | "", sizeof(config.Battery.MqttVoltageJsonPath));
 
     JsonObject huawei = doc["huawei"];
     config.Huawei.Enabled = huawei["enabled"] | HUAWEI_ENABLED;

diff --git a/src/MqttBattery.cpp b/src/MqttBattery.cpp
@@ -4,6 +4,7 @@
 #include "MqttBattery.h"
 #include "MqttSettings.h"
 #include "MessageOutput.h"
+#include "Utils.h"
 
 bool MqttBattery::init(bool verboseLogging)
 {
@@ -17,7 +18,8 @@ bool MqttBattery::init(bool verboseLogging)
                 std::bind(&MqttBattery::onMqttMessageSoC,
                     this, std::placeholders::_1, std::placeholders::_2,
                     std::placeholders::_3, std::placeholders::_4,
-                    std::placeholders::_5, std::placeholders::_6)
+                    std::placeholders::_5, std::placeholders::_6,
+                    config.Battery.MqttSocJsonPath)
                 );
 
         if (_verboseLogging) {
@@ -32,7 +34,8 @@ bool MqttBattery::init(bool verboseLogging)
                 std::bind(&MqttBattery::onMqttMessageVoltage,
                     this, std::placeholders::_1, std::placeholders::_2,
                     std::placeholders::_3, std::placeholders::_4,
-                    std::placeholders::_5, std::placeholders::_6)
+                    std::placeholders::_5, std::placeholders::_6,
+                    config.Battery.MqttVoltageJsonPath)
                 );
 
         if (_verboseLogging) {
@@ -55,25 +58,14 @@ void MqttBattery::deinit()
     }
 }
 
-std::optional<float> MqttBattery::getFloat(std::string const& src, char const* topic) {
-    float res = 0;
-
-    try {
-        res = std::stof(src);
-    }
-    catch(std::invalid_argument const& e) {
-        MessageOutput.printf("MqttBattery: Cannot parse payload '%s' in topic '%s' as float\r\n",
-                src.c_str(), topic);
-        return std::nullopt;
-    }
-
-    return res;
-}
-
 void MqttBattery::onMqttMessageSoC(espMqttClientTypes::MessageProperties const& properties,
-        char const* topic, uint8_t const* payload, size_t len, size_t index, size_t total)
+        char const* topic, uint8_t const* payload, size_t len, size_t index, size_t total,
+        char const* jsonPath)
 {
-    auto soc = getFloat(std::string(reinterpret_cast<const char*>(payload), len), topic);
+    auto soc = Utils::getNumericValueFromMqttPayload<float>("MqttBattery",
+            std::string(reinterpret_cast<const char*>(payload), len), topic,
+            jsonPath);
+
     if (!soc.has_value()) { return; }
 
     if (*soc < 0 || *soc > 100) {
@@ -91,9 +83,13 @@ void MqttBattery::onMqttMessageSoC(espMqttClientTypes::MessageProperties const&
 }
 
 void MqttBattery::onMqttMessageVoltage(espMqttClientTypes::MessageProperties const& properties,
-        char const* topic, uint8_t const* payload, size_t len, size_t index, size_t total)
+        char const* topic, uint8_t const* payload, size_t len, size_t index, size_t total,
+        char const* jsonPath)
 {
-    auto voltage = getFloat(std::string(reinterpret_cast<const char*>(payload), len), topic);
+    auto voltage = Utils::getNumericValueFromMqttPayload<float>("MqttBattery",
+            std::string(reinterpret_cast<const char*>(payload), len), topic,
+            jsonPath);
+
     if (!voltage.has_value()) { return; }
 
     // since this project is revolving around Hoymiles microinverters, which can

diff --git a/src/PowerMeterMqtt.cpp b/src/PowerMeterMqtt.cpp
@@ -38,45 +38,13 @@ void PowerMeterMqtt::onMessage(PowerMeterMqtt::MsgProperties const& properties,
         char const* topic, uint8_t const* payload, size_t len, size_t index,
         size_t total, float* targetVariable, PowerMeterMqttValue const* cfg)
 {
-    std::string value(reinterpret_cast<char const*>(payload), len);
-    std::string logValue = value.substr(0, 32);
-    if (value.length() > logValue.length()) { logValue += "..."; }
+    auto extracted = Utils::getNumericValueFromMqttPayload<float>("PowerMeterMqtt",
+            std::string(reinterpret_cast<const char*>(payload), len), topic,
+            cfg->JsonPath);
 
-    auto log= [topic](char const* format, auto&&... args) -> void {
-        MessageOutput.printf("[PowerMeterMqtt] Topic '%s': ", topic);
-        MessageOutput.printf(format, args...);
-        MessageOutput.println();
-    };
-
-    float newValue = 0;
-
-    if (strlen(cfg->JsonPath) == 0) {
-        try {
-            newValue = std::stof(value);
-        }
-        catch (std::invalid_argument const& e) {
-            return log("cannot parse payload '%s' as float", logValue.c_str());
-        }
-    }
-    else {
-        JsonDocument json;
+    if (!extracted.has_value()) { return; }
 
-        const DeserializationError error = deserializeJson(json, value);
-        if (error) {
-            return log("cannot parse payload '%s' as JSON", logValue.c_str());
-        }
-
-        if (json.overflowed()) {
-            return log("payload too large to process as JSON");
-        }
-
-        auto pathResolutionResult = Utils::getJsonValueByPath<float>(json, cfg->JsonPath);
-        if (!pathResolutionResult.second.isEmpty()) {
-            return log("%s", pathResolutionResult.second.c_str());
-        }
-
-        newValue = pathResolutionResult.first;
-    }
+    float newValue = *extracted;
 
     using Unit_t = PowerMeterMqttValue::Unit;
     switch (cfg->PowerUnit) {
@@ -98,7 +66,8 @@ void PowerMeterMqtt::onMessage(PowerMeterMqtt::MsgProperties const& properties,
     }
 
     if (_verboseLogging) {
-        log("new value: %5.2f, total: %5.2f", newValue, getPowerTotal());
+        MessageOutput.printf("[PowerMeterMqtt] Topic '%s': new value: %5.2f, "
+                "total: %5.2f\r\n", topic, newValue, getPowerTotal());
     }
 
     gotUpdate();

diff --git a/src/Utils.cpp b/src/Utils.cpp
@@ -203,3 +203,47 @@ std::pair<T, String> Utils::getJsonValueByPath(JsonDocument const& root, String
 }
 
 template std::pair<float, String> Utils::getJsonValueByPath(JsonDocument const& root, String const& path);
+
+template <typename T>
+std::optional<T> Utils::getNumericValueFromMqttPayload(char const* client,
+        std::string const& src, char const* topic, char const* jsonPath)
+{
+    std::string logValue = src.substr(0, 32);
+    if (src.length() > logValue.length()) { logValue += "..."; }
+
+    auto log = [client,topic](char const* format, auto&&... args) -> std::optional<T> {
+        MessageOutput.printf("[%s] Topic '%s': ", client, topic);
+        MessageOutput.printf(format, args...);
+        MessageOutput.println();
+        return std::nullopt;
+    };
+
+    if (strlen(jsonPath) == 0) {
+        auto res = getFromString<T>(src.c_str());
+        if (!res.has_value()) {
+            return log("cannot parse payload '%s' as float", logValue.c_str());
+        }
+        return res;
+    }
+
+    JsonDocument json;
+
+    const DeserializationError error = deserializeJson(json, src);
+    if (error) {
+        return log("cannot parse payload '%s' as JSON", logValue.c_str());
+    }
+
+    if (json.overflowed()) {
+        return log("payload too large to process as JSON");
+    }
+
+    auto pathResolutionResult = getJsonValueByPath<T>(json, jsonPath);
+    if (!pathResolutionResult.second.isEmpty()) {
+        return log("%s", pathResolutionResult.second.c_str());
+    }
+
+    return pathResolutionResult.first;
+}
+
+template std::optional<float> Utils::getNumericValueFromMqttPayload(char const* client,
+        std::string const& src, char const* topic, char const* jsonPath);
diff --git a/src/WebApi_battery.cpp b/src/WebApi_battery.cpp
@@ -41,7 +41,9 @@ void WebApiBatteryClass::onStatus(AsyncWebServerRequest* request)
     root["jkbms_interface"] = config.Battery.JkBmsInterface;
     root["jkbms_polling_interval"] = config.Battery.JkBmsPollingInterval;
     root["mqtt_soc_topic"] = config.Battery.MqttSocTopic;
+    root["mqtt_soc_json_path"] = config.Battery.MqttSocJsonPath;
     root["mqtt_voltage_topic"] = config.Battery.MqttVoltageTopic;
+    root["mqtt_voltage_json_path"] = config.Battery.MqttVoltageJsonPath;
 
     response->setLength();
     request->send(response);
@@ -80,7 +82,9 @@ void WebApiBatteryClass::onAdminPost(AsyncWebServerRequest* request)
     config.Battery.JkBmsInterface = root["jkbms_interface"].as<uint8_t>();
     config.Battery.JkBmsPollingInterval = root["jkbms_polling_interval"].as<uint8_t>();
     strlcpy(config.Battery.MqttSocTopic, root["mqtt_soc_topic"].as<String>().c_str(), sizeof(config.Battery.MqttSocTopic));
+    strlcpy(config.Battery.MqttSocJsonPath, root["mqtt_soc_json_path"].as<String>().c_str(), sizeof(config.Battery.MqttSocJsonPath));
     strlcpy(config.Battery.MqttVoltageTopic, root["mqtt_voltage_topic"].as<String>().c_str(), sizeof(config.Battery.MqttVoltageTopic));
+    strlcpy(config.Battery.MqttVoltageJsonPath, root["mqtt_voltage_json_path"].as<String>().c_str(), sizeof(config.Battery.MqttVoltageJsonPath));
 
     WebApi.writeConfig(retMsg);
 

diff --git a/webapp/src/locales/de.json b/webapp/src/locales/de.json
@@ -672,9 +672,12 @@
         "ProviderMqtt": "Batteriewerte aus MQTT Broker",
         "ProviderVictron": "Victron SmartShunt per VE.Direct Schnittstelle",
         "ProviderPytesCan": "Pytes per CAN-Bus",
-        "MqttConfiguration": "MQTT Einstellungen",
-        "MqttSocTopic": "Topic für Batterie-SoC",
-        "MqttVoltageTopic": "Topic für Batteriespannung",
+        "MqttSocConfiguration": "Einstellungen SoC",
+        "MqttVoltageConfiguration": "Einstellungen Spannung",
+        "MqttJsonPath": "Optional: JSON-Pfad",
+        "MqttJsonPathDescription": "Anwendungsspezifischer JSON-Pfad um den Wert in den JSON Nutzdatzen zu finden, z.B. 'electricLevel'. Leer lassen, falls die Nutzdaten des Topics einen numerischen Wert enthält.",
+        "MqttSocTopic": "Topic für SoC",
+        "MqttVoltageTopic": "Topic für Spannung",
         "JkBmsConfiguration": "JK BMS Einstellungen",
         "JkBmsInterface": "Schnittstellentyp",
         "JkBmsInterfaceUart": "TTL-UART an der MCU",

diff --git a/webapp/src/locales/en.json b/webapp/src/locales/en.json
@@ -675,8 +675,12 @@
         "ProviderVictron": "Victron SmartShunt using VE.Direct interface",
         "ProviderPytesCan": "Pytes using CAN bus",
         "MqttConfiguration": "MQTT Settings",
-        "MqttSocTopic": "SoC value topic",
-        "MqttVoltageTopic": "Voltage value topic",
+        "MqttSocConfiguration": "SoC Settings",
+        "MqttVoltageConfiguration": "Voltage Settings",
+        "MqttJsonPath": "Optional: JSON Path",
+        "MqttJsonPathDescription": "Application specific JSON path to find the value in the JSON payload, e.g., 'electricLevel'. Leave empty if the topic's payload contains a plain numeric value.",
+        "MqttSocTopic": "SoC Value Topic",
+        "MqttVoltageTopic": "Voltage Value Topic",
         "JkBmsConfiguration": "JK BMS Settings",
         "JkBmsInterface": "Interface Type",
         "JkBmsInterfaceUart": "TTL-UART on MCU",

diff --git a/webapp/src/locales/fr.json b/webapp/src/locales/fr.json
@@ -598,9 +598,12 @@
         "ProviderJkBmsSerial": "Jikong (JK) BMS using serial connection",
         "ProviderMqtt": "Battery data from MQTT broker",
         "ProviderVictron": "Victron SmartShunt using VE.Direct interface",
-        "MqttConfiguration": "MQTT Settings",
-        "MqttSocTopic": "SoC value topic",
-        "MqttVoltageTopic": "Voltage value topic",
+        "MqttSocConfiguration": "SoC Settings",
+        "MqttVoltageConfiguration": "Voltage Settings",
+        "MqttJsonPath": "Optional: JSON Path",
+        "MqttJsonPathDescription": "Application specific JSON path to find the value in the JSON payload, e.g., 'electricLevel'. Leave empty if the topic's payload contains a plain numeric value.",
+        "MqttSocTopic": "SoC Value Topic",
+        "MqttVoltageTopic": "Voltage Value Topic",
         "JkBmsConfiguration": "JK BMS Settings",
         "JkBmsInterface": "Interface Type",
         "JkBmsInterfaceUart": "TTL-UART on MCU",

diff --git a/webapp/src/types/BatteryConfig.ts b/webapp/src/types/BatteryConfig.ts
@@ -5,5 +5,7 @@ export interface BatteryConfig {
     jkbms_interface: number;
     jkbms_polling_interval: number;
     mqtt_soc_topic: string;
+    mqtt_soc_json_path: string;
     mqtt_voltage_topic: string;
+    mqtt_voltage_json_path: string;
 }
diff --git a/webapp/src/views/BatteryAdminView.vue b/webapp/src/views/BatteryAdminView.vue
@@ -49,29 +49,37 @@
                               type="number" min="2" max="90" step="1" :postfix="$t('batteryadmin.Seconds')"/>
             </CardElement>
 
-            <CardElement v-show="batteryConfigList.enabled && batteryConfigList.provider == 2"
-                         :text="$t('batteryadmin.MqttConfiguration')" textVariant="text-bg-primary" addSpace>
-                <div class="row mb-3">
-                    <label class="col-sm-2 col-form-label">
-                        {{ $t('batteryadmin.MqttSocTopic') }}
-                    </label>
-                    <div class="col-sm-10">
-                        <div class="input-group">
-                            <input type="text" class="form-control" v-model="batteryConfigList.mqtt_soc_topic" />
-                        </div>
-                    </div>
-                </div>
-                <div class="row mb-3">
-                    <label class="col-sm-2 col-form-label">
-                        {{ $t('batteryadmin.MqttVoltageTopic') }}
-                    </label>
-                    <div class="col-sm-10">
-                        <div class="input-group">
-                            <input type="text" class="form-control" v-model="batteryConfigList.mqtt_voltage_topic" />
-                        </div>
-                    </div>
-                </div>
-            </CardElement>
+            <template v-if="batteryConfigList.enabled && batteryConfigList.provider == 2">
+                <CardElement :text="$t('batteryadmin.MqttSocConfiguration')" textVariant="text-bg-primary" addSpace>
+
+                    <InputElement :label="$t('batteryadmin.MqttSocTopic')"
+                        v-model="batteryConfigList.mqtt_soc_topic"
+                        type="text"
+                        maxlength="256" />
+
+                    <InputElement :label="$t('batteryadmin.MqttJsonPath')"
+                        v-model="batteryConfigList.mqtt_soc_json_path"
+                        type="text"
+                        maxlength="128"
+                        :tooltip="$t('batteryadmin.MqttJsonPathDescription')" />
+
+                </CardElement>
+
+                <CardElement :text="$t('batteryadmin.MqttVoltageConfiguration')" textVariant="text-bg-primary" addSpace>
+
+                    <InputElement :label="$t('batteryadmin.MqttVoltageTopic')"
+                        v-model="batteryConfigList.mqtt_voltage_topic"
+                        type="text"
+                        maxlength="256" />
+
+                    <InputElement :label="$t('batteryadmin.MqttJsonPath')"
+                        v-model="batteryConfigList.mqtt_voltage_json_path"
+                        type="text"
+                        maxlength="128"
+                        :tooltip="$t('batteryadmin.MqttJsonPathDescription')" />
+
+                </CardElement>
+            </template>
 
             <FormFooter @reload="getBatteryConfig"/>
         </form>