feat(z2m-light-controller): allow direct mqtt calls

related to #168

apps/controllerx/cx_core/controller.py

@@ -23,6 +23,7 @@
 
 import appdaemon.utils as utils
 import cx_version
+from appdaemon.adapi import ADAPI
 from appdaemon.plugins.hass.hassapi import Hass
 from appdaemon.plugins.mqtt.mqttapi import Mqtt
 from cx_const import (
@@ -193,9 +194,8 @@ def filter_actions(
             if key in allowed_actions
         }
 
-    def get_option(
-        self, value: str, options: List[str], ctx: Optional[str] = None
-    ) -> str:
+    @staticmethod
+    def get_option(value: str, options: List[str], ctx: Optional[str] = None) -> str:
         if value in options:
             return value
         else:
@@ -309,7 +309,10 @@ def format_multiple_click_action(
 
     async def _render_template(self, template: str) -> Any:
         result = await self.call_service(
-            "template/render", template=template, return_result=True
+            "template/render",
+            render_template=False,
+            template=template,
+            return_result=True,
         )
         if result is None:
             raise ValueError(f"Template {template} returned None")
@@ -341,17 +344,19 @@ async def render_attributes(self, attributes: Dict[str, Any]) -> Dict[str, Any]:
             new_attributes[key] = new_value
         return new_attributes
 
-    async def call_service(self, service: str, **attributes: Any) -> Optional[Any]:
+    async def call_service(
+        self, service: str, render_template: bool = True, **attributes: Any
+    ) -> Optional[Any]:
         service = service.replace(".", "/")
         to_log = ["\n", f"🤖 Service: \033[1m{service.replace('/', '.')}\033[0m"]
-        if service != "template/render":
+        if service != "template/render" or render_template:
             attributes = await self.render_attributes(attributes)
         for attribute, value in attributes.items():
             if isinstance(value, float):
                 value = f"{value:.2f}"
             to_log.append(f"  - {attribute}: {value}")
         self.log("\n".join(to_log), level="INFO", ascii_encode=False)
-        return await Hass.call_service(self, service, **attributes)
+        return await ADAPI.call_service(self, service, **attributes)
 
     @utils.sync_wrapper  # type: ignore[misc]
     async def get_state(

apps/controllerx/cx_core/type/light_controller.py

@@ -30,10 +30,10 @@
 DEFAULT_TRANSITION_TURN_TOGGLE = False
 DEFAULT_HOLD_TOGGLE_DIRECTION_INIT = "up"
 
-ColorMode = str
 # Once the minimum supported version of Python is 3.8,
 # we can declare the ColorMode as a Literal
 # ColorMode = Literal["auto", "xy_color", "color_temp"]
+ColorMode = str
 
 COLOR_MODES = {"hs", "xy", "rgb", "rgbw", "rgbww"}
 STEPPER_MODES: Dict[str, Type[Stepper]] = {

apps/controllerx/cx_core/type/z2m_light_controller.py

@@ -1,10 +1,10 @@
 import asyncio
 import json
 from functools import lru_cache
-from typing import Any, Dict, Optional, Set, Type
+from typing import Any, Awaitable, Callable, Dict, List, Optional, Type
 
 from cx_const import PredefinedActionsMapping, StepperDir, Z2MLight
-from cx_core.controller import action
+from cx_core.controller import Controller, action
 from cx_core.integration import EventData
 from cx_core.integration.z2m import Z2MIntegration
 from cx_core.stepper import InvertStepper, MinMax
@@ -14,8 +14,27 @@
 DEFAULT_HOLD_STEPS = 70
 DEFAULT_TRANSITION = 0.5
 
+# Once the minimum supported version of Python is 3.8,
+# we can declare the Mode as a Literal
+# Mode = Literal["ha", "mqtt"]
+Mode = str
 
-class Z2MLightController(TypeController[Entity]):
+
+class Z2MLightEntity(Entity):
+    mode: Mode
+
+    def __init__(
+        self,
+        name: str,
+        entities: Optional[List[str]] = None,
+        mode: Mode = "ha",
+    ) -> None:
+        super().__init__(name, entities)
+        mode = Controller.get_option(mode, ["ha", "mqtt"])
+        self.mode = mode
+
+
+class Z2MLightController(TypeController[Z2MLightEntity]):
     """
     This is the main class that controls the Zigbee2MQTT lights for different devices.
     Type of actions:
@@ -44,18 +63,23 @@ class Z2MLightController(TypeController[Entity]):
     transition: float
     use_onoff: bool
 
-    _supported_color_modes: Optional[Set[str]]
+    _mqtt_fn: Dict[Mode, Callable[[str, str], Awaitable[None]]]
 
     async def init(self) -> None:
         self.click_steps = self.args.get("click_steps", DEFAULT_CLICK_STEPS)
         self.hold_steps = self.args.get("hold_steps", DEFAULT_HOLD_STEPS)
         self.transition = self.args.get("transition", DEFAULT_TRANSITION)
         self.use_onoff = self.args.get("use_onoff", False)
 
+        self._mqtt_fn = {
+            "ha": self._ha_mqtt_call,
+            "mqtt": self._mqtt_plugin_call,
+        }
+
         await super().init()
 
-    def _get_entity_type(self) -> Type[Entity]:
-        return Entity
+    def _get_entity_type(self) -> Type[Z2MLightEntity]:
+        return Z2MLightEntity
 
     def get_predefined_actions_mapping(self) -> PredefinedActionsMapping:
         return {
@@ -171,11 +195,17 @@ def get_predefined_actions_mapping(self) -> PredefinedActionsMapping:
             Z2MLight.BRIGHTNESS_FROM_CONTROLLER_ANGLE: self.brightness_from_controller_angle,
         }
 
-    async def _mqtt_call(self, payload: Dict[str, Any]) -> None:
+    async def _ha_mqtt_call(self, topic: str, payload: str) -> None:
+        await self.call_service("mqtt.publish", topic=topic, payload=payload)
+
+    async def _mqtt_plugin_call(self, topic: str, payload: str) -> None:
         await self.call_service(
-            "mqtt.publish",
-            topic=f"zigbee2mqtt/{self.entity.name}/set",
-            payload=json.dumps(payload),
+            "mqtt.publish", topic=topic, payload=payload, namespace="mqtt"
+        )
+
+    async def _mqtt_call(self, payload: Dict[str, Any]) -> None:
+        await self._mqtt_fn[self.entity.mode](
+            f"zigbee2mqtt/{self.entity.name}/set", json.dumps(payload)
         )
 
     async def _on(self, **attributes: Any) -> None:

docs/docs/faq.md

@@ -70,4 +70,4 @@ example_app:
   media_player: media_player.my_media_player
 ```
 
-Notice how we added the `listen_to` attribute and change the `controller` to the Zigbee2MQTT friendly name. Then, you will also need to add the MQTT broker and the credentials in the `appdaemon.yaml` as described in the [MQTT section](/controllerx/start/integrations#mqtt) from the integrations page. Then you can just restart the AppDaemon addon/server.
+Notice how we added the `listen_to` attribute and change the `controller` to the Zigbee2MQTT friendly name. Finally, you will also need to have the [MQTT plugin enabled](/controllerx/others/enable-mqtt-plugin).

docs/docs/others/enable-mqtt-plugin.md

@@ -0,0 +1,37 @@
+---
+title: Enable MQTT Plugin
+layout: page
+---
+
+If we want to use the `mqtt` integration or the `listen_to: mqtt` from `z2m` integration as well as the `Z2MLightController`, we will need to activate the MQTT Plugin on the AppDaemon configuration (normally located in `/config/appdaemon/appdaemon.yaml`). We will need the add the following highlighted section in that file:
+
+```yaml hl_lines="12 13 14 15 16 17"
+---
+secrets: /config/secrets.yaml
+appdaemon:
+  latitude: X.XXXXXXX
+  longitude: X.XXXXXXX
+  elevation: XXXX
+  time_zone: XXXXXXXX
+  missing_app_warnings: 0 # (1)
+  plugins:
+    HASS:
+      type: hass
+    MQTT:
+      type: mqtt
+      namespace: mqtt # This is important
+      client_host: host # (2)
+      client_user: XXXXX # (3)
+      client_password: XXXXX
+http:
+  url: http://127.0.0.1:5050
+admin:
+api:
+hadashboard:
+```
+
+1. Extra tip: you can add `missing_app_warnings` if you don't want any warning spam from ControllerX when starting AppDaemon.
+2. This is the host without indicating the port (e.g. 192.168.1.10).
+3. You should be able to get user and password from MQTT broker.
+
+Then you can just restart the AppDaemon addon/server.

docs/docs/others/zigbee2mqtt-light-controller.md

@@ -0,0 +1,85 @@
+---
+title: Zigbee2MQTT Light Controller
+layout: page
+---
+
+_This is supported from ControllerX v4.21.0_
+
+ControllerX has always given support for [Light Controller](/controllerx/start/type-configuration#light-controller) which allows amongst other features to smoothly change attributes (brightness, color temperature) values by requesting the changes periodically to Home Assistant. This has allowed to work with lights integrated with many integrations (e.g.: Zigbee2MQTT, deCONZ, WLED, Hue). However, this generalization has penalized Zigbee2MQTT which has its own mechanism to change brightness and color temp over time which works much smoother than the Light Controller.
+
+Zigbee2MQTT allows to send the following topic `zigbee2mqtt/FRIENDLY_NAME/set` with a payload like `{"brightness_move": -40}` which will change the brightness down with 40 steps over time. Then, we can send to the same topic the following payload to make it stop: `{"brightness_move": "stop"}`. Zigbee2MQTT does not have an specific page with this documentation since it depends on the device itself. For example, we can see all this further explained for the [LED1545G12](https://www.zigbee2mqtt.io/devices/LED1545G12.html#light) light.
+
+ControllerX has always wanted to integrate this inside the Light Controller, but there are many features that are not compatible with what Zigbee2MQTT offers:
+
+- [Hold/Click modes](/controllerx/advanced/hold-click-modes) (bounce, loop)
+- Color looping
+- Define a minimum and maximum attribute values
+
+For this reason, it has been decided to create a new controller type, [Z2M Light Controller](/controllerx/start/type-configuration#z2m-light-controller), which allows most of the same functionalities as Light Controller, but using MQTT features from Zigbee2MQTT.
+
+Imagine we have a light that in Zigbee2MQTT has friendly name `livingroom` and entity `light.livingroom` in Home Assistant. Then, let's say we had the following ControllerX configuration for [E1810](/controllerx/controllers/E1810):
+
+```yaml
+livingroom_controller:
+  module: controllerx
+  class: E1810Controller # (1)
+  controller: sensor.livingroom_controller_action
+  integration: z2m
+  light: light.livingroom
+```
+
+1. `E1810Controller` is a `Light Controller`
+
+This allows us to control the the `livingroom` light with the Light Controller, however if we replace `E1810Controller` for the new `E1810Z2MLightController` and the `light.livingroom` for `livingroom` we will be using Z2M Light Controller:
+
+```yaml hl_lines="3 6"
+livingroom_controller:
+  module: controllerx
+  class: E1810Z2MLightController # (1)
+  controller: sensor.livingroom_controller_action
+  integration: z2m
+  light: livingroom # (2)
+```
+
+1.  This is a `Z2M Light Controller`
+2.  This the Zigbee2MQTT friendly name
+
+This will be sending MQTT messages through Home Assistant, but if we have the [MQTT plugin enabled](/controllerx/others/enable-mqtt-plugin) in AppDaemon, then we could send the MQTT through MQTT plugin:
+
+```yaml hl_lines="6 7 8"
+livingroom_controller:
+  module: controllerx
+  class: E1810Z2MLightController
+  controller: sensor.livingroom_controller_action
+  integration: z2m
+  light:
+    name: livingroom
+    mode: mqtt # (1)
+```
+
+1.  `mode` can either be `ha` or `mqtt` (default: `ha`). On the one hand, `ha` will send the mqtt messages through Home Assistant with [`mqtt.publish` service](https://www.home-assistant.io/docs/mqtt/service/#service-mqttpublish). On the other hand, `mqtt` will send the MQTT messages through MQTT plugin from AppDaemon (hence skipping HA).
+
+Finally, we can have the full ControllerX configuration listening and sending to MQTT broker directly without going through Home Assistant:
+
+```yaml hl_lines="5 6 7"
+livingroom_controller:
+  module: controllerx
+  class: E1810Z2MLightController
+  controller: livingroom_controller # (1)
+  integration:
+    name: z2m
+    listen_to: mqtt
+  light:
+    name: livingroom
+    mode: mqtt
+```
+
+1.  `livingroom_controller` is the Zigbee2MQTT friendly name of the controller
+
+With this latest configuration, we can keep using the light even if Home Assistant is down since all interactions go:
+
+```
+Zigbee2MQTT <> MQTT Broker <> AppDaemon (ControllerX)
+```
+
+Many of the existing devices now have support to `Z2MLightController`, and you can use it in the `class` as you can now use `LightController` as well. Check [Supported controllers](/controllerx/controllers) pages to see the class names.

docs/docs/start/integrations.md

@@ -34,7 +34,7 @@ livingroom_controller:
 
 Three things to clarify when using the `z2m` integration listening to MQTT:
 
-- `appdaemon.yaml` needs to be changed by adding the MQTT plugin (see `MQTT` section below).
+- `appdaemon.yaml` needs to be changed by adding the MQTT plugin (see [here](/controllerx/others/enable-mqtt-plugin))
 - The Zigbee2MQTT friendly name from the z2m needs to be specified in the `controller` attribute.
 - `action_key` is the key inside the topic payload that contains the fired action from the controller. It is normally `action` or `click`. By default will be `action`.
 - `action_group` is a list of allowed action groups for the controller configuration. Read more about it [here](https://github.com/xaviml/controllerx/pull/150).
@@ -116,34 +116,7 @@ This example will turn on the light when the following payload is shown for one
 }
 ```
 
-By default, mqtt will read non-JSON values. Last but not least, MQTT needs to be configured on `appdaemon.yaml` by adding the `MQTT` plugin, apart from the `HASS` plugin. The whole file should look like the following:
-
-```yaml
----
-secrets: /config/secrets.yaml
-appdaemon:
-  latitude: X.XXXXXXX
-  longitude: X.XXXXXXX
-  elevation: XXXX
-  time_zone: XXXXXXXX
-  # You can add `missing_app_warnings` if you don't want any
-  # warning spam from ControllerX when starting AppDaemon
-  missing_app_warnings: 0
-  plugins:
-    HASS:
-      type: hass
-    MQTT:
-      type: mqtt
-      namespace: mqtt # This is important
-      client_host: <Host without indicating the port (e.g. 192.168.1.10)>
-      client_user: XXXXX
-      client_password: XXXXX
-http:
-  url: http://127.0.0.1:5050
-admin:
-api:
-hadashboard:
-```
+By default, mqtt will read non-JSON values. Last but not least, the [MQTT plugin needs to be enabled](/controllerx/others/enable-mqtt-plugin).
 
 #### State
 

docs/docs/start/type-configuration.md

@@ -86,16 +86,25 @@ This controller (`Z2MLightController`) allows the devices to control Zigbe2MQTT
 - Manual increase/decrease of brightness and color
 - Smooth increase/decrease (holding button) of brightness and color
 
-| key           | type   | value      | description                                                                                            |
-| ------------- | ------ | ---------- | ------------------------------------------------------------------------------------------------------ |
-| `light`\*     | string | `my_light` | The light you want to control. This is the friendly name light from Zigbee2MQTT.                       |
-| `click_steps` | float  | 70         | Number of steps that are passed to Zigbee2MQTT for click actions.                                      |
-| `hold_steps`  | float  | 70         | Number of steps that are passed to Zigbee2MQTT for hold actions.                                       |
-| `transition`  | float  | 0.5        | Transition sent to Zigbee2MQTT when changing brightness or color temp.                                 |
-| `use_onoff`   | bool   | `false`    | This allows click and hold actions to turn on/off the light when off or minimum brightness is reached. |
+| key           | type                 | value      | description                                                                                            |
+| ------------- | -------------------- | ---------- | ------------------------------------------------------------------------------------------------------ |
+| `light`\*     | string \| dictionary | `my_light` | The light you want to control. This is the friendly name light from Zigbee2MQTT.                       |
+| `click_steps` | float                | 70         | Number of steps that are passed to Zigbee2MQTT for click actions.                                      |
+| `hold_steps`  | float                | 70         | Number of steps that are passed to Zigbee2MQTT for hold actions.                                       |
+| `transition`  | float                | 0.5        | Transition sent to Zigbee2MQTT when changing brightness or color temp.                                 |
+| `use_onoff`   | bool                 | `false`    | This allows click and hold actions to turn on/off the light when off or minimum brightness is reached. |
 
 _\* Required fields_
 
+_Light dictionary for the `light` attribute:_
+
+| key      | type   | value           | description                                                                                                                                                                                                                                                                                                                       |
+| -------- | ------ | --------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `name`\* | string | `light.kitchen` | The light you want to control. This is the friendly name light from Zigbee2MQTT.                                                                                                                                                                                                                                                  |
+| `mode`   | string | `ha`            | This attribute can take `ha`, `mqtt`. On the one hand, `ha` will send the mqtt messages through Home Assistant with [`mqtt.publish` service](https://www.home-assistant.io/docs/mqtt/service/#service-mqttpublish). On the other hand, `mqtt` will send the MQTT messages through MQTT plugin from AppDaemon (hence skipping HA). |
+
+_\* Required fields_\*\*\*\*
+
 ## Media player controller
 
 This allows you to control media players. It supports volume, play/pause and skipping forward/backward the track and the source.

docs/mkdocs.yml

@@ -35,6 +35,8 @@ nav:
       - others/extract-controller-id.md
       - others/run-appdaemon.md
       - others/update.md
+      - others/zigbee2mqtt-light-controller.md
+      - others/enable-mqtt-plugin.md
   - faq.md
 
 theme:
@@ -48,6 +50,7 @@ theme:
     - search.suggest
     - search.highlight
     - search.share
+    - content.code.annotate
   icon:
     repo: material/github
   logo: assets/logo_white.png

tests/conftest.py

@@ -5,6 +5,7 @@
 import appdaemon.plugins.hass.hassapi as hass
 import appdaemon.plugins.mqtt.mqttapi as mqtt
 import pytest
+from appdaemon.adapi import ADAPI
 from cx_core import Controller
 from pytest import MonkeyPatch
 
@@ -37,7 +38,7 @@ def hass_mock(monkeypatch: MonkeyPatch) -> None:
     monkeypatch.setattr(mqtt.Mqtt, "listen_event", fake_fn(async_=True))
     monkeypatch.setattr(hass.Hass, "listen_state", fake_fn(async_=True))
     monkeypatch.setattr(hass.Hass, "log", fake_fn())
-    monkeypatch.setattr(hass.Hass, "call_service", fake_fn(async_=True))
+    monkeypatch.setattr(ADAPI, "call_service", fake_fn(async_=True))
     monkeypatch.setattr(hass.Hass, "get_ad_version", fake_fn(to_return="4.0.0"))
     monkeypatch.setattr(hass.Hass, "run_in", fake_run_in)
     monkeypatch.setattr(hass.Hass, "cancel_timer", fake_cancel_timer)