feat: Support for HA long-term statistics (RomRider#208)

* ADD: Support for HA long-term statistics

* refactor

* fix readme

* fix typings

* refactor and fix

* minor refactor

* fix

* use start instead of end date to match statistics card

* Revert "use start instead of end date to match statistics card"

This reverts commit 45bd399.

* more options

* disable cache for statistics data

* data points alignment support

Fix RomRider#308 

Co-authored-by: Jérôme Wiedemann <jerome@wnetworks.org>

.devcontainer/ui-lovelace.yaml

@@ -1317,3 +1317,20 @@ views:
                     [tomorrow.getTime(), 30],
                     [tomorrow2.getTime(), 20]
                   ];
+  - title: Long Term Stats
+    panel: true
+    cards:
+      - type: custom:apexcharts-card
+        cache: false
+        graph_span: 1d
+        header:
+          show: true
+          title: Test Stats
+        all_series_config:
+          stroke_width: 1
+        series:
+          - entity: sensor.today_energy
+            type: column
+            statistics:
+              type: mean
+              period: 5minute

README.md

@@ -34,6 +34,7 @@ However, some things might be broken :grin:
   - [`header_actions` options](#header_actions-options)
   - [`*_action` options](#_action-options)
   - [`confirmation` options](#confirmation-options)
+  - [`statistics` options](#statistics-options)
   - [Main `show` Options](#main-show-options)
   - [`header` Options](#header-options)
   - [`now` Options](#now-options)
@@ -175,6 +176,7 @@ The card stricly validates all the options available (but not for the `apex_conf
 | `invert` | boolean | `false` | v1.2.0 | Negates the data (`1` -> `-1`). Usefull to display opposites values like network in (standard)/out (inverted) |
 | `transform` | string | | v1.5.0 | Transform your raw data in any way you like. See [transform](#transform-option) |
 | `data_generator` | string | | v1.2.0 | See [data_generator](#data_generator-option) |
+| `statistics` | object | | NEXT_VERSION | Use HA statistical data (long-term). See [statistics](#statistics-options) |
 | `offset` | string | | v1.3.0 | This is different from the main `offset` parameter. This is at the series level. It is only usefull if you want to display data from for eg. yesterday on top of the data from today for the same sensor and compare the data. The time displayed in the tooltip will be wrong as will the x axis information. Valid values are any negative time string, eg: `-1h`, `-12min`, `-1d`, `-1h25`, `-10sec`, ... `month` (365.25 days / 12) and `year` (365.25 days) as unit will generate inconsistent result, you should use days instead. |
 | `time_delta` | string | | NEXT_VERSION | This applies a time delta to all the datapoints of your chart **after** fetching them. You can cumulate it with `offset`. Valid values are any time strings starting with `+` or `-`, eg: `-30min`, `+2h`, `-2d`, ... |
 | `min` | number | `0` | v1.4.0 | Only used when `chart_type = radialBar`, see [chart_type](#chart_type-options). Used to convert the value into a percentage. Minimum value of the sensor |
@@ -261,6 +263,14 @@ series:
             - user: befc8496799848bda1824f2a8111e30a
 ```
 
+### `statistics` options
+
+| Name | Type | Default | Since | Description |
+| ---- | :--: | :-----: | :---: | ----------- |
+| `type` | string | `mean` | NEXT_VERSION | Type of long term statistic to pull. Can be one of `min`, `max`, `mean` or `sum` |
+| `period` | string | `hour` | NEXT_VERSION | Period of statistics to pull. Can be one of `5minute`, `hour`, `day` or `month` |
+| `align` | string | `middle` | NEXT_VERSION | Align the data points to the `start`, `end` or `middle` of the period of the statistics |
+
 ### Main `show` Options
 
 | Name | Type | Default | Since | Description |

src/apexcharts-card.ts

@@ -401,6 +401,7 @@ class ChartsCard extends LitElement {
             ['donut', 'pie', 'radialBar'].includes(this._config?.chart_type) &&
             (!serie.group_by || serie.group_by?.func === 'raw') &&
             !serie.data_generator &&
+            !serie.statistics &&
             !serie.offset
           );
           if (!this._headerColors[index]) {
@@ -451,9 +452,9 @@ class ChartsCard extends LitElement {
 
           if (serie.entity) {
             const editMode = getLovelace()?.editMode;
-            // disable caching for editor
+            // disable caching for editor or statistics data
             // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-            const caching = editMode === true ? false : this._config!.cache;
+            const caching = editMode === true || serie.statistics ? false : this._config!.cache;
             const graphEntry = new GraphEntry(
               index,
               // eslint-disable-next-line @typescript-eslint/no-non-null-assertion

src/const.ts

@@ -18,6 +18,8 @@ export const DEFAULT_SHOW_IN_HEADER = true;
 export const DEFAULT_SHOW_IN_CHART = true;
 export const DEFAULT_SHOW_NAME_IN_HEADER = true;
 export const DEFAULT_SHOW_OFFSET_IN_NAME = true;
+export const DEFAULT_STATISTICS_TYPE = 'mean';
+export const DEFAULT_STATISTICS_PERIOD = 'hour';
 
 export const DEFAULT_FLOAT_PRECISION = 1;
 

src/graphEntry.ts

@@ -7,15 +7,17 @@ import {
   HassHistoryEntry,
   HistoryBuckets,
   HistoryPoint,
+  Statistics,
+  StatisticValue,
 } from './types';
 import { compress, decompress, log } from './utils';
 import localForage from 'localforage';
 import { HassEntity } from 'home-assistant-js-websocket';
 import { DateRange } from 'moment-range';
-import { moment } from './const';
+import { DEFAULT_STATISTICS_PERIOD, DEFAULT_STATISTICS_TYPE, moment } from './const';
 import parse from 'parse-duration';
 import SparkMD5 from 'spark-md5';
-import { ChartCardSpanExtConfig } from './types-config';
+import { ChartCardSpanExtConfig, StatisticsPeriod } from './types-config';
 import * as pjson from '../package.json';
 
 export default class GraphEntry {
@@ -241,57 +243,95 @@ export default class GraphEntry {
         history.data.length !== 0 &&
         history.data[history.data.length - 1]
       );
-      const newHistory = await this._fetchRecent(
-        // if data in cache, get data from last data's time + 1ms
-        usableCache
-          ? // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-            new Date(history!.data[history!.data.length - 1][0] + 1)
-          : new Date(startHistory.getTime() + (this._config.group_by.func !== 'raw' ? 0 : -1)),
-        end,
-        this._config.attribute || this._config.transform ? false : skipInitialState,
-      );
-      if (newHistory && newHistory[0] && newHistory[0].length > 0) {
-        /*
-        hack because HA doesn't return anything if skipInitialState is false
-        when retrieving for attributes so we retrieve it and we remove it.
-        */
-        if ((this._config.attribute || this._config.transform) && skipInitialState) {
-          newHistory[0].shift();
-        }
-        let lastNonNull: number | null = null;
-        if (history && history.data && history.data.length > 0) {
-          lastNonNull = history.data[history.data.length - 1][1];
-        }
-        const newStateHistory: EntityCachePoints = newHistory[0].map((item) => {
-          let currentState: unknown = null;
-          if (this._config.attribute) {
-            if (item.attributes && item.attributes[this._config.attribute] !== undefined) {
-              currentState = item.attributes[this._config.attribute];
+
+      // if data in cache, get data from last data's time + 1ms
+      const fetchStart = usableCache
+        ? // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+          new Date(history!.data[history!.data.length - 1][0] + 1)
+        : new Date(startHistory.getTime() + (this._config.group_by.func !== 'raw' ? 0 : -1));
+      const fetchEnd = end;
+
+      let newStateHistory: EntityCachePoints = [];
+      let updateGraphHistory = false;
+
+      if (this._config.statistics) {
+        const newHistory = await this._fetchStatistics(fetchStart, fetchEnd, this._config.statistics.period);
+        if (newHistory && newHistory.length > 0) {
+          updateGraphHistory = true;
+          let lastNonNull: number | null = null;
+          if (history && history.data && history.data.length > 0) {
+            lastNonNull = history.data[history.data.length - 1][1];
+          }
+          newStateHistory = newHistory.map((item) => {
+            let stateParsed: number | null = null;
+            [lastNonNull, stateParsed] = this._transformAndFill(
+              item[this._config.statistics?.type || DEFAULT_STATISTICS_TYPE],
+              item,
+              lastNonNull,
+            );
+
+            let displayDate: Date | null = null;
+            const startDate = new Date(item.start);
+            if (!this._config.statistics?.align || this._config.statistics?.align === 'middle') {
+              if (this._config.statistics?.period === '5minute') {
+                displayDate = new Date(startDate.getTime() + 150000); // 2min30s
+              } else if (!this._config.statistics?.period || this._config.statistics.period === 'hour') {
+                displayDate = new Date(startDate.getTime() + 1800000); // 30min
+              } else if (this._config.statistics.period === 'day') {
+                displayDate = new Date(startDate.getTime() + 43200000); // 12h
+              } else {
+                displayDate = new Date(startDate.getTime() + 1296000000); // 15d
+              }
+            } else if (this._config.statistics.align === 'start') {
+              displayDate = new Date(item.start);
+            } else {
+              displayDate = new Date(item.end);
             }
-          } else {
-            currentState = item.state;
+
+            return [displayDate.getTime(), !Number.isNaN(stateParsed) ? stateParsed : null];
+          });
+        }
+      } else {
+        const newHistory = await this._fetchRecent(
+          fetchStart,
+          fetchEnd,
+          this._config.attribute || this._config.transform ? false : skipInitialState,
+        );
+        if (newHistory && newHistory[0] && newHistory[0].length > 0) {
+          updateGraphHistory = true;
+          /*
+          hack because HA doesn't return anything if skipInitialState is false
+          when retrieving for attributes so we retrieve it and we remove it.
+          */
+          if ((this._config.attribute || this._config.transform) && skipInitialState) {
+            newHistory[0].shift();
           }
-          if (this._config.transform) {
-            currentState = this._applyTransform(currentState, item);
+          let lastNonNull: number | null = null;
+          if (history && history.data && history.data.length > 0) {
+            lastNonNull = history.data[history.data.length - 1][1];
           }
-          let stateParsed: number | null = parseFloat(currentState as string);
-          stateParsed = !Number.isNaN(stateParsed) ? stateParsed : null;
-          if (stateParsed === null) {
-            if (this._config.fill_raw === 'zero') {
-              stateParsed = 0;
-            } else if (this._config.fill_raw === 'last') {
-              stateParsed = lastNonNull;
+          newStateHistory = newHistory[0].map((item) => {
+            let currentState: unknown = null;
+            if (this._config.attribute) {
+              if (item.attributes && item.attributes[this._config.attribute] !== undefined) {
+                currentState = item.attributes[this._config.attribute];
+              }
+            } else {
+              currentState = item.state;
             }
-          } else {
-            lastNonNull = stateParsed;
-          }
+            let stateParsed: number | null = null;
+            [lastNonNull, stateParsed] = this._transformAndFill(currentState, item, lastNonNull);
 
-          if (this._config.attribute) {
-            return [new Date(item.last_updated).getTime(), !Number.isNaN(stateParsed) ? stateParsed : null];
-          } else {
-            return [new Date(item.last_changed).getTime(), !Number.isNaN(stateParsed) ? stateParsed : null];
-          }
-        });
+            if (this._config.attribute) {
+              return [new Date(item.last_updated).getTime(), !Number.isNaN(stateParsed) ? stateParsed : null];
+            } else {
+              return [new Date(item.last_changed).getTime(), !Number.isNaN(stateParsed) ? stateParsed : null];
+            }
+          });
+        }
+      }
+
+      if (updateGraphHistory) {
         if (history?.data.length) {
           history.span = this._graphSpan;
           history.last_fetched = new Date();
@@ -337,7 +377,29 @@ export default class GraphEntry {
     return true;
   }
 
-  private _applyTransform(value: unknown, historyItem: HassHistoryEntry): number | null {
+  private _transformAndFill(
+    currentState: unknown,
+    item: HassHistoryEntry | StatisticValue,
+    lastNonNull: number | null,
+  ): [number | null, number | null] {
+    if (this._config.transform) {
+      currentState = this._applyTransform(currentState, item);
+    }
+    let stateParsed: number | null = parseFloat(currentState as string);
+    stateParsed = !Number.isNaN(stateParsed) ? stateParsed : null;
+    if (stateParsed === null) {
+      if (this._config.fill_raw === 'zero') {
+        stateParsed = 0;
+      } else if (this._config.fill_raw === 'last') {
+        stateParsed = lastNonNull;
+      }
+    } else {
+      lastNonNull = stateParsed;
+    }
+    return [lastNonNull, stateParsed];
+  }
+
+  private _applyTransform(value: unknown, historyItem: HassHistoryEntry | StatisticValue): number | null {
     return new Function('x', 'hass', 'entity', `'use strict'; ${this._config.transform}`).call(
       this,
       value,
@@ -395,6 +457,24 @@ export default class GraphEntry {
     };
   }
 
+  private async _fetchStatistics(
+    start: Date | undefined,
+    end: Date | undefined,
+    period: StatisticsPeriod = DEFAULT_STATISTICS_PERIOD,
+  ): Promise<StatisticValue[] | undefined> {
+    const statistics = await this._hass?.callWS<Statistics>({
+      type: 'history/statistics_during_period',
+      start_time: start?.toISOString(),
+      end_time: end?.toISOString(),
+      statistic_ids: [this._entityID],
+      period,
+    });
+    if (statistics && this._entityID in statistics) {
+      return statistics[this._entityID];
+    }
+    return undefined;
+  }
+
   private _dataBucketer(history: EntityEntryCache, timeRange: DateRange): HistoryBuckets {
     const ranges = Array.from(timeRange.reverseBy('milliseconds', { step: this._groupByDurationMs })).reverse();
     // const res: EntityCachePoints[] = [[]];

src/types-config-ti.ts

@@ -61,6 +61,8 @@ export const ChartCardSpanExtConfig = t.iface([], {
 
 export const ChartCardStartEnd = t.union(t.lit('minute'), t.lit('hour'), t.lit('day'), t.lit('week'), t.lit('month'), t.lit('year'), t.lit('isoWeek'));
 
+export const StatisticsPeriod = t.union(t.lit('5minute'), t.lit('hour'), t.lit('day'), t.lit('month'));
+
 export const ChartCardAllSeriesExternalConfig = t.iface([], {
   "entity": t.opt("string"),
   "attribute": t.opt("string"),
@@ -74,6 +76,11 @@ export const ChartCardAllSeriesExternalConfig = t.iface([], {
   "unit": t.opt("string"),
   "invert": t.opt("boolean"),
   "data_generator": t.opt("string"),
+  "statistics": t.opt(t.iface([], {
+    "type": t.opt(t.union(t.lit('mean'), t.lit('max'), t.lit('min'), t.lit('sum'))),
+    "period": t.opt("StatisticsPeriod"),
+    "align": t.opt(t.union(t.lit('start'), t.lit('end'), t.lit('middle'))),
+  })),
   "float_precision": t.opt("number"),
   "min": t.opt("number"),
   "max": t.opt("number"),
@@ -220,6 +227,7 @@ const exportedTypeSuite: t.ITypeSuite = {
   ChartCardBrushExtConfig,
   ChartCardSpanExtConfig,
   ChartCardStartEnd,
+  StatisticsPeriod,
   ChartCardAllSeriesExternalConfig,
   ActionsConfig,
   ChartCardSeriesShowConfigExt,

src/types-config.ts

@@ -61,6 +61,8 @@ export interface ChartCardSpanExtConfig {
 
 export type ChartCardStartEnd = 'minute' | 'hour' | 'day' | 'week' | 'month' | 'year' | 'isoWeek';
 
+export type StatisticsPeriod = '5minute' | 'hour' | 'day' | 'month';
+
 export interface ChartCardAllSeriesExternalConfig {
   entity?: string;
   attribute?: string;
@@ -74,6 +76,11 @@ export interface ChartCardAllSeriesExternalConfig {
   unit?: string;
   invert?: boolean;
   data_generator?: string;
+  statistics?: {
+    type?: 'mean' | 'max' | 'min' | 'sum';
+    period?: StatisticsPeriod;
+    align?: 'start' | 'end' | 'middle';
+  };
   float_precision?: number;
   min?: number;
   max?: number;

src/types.ts

@@ -50,6 +50,22 @@ export type EntityCachePoints = Array<HistoryPoint>;
 
 export type HistoryPoint = [number, number | null];
 
+export interface Statistics {
+  [statisticId: string]: StatisticValue[];
+}
+
+export interface StatisticValue {
+  statistic_id: string;
+  start: string;
+  end: string;
+  last_reset: string | null;
+  max: number | null;
+  mean: number | null;
+  min: number | null;
+  sum: number | null;
+  state: number | null;
+}
+
 export type HassHistory = Array<[HassHistoryEntry] | undefined>;
 
 export interface HassHistoryEntry {