fix(group_by): group_by reporting erronerous values for the first b…

…ucket of data

Fixes RomRider#110

.devcontainer/configuration.yaml

@@ -38,7 +38,23 @@ sensor:
     name: random_0_1000
     minimum: 0
     maximum: 1000
+  - platform: template
+    sensors:
+      counter:
+        friendly_name: 'Counter'
+        unit_of_measurement: 's'
+        value_template: '{{ as_timestamp(now()) }}'
 
 input_boolean:
   test_boolean:
     name: Test Input Boolean
+
+automation:
+  - alias: 'Update counter'
+    trigger:
+      - platform: time_pattern
+        seconds: '/1'
+    action:
+      - service: homeassistant.update_entity
+        target:
+          entity_id: sensor.counter

.devcontainer/ui-lovelace.yaml

@@ -257,10 +257,13 @@ views:
               - entity: sensor.humidity
                 type: line
                 name: Outside Humidity
+                fill_raw: last
                 group_by:
                   func: avg
+                  fill: last
                   duration: 30min
               - entity: sensor.random0_100
+                fill_raw: last
                 type: column
                 name: Office Humidity
                 group_by:
@@ -540,7 +543,7 @@ views:
             series:
               - entity: sensor.outside_temperature
                 curve: stepline
-                extend_to_end: false
+                extend_to_end: true
 
           - type: custom:apexcharts-card
             update_delay: 3s
@@ -943,3 +946,19 @@ views:
                   duration: 1h
                 show:
                   in_header: before_now
+
+          - type: custom:apexcharts-card
+            header:
+              show: true
+              title: Shouldn't show a spike at the begining
+            graph_span: 60m
+            update_interval: 1m
+            span:
+              end: hour
+            series:
+              - entity: sensor.counter
+                type: column
+                # offset: +5min
+                group_by:
+                  func: diff
+                  duration: 10s

src/graphEntry.ts

@@ -164,8 +164,8 @@ export default class GraphEntry {
     let startHistory = new Date(start);
     if (this._config.group_by.func !== 'raw') {
       const range = end.getTime() - start.getTime();
-      const nbBuckets = Math.abs(range / this._groupByDurationMs) + (range % this._groupByDurationMs > 0 ? 1 : 0);
-      startHistory = new Date(end.getTime() - nbBuckets * this._groupByDurationMs);
+      const nbBuckets = Math.floor(range / this._groupByDurationMs) + (range % this._groupByDurationMs > 0 ? 1 : 0);
+      startHistory = new Date(end.getTime() - (nbBuckets + 1) * this._groupByDurationMs);
     }
     if (!this._entityState || this._updating) return false;
     this._updating = true;
@@ -182,7 +182,9 @@ export default class GraphEntry {
       history = this._cache ? await this._getCache(this._entityID, this._useCompress) : undefined;
 
       if (history && history.span === this._graphSpan) {
-        const currDataIndex = history.data.findIndex((item) => item && new Date(item[0]).getTime() > start.getTime());
+        const currDataIndex = history.data.findIndex(
+          (item) => item && new Date(item[0]).getTime() > startHistory.getTime(),
+        );
         if (currDataIndex !== -1) {
           // skip initial state when fetching recent/not-cached data
           skipInitialState = true;
@@ -197,12 +199,18 @@ export default class GraphEntry {
       } else {
         history = undefined;
       }
+      const usableCache = !!(
+        history &&
+        history.data &&
+        history.data.length !== 0 &&
+        history.data[history.data.length - 1]
+      );
       const newHistory = await this._fetchRecent(
         // if data in cache, get data from last data's time + 1ms
-        history && history.data && history.data.length !== 0 && history.data.slice(-1)[0]
+        usableCache
           ? // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-            new Date(history.data.slice(-1)[0]![0] + 1)
-          : startHistory,
+            new Date(history!.data[history!.data.length - 1][0] + 1)
+          : new Date(startHistory.getTime() + (this._config.group_by.func !== 'raw' ? 0 : -1)),
         end,
         this._config.attribute || this._config.transform ? false : skipInitialState,
         this._config.attribute || this._config.transform ? true : false,
@@ -353,24 +361,13 @@ export default class GraphEntry {
     ranges.forEach((range, index) => {
       buckets[index] = { timestamp: range.valueOf(), data: [] };
     });
-    let lastNotNullValue: number | null = null;
     history?.data.forEach((entry) => {
-      let properEntry = entry;
-      // Fill null values
-      if (properEntry[1] === null) {
-        if (this._config.group_by.fill === 'last') {
-          properEntry = [entry[0], lastNotNullValue];
-        } else if (this._config.group_by.fill === 'zero') {
-          properEntry = [entry[0], 0];
-        }
-      } else {
-        lastNotNullValue = properEntry[1];
-      }
-
       buckets.some((bucket, index) => {
-        if (bucket.timestamp > properEntry[0] && index > 0) {
-          buckets[index - 1].data.push(properEntry);
-          return true;
+        if (bucket.timestamp > entry[0] && index > 0) {
+          if (entry[0] >= buckets[index - 1].timestamp) {
+            buckets[index - 1].data.push(entry);
+            return true;
+          }
         }
         return false;
       });
@@ -406,6 +403,7 @@ export default class GraphEntry {
         }
       }
     });
+    buckets.shift();
     buckets.pop();
     // Remove nulls at the end
     while (
@@ -415,13 +413,6 @@ export default class GraphEntry {
     ) {
       buckets.pop();
     }
-    // Remove nulls at the beginning
-    while (
-      buckets.length > 0 &&
-      (buckets[0].data.length === 0 || (buckets[0].data.length === 1 && buckets[0].data[0][1] === null))
-    ) {
-      buckets.shift();
-    }
     return buckets;
   }