Fix upscalling of data

The initial upscalling was kind of naive, it turns out that if you have
a serie with only a few datapoints but a very large time range we would
strech them and make some value appears when they don't really exist.

.config/jest.config.js

@@ -13,6 +13,10 @@ module.exports = {
     '\\.(css|scss|sass)$': 'identity-obj-proxy',
     'react-inlinesvg': path.resolve(__dirname, 'jest', 'mocks', 'react-inlinesvg.tsx'),
   },
+  watchPathIgnorePatterns: [
+    '/node_modules/', // Ignore changes in node_modules directory
+    '/dist/', // Ignore changes in the dist directory
+  ],
   modulePaths: ['<rootDir>/src'],
   setupFilesAfterEnv: ['<rootDir>/jest-setup.js'],
   testEnvironment: 'jest-environment-jsdom',

jest.config.js

@@ -2,7 +2,10 @@
 // generally used by snapshots, but can affect specific tests
 process.env.TZ = 'UTC';
 
-module.exports = {
+exports = {
   // Jest configuration provided by Grafana scaffolding
   ...require('./.config/jest.config'),
+  coverageDirectory: 'coverage/jest',
 };
+exports.coveragePathIgnorePatterns = ['.config/jest-setup.js'];
+module.exports = exports;

package.json

@@ -6,17 +6,26 @@
     "build": "webpack -c ./webpack.config.ts --env production",
     "dev": "webpack -w -c ./webpack.config.ts --env development",
     "test": "jest --watch --onlyChanged",
-    "test:ci": "jest --passWithNoTests --maxWorkers 4",
+    "test:ci": "jest --passWithNoTests --maxWorkers 4 --coverage",
     "typecheck": "tsc --noEmit",
     "lint": "eslint --cache --ignore-path ./.gitignore --ignore-path ./3rdparty/.eslintignore --ext .js,.jsx,.ts,.tsx .",
     "lint:fix": "npm run lint -- --fix",
     "e2e": "npm exec cypress install && npm exec grafana-e2e run",
     "e2e:update": "npm exec cypress install && npm exec grafana-e2e run --update-screenshots",
+    "copy:reports": "mkdir -p coverage/combined && cp coverage/cypress/coverage-final.json coverage/combined/from-cypress.json && cp coverage/jest/coverage-final.json coverage/combined/from-jest.json",
+    "combine:reports": "npx nyc merge coverage/combined && mv coverage.json .nyc_output/out.json",
+    "prereport:combined": "npm run combine:reports",
+    "report:combined": "npx nyc report --reporter text --reporter clover --reporter json --report-dir coverage",
+    "prereport:lcov": "npm run precombine:reports",
+    "report:lcov": "genhtml coverage/cypress/lcov.info coverage/jest/lcov.info --output-directory=coverage/lcov-report",
     "server": "docker-compose up --build",
     "sign": "npx --yes @grafana/sign-plugin@latest",
     "pretty": "prettier --single-quote  --write \"{src,__{tests,demo,dist}__}/**/*.ts*\"",
     "prepare": "relative-deps"
   },
+  "nyc": {
+    "report-dir": "coverage/cypress"
+  },
   "author": "Matthieu Patou",
   "license": "Apache-2.0",
   "devDependencies": {

src/components/SimplePanel.tsx

@@ -7,7 +7,7 @@ import * as cubism from 'cubism-es';
 import * as d3 from 'd3';
 
 import { config } from '@grafana/runtime';
-import { convertDataToCubism } from '../cubism_utils';
+import { convertAllDataToCubism} from '../cubism_utils';
 import { log_debug } from '../misc_utils';
 import { calculateSecondOffset } from '../date_utils';
 
@@ -21,15 +21,11 @@ interface Props extends PanelProps<SimpleOptions> {}
 
 type CSS = string;
 
-type Styles = {
-  wrapper: CSS;
-  d3inner: CSS;
-  d3outer: CSS;
-  svg: CSS;
-  textBox: CSS;
+interface CSSStyles {
+  [ key: string ]: CSS
 };
 
-const getStyles = (showText: boolean): (() => Styles) => {
+const getStyles = (showText: boolean): (() => CSSStyles) => {
   return function () {
     // 28px is the height of the axis
     let innerheight = 'calc(100% - 28px)';
@@ -38,27 +34,27 @@ const getStyles = (showText: boolean): (() => Styles) => {
       outerheight = 'calc(100% - 2em)';
     }
     return {
-      wrapper: css`
+      'wrapper': css`
         height: 100%;
         font-family: Open Sans;
         position: relative;
         overflow: hidden;
       `,
-      d3inner: css`
+      'd3inner': css`
         overflow: auto;
         height: ${innerheight};
       `,
-      d3outer: css`
+      'd3outer': css`
         position: relative;
         height: ${outerheight};
         overflow: hidden;
       `,
-      svg: css`
+      'svg': css`
         position: absolute;
         top: 0;
         left: 0;
       `,
-      textBox: css`
+      'textBox': css`
         max-height: 2em;
       `,
     };
@@ -71,7 +67,7 @@ export const D3Graph: React.FC<{
   width: number;
   data: PanelData;
   options: SimpleOptions;
-  stylesGetter: Styles;
+  stylesGetter: CSSStyles;
 }> = ({ height, width, data, options, stylesGetter }) => {
   const theme = useTheme2();
   let context = cubism.context();
@@ -107,10 +103,16 @@ export const D3Graph: React.FC<{
       }
       log_debug('Step is:', step);
       log_debug('Length of timestamps is ', cubismTimestamps.length);
+      log_debug('Size of the graph is ', size);
+
+      wrapperDiv.innerHTML = '';
+      wrapperDiv.className = stylesGetter["wrapper"];
+      if (data.series.length === 0) {
+        wrapperDiv.innerHTML = 'The series contained no data, check your query';
+        return;
+      }
+      let cubismData = convertAllDataToCubism(data.series, cubismTimestamps, context, step);
 
-      let cubismData = data.series.map(function (series, seriesIndex) {
-        return convertDataToCubism(series, seriesIndex, cubismTimestamps, context);
-      });
       cubismData = cubismData.filter(function (el) {
         if (el !== null) {
           return el;
@@ -121,8 +123,6 @@ export const D3Graph: React.FC<{
       now = Date.now();
       log_debug(`Took ${now - prev} to convert the series`);
 
-      wrapperDiv.innerHTML = '';
-      wrapperDiv.className = stylesGetter.wrapper;
 
       if (cubismData.length === 0) {
         wrapperDiv.innerHTML = 'The series contained no data, check your query';
@@ -132,11 +132,11 @@ export const D3Graph: React.FC<{
       // setup Div layout and set classes
       const delta = calculateSecondOffset(begin, +(end), request.timezone, request.range.from.utcOffset());
       const outerDiv = d3.create('div');
-      outerDiv.node()!.className = stylesGetter.d3outer;
+      outerDiv.node()!.className = stylesGetter["d3outer"];
       // TODO rename that to canvas
       const innnerDiv = d3.create('div');
       const axisDiv = d3.create('div');
-      innnerDiv.node()!.className = stylesGetter.d3inner;
+      innnerDiv.node()!.className = stylesGetter["d3inner"];
 
       // setup the context
       // size is the nubmer of pixel
@@ -226,7 +226,7 @@ export const D3Graph: React.FC<{
         log_debug('showing text');
         let msg = `${options.text}`;
         const msgDivContainer = d3.create('div');
-        msgDivContainer.node()!.className = stylesGetter.textBox;
+        msgDivContainer.node()!.className = stylesGetter["textBox"];
         msgDivContainer.append('div').text(msg);
         wrapperDiv.append(msgDivContainer.node()!);
       }

src/cubism_utils.ts

@@ -1,7 +1,10 @@
-import _ from 'lodash';
 import { DataFrame, getFieldDisplayName } from '@grafana/data';
+import { log_debug } from './misc_utils';
+import _ from 'lodash';
 
-export function upSampleData(dataPoints: number[], dataPointsTS: number[], pointIndex: number) {
+// Take the datapoints of the timeseries and its associated timestamps
+export function upSampleData(dataPoints: number[], dataPointsTS: number[]) {
+  let pointIndex = 0;
   return function (ts: number, tsIndex: number) {
     let point = dataPoints[pointIndex];
     let nextPoint = null;
@@ -35,7 +38,22 @@ export function downSampleData(timestamps: number[], dataAndTS: number[][], over
       });
 
     if (values.length === 0) {
-      return val;
+      if (tsIndex === 0 || nextTs === null) {
+        return null;
+      }
+      // Potentially extrapolate some points but it's not clear why we should do that
+      let lastTS = timestamps[tsIndex - 1];
+      values = dataAndTS
+        .filter(function (point) {
+          return point[0] >= lastTS && point[0] < ts;
+        })
+        .map(function (point) {
+          return point[1];
+        });
+
+      if (values.length === 0) {
+        return null;
+      }
     }
 
     if (override.summaryType === 'sum') {
@@ -76,28 +94,84 @@ export function minValue(values: number[]) {
   });
 }
 
-export function convertDataToCubism(series: DataFrame, seriesIndex: number, timestamps: number[], context: any) {
-  if (series.length > 0) {
-    let name = getFieldDisplayName(series.fields[1], series);
+// Take an array of timestamps that map to the way we want to display the timeseries in grafana
+// take also a serie
+export function convertDataToCubism(
+  serie: DataFrame,
+  serieIndex: number,
+  timestamps: number[],
+  context: any,
+  downSample: boolean
+) {
+  if (serie.length > 0) {
+    let name = getFieldDisplayName(serie.fields[1], serie);
     return context.metric(function (start: number, stop: number, step: number, callback: any) {
-      let dataPoints: number[] = series.fields[1].values;
-      let dataPointsTS: number[] = series.fields[0].values;
-      let values: number[] = [];
-      if (timestamps.length === dataPoints.length) {
-        values = dataPoints.map(function (point: number) {
-          return point;
-        });
-      } else if (timestamps.length > dataPoints.length) {
-        let pointIndex = 0;
-        values = _.chain(timestamps).map(upSampleData(dataPoints, dataPointsTS, pointIndex)).value();
-      } else {
-        let override = { summaryType: 'avg' };
-        let dataAndTS = dataPointsTS.map((item, index) => [item, dataPoints[index]]);
+      let dataPoints: number[] = serie.fields[1].values;
+      let dataPointsTS: number[] = serie.fields[0].values;
+      let values: Array<number | null> = [];
+      let override = { summaryType: 'avg' };
+      let dataAndTS = dataPointsTS.map((item, index) => [item, dataPoints[index]]);
+      if (downSample) {
         values = _.chain(timestamps).map(downSampleData(timestamps, dataAndTS, override)).value();
+      } else {
+        values = _.chain(timestamps).map(upSampleData(dataPoints, dataPointsTS)).value();
       }
       callback(null, values);
     }, name);
   } else {
     return null;
   }
 }
+
+export function convertAllDataToCubism(series: DataFrame[], cubismTimestamps: number[], context: any, step: number) {
+  let longest = series[0].length;
+  let longestIndex = 0;
+
+  for (let i = 1; i < series.length; i++) {
+    if (series[i].length > longest) {
+      longest = series[i].length;
+      longestIndex = i;
+    }
+  }
+  // Let's look at the longest one, if the step is bigger than what we have in the serie we downsample
+  const name = 'Time';
+  let s = series[longestIndex];
+  let ts = s.fields.filter(function (v) {
+    return v.name === name ? true : false;
+  })[0];
+  let previousts = -1;
+  let v: number[] = [];
+  if (ts === undefined) {
+    log_debug(`Couldn't find a field with name ${name} using field 0`);
+    ts = s.fields[0];
+  }
+  log_debug(`There is ${ts.values.length} elements in the longest`);
+  for (let i = 0; i < ts.values.length; i++) {
+    if (previousts !== -1) {
+      v.push(ts.values[i] - previousts);
+    }
+    previousts = ts.values[i];
+  }
+  v.sort((a: number, b: number) => a - b);
+
+  // Calculate the index for P99
+  const index = Math.ceil(0.99 * v.length) - 1;
+  // Look at what is the ratio when comparing the smallest step (ie. when we have most of the data
+  // to the largest (ie. when there is gaps), if the ratio is more than 3 we will
+  // downsample because it means that there is massive gaps
+  const stepRatio = v[index] / v[0];
+
+  let downsample = false;
+  // if there is too much missing points (ie. p99 has has at least 3x the smallest step then force
+  // downsample because upSample will not give good results
+  if (stepRatio > 3 || s.fields[0].values.length > cubismTimestamps.length) {
+    downsample = true;
+  }
+  log_debug(
+    `downsample = ${downsample} v[index] = ${v[index]} stepRatio = ${stepRatio} index = ${index}, step = ${step}`
+  );
+
+  return series.map(function (serie, serieIndex) {
+    return convertDataToCubism(serie, serieIndex, cubismTimestamps, context, downsample);
+  });
+}

src/misc_utils.ts

@@ -1,8 +1,14 @@
+let debug = false;
+export function enableDebug(): void {
+  debug = true;
+}
 export function log_debug(message?: any, ...optionalParams: any[]): void {
   // Set it to true to enable debugging
-  let debug = false;
   if (debug) {
-    return console.log(message, optionalParams);
+    if (optionalParams.length > 0) {
+      return console.log(message, optionalParams.join(' '));
+    } else {
+      return console.log(message);
+    }
   }
 }
-