feature(tools): finalize manual data export to opensearch

README.md

@@ -157,7 +157,13 @@ Small CLI program to diagnose reading of teleinfo data on serial input: `npm run
 Configuration is set via `tools/scripts/teleinfo-reader/config/teleinfo-reader.json` file; see 'Teleinfo section' above.
 
 ### opensearch-data-export
-This script will push info from the persisted datastore to opensearch indexes.
+This script will push info from the persisted datastore to an opensearch index: `npm run tool:opensearch-data-export <data store file> <index name>`.
+
+- `data store file`: path to the data store JSON file
+- `index name`: index name to use on opensearch instance
+
+Configuration is set via `tools/scripts/opensearch-data-export/config/opensearch-data-export.json`.
+
 
 ### Diagnostics with picocom
 It's also possible to capture raw data on serial input, using `picocom` command line tool: https://linux.die.net/man/8/picocom
@@ -171,7 +177,7 @@ with (assuming teleinfo data is on *historical* mode):
 - -b = 1200 (bauds rate)
 - -d = 7 (databits) 
 - -p = e (even parity)
-- -f = n (no flow control`
+- -f = n (no flow control)
 - by default, stop bit count is set to 1.
 
 For people looking for alternatives, `minicom` should also work with equivalent settings.

tools/scripts/opensearch-data-export/model/datastore.ts

@@ -0,0 +1,20 @@
+export type StatsItem = {
+  min: number,
+  minTimestamp: number,
+  max: number,
+  maxTimestamp: number,
+}
+
+export type Stats = {
+  apparentPower: StatsItem,
+  instantIntensity: StatsItem,
+};
+
+export type EntryValue = number | number[] | Stats;
+
+export type StoreEntries = Record<string, EntryValue>;
+
+export type Store = {
+  meta: unknown,
+  data: StoreEntries,
+};

tools/scripts/opensearch-data-export/model/export.ts

@@ -0,0 +1,6 @@
+import type { DocumentByDate } from './opensearch';
+
+export type GroupedData = {
+  perDay: DocumentByDate,
+  perMonth: DocumentByDate,
+};

tools/scripts/opensearch-data-export/model/opensearch.ts

@@ -0,0 +1,29 @@
+export type DocStatsItem = {
+  min: number,
+  minDate: Date,
+  max: number,
+  maxDate: Date,
+};
+
+export type DocPeriodicInfo = {
+  period1: number,
+  period2: number,
+};
+
+export type Document = {
+  costs?: number,
+  date: Date,
+  indexes?: DocPeriodicInfo,
+  options: {
+    fareOption: string,
+    period1Label: string,
+    period2Label: string,
+  },
+  statistics?: {
+    apparentPower: DocStatsItem,
+    instantIntensity: DocStatsItem,
+  },
+  supplied?: DocPeriodicInfo,
+}
+
+export type DocumentByDate = Record<string, Document>;

tools/scripts/opensearch-data-export/opensearch-data-export.ts

@@ -1,72 +1,151 @@
 import fs from 'fs/promises';
-import appRootDir from 'app-root-dir';
 import { Client } from '@opensearch-project/opensearch';
-import path from 'path';
+import parseDate from 'date-fns/parse';
+import toDate from 'date-fns/toDate';
 import config from './config/opensearch-data-export.json';
 
+import type { EntryValue, StatsItem, Stats, Store, StoreEntries } from './model/datastore';
+import type { DocStatsItem, DocumentByDate } from './model/opensearch';
+import type { GroupedData } from './model/export';
+
+const IGNORED_STORE_KEYS_PREFIXES = ['FIRST_DATA_TIMESTAMP', 'INITIAL_', 'TOTAL_', 'OVERALL_', 'YEAR_'];
+
 function createOpenSearchClient() {
   const { opensearchInstance, user, password } = config;
-  const url = new URL(config.opensearchInstance);
+  const url = new URL(opensearchInstance);
 
   // Optional client certificates if you don't want to use HTTP basic authentication.
   // var client_cert_path = '/full/path/to/client.pem'
   // var client_key_path = '/full/path/to/client-key.pem'
 
   // Create a client with SSL/TLS enabled.
   const client = new Client({
-    node: `${url.protocol}://${user}:${password}@${url.hostname}:${url.port}`,
+    node: `${url.protocol}//${user}:${password}@${url.hostname}:${url.port}`,
     ssl: {
       rejectUnauthorized: false,
     },
   });
   return client;
 }
 
-async function readDatastore() {
-  const datastoreFilePath = path.join(
-    appRootDir.get(),
-    'config',
-    'MMM-LKY-TIC.datastore.json'
-  );
-  const contents = await fs.readFile(datastoreFilePath, 'utf8');
-  return JSON.parse(contents);
+async function readDatastore(storeFilePath: string) {
+  const contents = await fs.readFile(storeFilePath, 'utf8');
+  return JSON.parse(contents) as Store;
+}
+
+function extractDate(storeKey: string) {
+  return storeKey.substring(storeKey.length - 8);
+}
+
+function convertStats(statsItem: StatsItem): DocStatsItem {
+  const { min, max, minTimestamp, maxTimestamp } = statsItem;
+  return {
+    max,
+    maxDate: toDate(maxTimestamp),
+    min,
+    minDate: toDate(minTimestamp),
+  };
+}
+
+function parseData(target: DocumentByDate, storeKey: string, storeValue: EntryValue) {
+  const date = extractDate(storeKey)
+  let docItem = target[date];
+  if (!docItem) {
+    docItem = {
+      date: parseDate(date, 'yyyyMMdd', new Date()),
+      options: {
+        fareOption: 'HP/HC',
+        period1Label: 'HP',
+        period2Label: 'HC',  
+      },
+    };
+    target[date] = docItem;
+  }
+
+  const isIndexesData = storeKey.includes('_INDEXES_');
+  const isSuppliedData = storeKey.includes('_SUPPLIED_');
+  const isCostsData = storeKey.includes('_COSTS_');
+  const isStatsData = storeKey.includes('_STATS_');
+
+  if (isIndexesData || isSuppliedData) {
+    const [period1, period2] = storeValue as number[];
+    const periodicItemKey = isIndexesData ? 'indexes' : 'supplied';
+    docItem[periodicItemKey] = { period1, period2 };
+  } else if (isCostsData) {
+    docItem.costs = storeValue as number;
+  } else if (isStatsData) {
+    const { apparentPower, instantIntensity } = storeValue as Stats;
+    docItem.statistics = {
+      apparentPower: convertStats(apparentPower),
+      instantIntensity: convertStats(instantIntensity),
+    };
+  }
+
+  // console.log({ dateItem });
+
+}
+
+function groupData(data: StoreEntries): GroupedData {
+  return Object.entries(data).reduce((grouped: GroupedData, [storeKey, storeValue]) => {
+    if (IGNORED_STORE_KEYS_PREFIXES.some((prefix) => storeKey.startsWith(prefix))) {
+      return grouped;
+    }
+
+    const { perDay, perMonth } = grouped;
+    let category: DocumentByDate;
+    if (storeKey.startsWith('DAY')) {
+      category = perDay;
+    } else if (storeKey.startsWith('MONTH')) {
+      category = perMonth;
+    }
+
+    parseData(category, storeKey, storeValue);
+
+    return grouped;
+  }, {
+    perDay: {},
+    perMonth: {},
+  });
 }
 
-async function main(args) {
-  console.log('Loaded configuration:', config);
+async function main(args: string[]) {
+  if (args.length !== 4) {
+    console.error('Arguments: <store json file> <index name>')
+    return;
+  }
 
-  const [datastorePath, dayOrMonth, indexName] = args;
+  const [, , storeFilePath, indexName] = args;
 
-  console.log('Ready!');
+  console.log('> Loaded configuration:', config);
 
   const client = createOpenSearchClient();
 
-  const dataStore = await readDatastore();
+  const dataStore = await readDatastore(storeFilePath);
 
-  Object.entries(dataStore).reduce(acc, ([k, v]) => {
-
-  }, {});
+  console.log('> Ready!', { args });
 
-  console.log('Adding document:');
+  const groupedData = groupData(dataStore.data);
 
-  const document = {
-    title: 'The Outsider',
-    author: 'Stephen King',
-    year: '2018',
-    genre: 'Crime fiction',
-  };
+  // console.log({ groupedData });
 
-  const id = ``;
+  const docPromises = Object.entries(groupedData.perDay)
+    .map(([dateKey, document]) => {
+      console.log('Adding document:', { dateKey });
 
-  var response = await client.index({
-    id: id,
-    index: indexName,
-    body: document,
-    refresh: true,
-  });
+      return client.index({
+        id: `day-${dateKey}`,
+        index: indexName,
+        body: document,
+        refresh: true,
+      });
+    });
+
+  /*const responses = */
+  await Promise.all(docPromises);
 
-  console.log(response.body);
+  // console.log(responses);
 
+  console.log('Done!');
 }
 
 main(process.argv);