Gladys Plus should upload backups in several chunks (#1519)

* Gladys Plus should upload backups in several chunks
* Display Gladys Plus backups in background jobs
* Redirect to background jobs after backup
* Should upload backup only at night from 0 to 2 AM
* Remove backup at startup

front/src/actions/gateway.js

@@ -1,5 +1,6 @@
 import get from 'get-value';
 import update from 'immutability-helper';
+import { route } from 'preact-router';
 import { RequestStatus, LoginStatus } from '../utils/consts';
 import { validateEmail } from '../utils/validator';
 import { ERROR_MESSAGES, SYSTEM_VARIABLE_NAMES } from '../../../server/utils/constants';
@@ -148,15 +149,12 @@ function createActions(store) {
         store.setState({
           gatewayCreateBackupStatus: RequestStatus.Success
         });
-        // we refresh backups
-        setTimeout(() => actions.getBackups(store.getState()), 1000);
-        setTimeout(() => actions.getBackups(store.getState()), 3000);
-        setTimeout(() => actions.getBackups(store.getState()), 8000);
         // we remove the backup status after 1 second
         setTimeout(() => {
           store.setState({
             gatewayCreateBackupStatus: null
           });
+          route('/dashboard/settings/jobs');
         }, 1000);
       } catch (e) {
         store.setState({

front/src/config/i18n/en.json

@@ -1400,7 +1400,8 @@
     "jobTypes": {
       "monthly-device-state-aggregate": "Monthly sensors aggregation",
       "daily-device-state-aggregate": "Daily sensors aggregation",
-      "hourly-device-state-aggregate": "Hourly sensors aggregation"
+      "hourly-device-state-aggregate": "Hourly sensors aggregation",
+      "gladys-gateway-backup": "Gladys Plus backup"
     },
     "jobErrors": {
       "purged-when-restarted": "Gladys Assistant restarted while this job was still running, so it was purged. It doesn't mean the job has failed, it's a normal behavior."

front/src/config/i18n/fr.json

@@ -1400,7 +1400,8 @@
     "jobTypes": {
       "monthly-device-state-aggregate": "Aggrégation donnée capteur mensuelle",
       "daily-device-state-aggregate": "Aggrégation donnée capteur journalière",
-      "hourly-device-state-aggregate": "Aggrégation donnée capteur horaire"
+      "hourly-device-state-aggregate": "Aggrégation donnée capteur horaire",
+      "gladys-gateway-backup": "Sauvegarde Gladys Plus"
     },
     "jobErrors": {
       "purged-when-restarted": "Gladys Assistant a redémarré alors que cette tâche était en cours. Cela ne veut pas dire que cette tâche a échouée, c'est un comportement normal."

server/api/controllers/gateway.controller.js

@@ -83,7 +83,7 @@ module.exports = function GatewayController(gladys) {
    * @apiGroup Gateway
    */
   async function createBackup(req, res) {
-    gladys.event.emit(EVENTS.GATEWAY.CREATE_BACKUP, null);
+    gladys.event.emit(EVENTS.GATEWAY.CREATE_BACKUP);
     res.json({
       success: true,
     });

server/config/scheduler-jobs.js

@@ -1,11 +1,6 @@
 const { EVENTS } = require('../utils/constants');
 
 const jobs = [
-  {
-    name: 'backup-gateway',
-    frequencyInSeconds: 2 * 60 * 60, // we check every 2 hour if needed, but it will backup only once a day
-    event: EVENTS.GATEWAY.CHECK_IF_BACKUP_NEEDED,
-  },
   {
     name: 'check-gladys-upgrade',
     frequencyInSeconds: 6 * 60 * 60,

server/lib/gateway/gateway.backup.js

@@ -1,33 +1,42 @@
 const Sequelize = require('sequelize');
 const path = require('path');
 const fse = require('fs-extra');
-const fs = require('fs');
+const Promise = require('bluebird');
 const fsPromise = require('fs').promises;
-const FormData = require('form-data');
 const retry = require('async-retry');
 const db = require('../../models');
 const logger = require('../../utils/logger');
 const { exec } = require('../../utils/childProcess');
+const { readChunk } = require('../../utils/readChunk');
 const { NotFoundError } = require('../../utils/coreErrors');
 
 const BACKUP_NAME_BASE = 'gladys-db-backup';
 
-const RETRY_OPTIONS = {
+const SQLITE_BACKUP_RETRY_OPTIONS = {
   retries: 3,
   factor: 2,
   minTimeout: 2000,
 };
 
+const UPLOAD_ONE_CHUNK_RETRY_OPTIONS = {
+  retries: 4,
+  factor: 2,
+  minTimeout: 50,
+};
+
 /**
  * @description Create a backup and upload it to the Gateway
+ * @param {string} jobId - The job id.
+ * @returns {Promise} - Resolve when backup is finished.
  * @example
  * backup();
  */
-async function backup() {
+async function backup(jobId) {
   const encryptKey = await this.variable.getValue('GLADYS_GATEWAY_BACKUP_KEY');
   if (encryptKey === null) {
     throw new NotFoundError('GLADYS_GATEWAY_BACKUP_KEY_NOT_FOUND');
   }
+  const systemInfos = await this.system.getInfos();
   const now = new Date();
   const date = `${now.getFullYear()}-${now.getMonth() +
     1}-${now.getDate()}-${now.getHours()}-${now.getMinutes()}-${now.getSeconds()}`;
@@ -51,28 +60,83 @@ async function backup() {
       await exec(`sqlite3 ${this.config.storage} ".backup '${backupFilePath}'"`);
       logger.info(`Gateway backup: Unlocking Database`);
     });
-  }, RETRY_OPTIONS);
+  }, SQLITE_BACKUP_RETRY_OPTIONS);
+  await this.job.updateProgress(jobId, 10);
   const fileInfos = await fsPromise.stat(backupFilePath);
   const fileSizeMB = Math.round(fileInfos.size / 1024 / 1024);
   logger.info(`Gateway backup : Success! File size is ${fileSizeMB}mb.`);
   // compress backup
   logger.info(`Gateway backup: Gzipping backup`);
   await exec(`gzip ${backupFilePath}`);
+  await this.job.updateProgress(jobId, 20);
   // encrypt backup
   logger.info(`Gateway backup: Encrypting backup`);
   await exec(
     `openssl enc -aes-256-cbc -pass pass:${encryptKey} -in ${compressedBackupFilePath} -out ${encryptedBackupFilePath}`,
   );
-  // Read backup file in stream
-  const form = new FormData();
-  form.append('upload', fs.createReadStream(encryptedBackupFilePath));
-  // and upload it to the Gladys Gateway
-  logger.info(`Gateway backup: Uploading backup`);
-  await this.gladysGatewayClient.uploadBackup(form, (progressEvent) => {
-    logger.info(`Gladys Plus backup: Upload backup progress, ${progressEvent.loaded} / ${progressEvent.total}`);
+  await this.job.updateProgress(jobId, 30);
+  // Upload file to the Gladys Gateway
+  const encryptedFileInfos = await fsPromise.stat(encryptedBackupFilePath);
+  logger.info(
+    `Gateway backup: Uploading backup, size of encrypted backup = ${Math.round(
+      encryptedFileInfos.size / 1024 / 1024,
+    )}mb.`,
+  );
+  const initializeBackupResponse = await this.gladysGatewayClient.initializeMultiPartBackup({
+    file_size: encryptedFileInfos.size,
   });
-  // done!
-  logger.info(`Gladys backup uploaded with success to Gladys Gateway. ${backupFileName}`);
+  try {
+    let numberOfchunksUploaded = 0;
+    const totalOfChunksToUpload = initializeBackupResponse.parts.length;
+
+    const partsUploaded = await Promise.map(
+      initializeBackupResponse.parts,
+      async (part, index) => {
+        const startPosition = index * initializeBackupResponse.chunk_size;
+        const chunk = await readChunk(encryptedBackupFilePath, {
+          length: initializeBackupResponse.chunk_size,
+          startPosition,
+        });
+
+        // each chunk is retried
+        return retry(async () => {
+          const { headers } = await this.gladysGatewayClient.uploadOneBackupChunk(
+            part.signed_url,
+            chunk,
+            systemInfos.gladys_version,
+          );
+
+          numberOfchunksUploaded += 1;
+
+          const percent = 30 + ((numberOfchunksUploaded * 100) / totalOfChunksToUpload) * 0.7;
+
+          await this.job.updateProgress(jobId, percent);
+
+          return {
+            PartNumber: part.part_number,
+            ETag: headers.etag.replace(/"/g, ''),
+          };
+        }, UPLOAD_ONE_CHUNK_RETRY_OPTIONS);
+      },
+      { concurrency: 2 },
+    );
+    await this.gladysGatewayClient.finalizeMultiPartBackup({
+      file_key: initializeBackupResponse.file_key,
+      file_id: initializeBackupResponse.file_id,
+      parts: partsUploaded,
+      backup_id: initializeBackupResponse.backup_id,
+    });
+    await this.job.updateProgress(jobId, 100);
+    // done!
+    logger.info(`Gladys backup uploaded with success to Gladys Gateway.`);
+  } catch (e) {
+    await this.gladysGatewayClient.abortMultiPartBackup({
+      file_key: initializeBackupResponse.file_key,
+      file_id: initializeBackupResponse.file_id,
+      backup_id: initializeBackupResponse.backup_id,
+    });
+    throw e;
+  }
   return {
     encryptedBackupFilePath,
   };

server/lib/gateway/gateway.checkIfBackupNeeded.js

@@ -1,4 +1,19 @@
 const logger = require('../../utils/logger');
+const { EVENTS } = require('../../utils/constants');
+
+/**
+ * @description Generate a number between 0 and max included.
+ *
+ * @param {number} max - The upper bound included.
+ * @returns {number} - Return a random value.
+ * @example
+ *
+ * const rand = generateRandomInteger(1000);
+ */
+function generateRandomInteger(max) {
+  return Math.floor(Math.random() * max) + 1;
+}
+
 /**
  * @description Check if a backup is needed
  * @example
@@ -21,8 +36,15 @@ async function checkIfBackupNeeded() {
     }
   }
   if (shouldBackup) {
-    logger.info(`Trying to backup instance to Gladys Gateway`);
-    await this.backup();
+    // To avoid being too brutal with the Gladys Gateway server
+    // we start backups at a random moment
+    // between now (0ms wait) and RAND_INTERVAL_IN_MS wait
+    const randomMoment = generateRandomInteger(this.backupRandomInterval);
+    logger.info(`Backup will be started in ${Math.round(randomMoment / 1000)} seconds`);
+    setTimeout(() => {
+      logger.info(`Starting backup!`);
+      this.event.emit(EVENTS.GATEWAY.CREATE_BACKUP);
+    }, randomMoment);
   } else {
     logger.info(`Not backing up instance to Gladys Gateway, last backup is recent.`);
   }

server/lib/gateway/gateway.downloadBackup.js

@@ -20,7 +20,8 @@ async function downloadBackup(fileUrl) {
     throw new NotFoundError('GLADYS_GATEWAY_BACKUP_KEY_NOT_FOUND');
   }
   // extract file name
-  const encryptedBackupName = path.basename(fileUrl, '.enc');
+  const fileWithoutSignedParams = fileUrl.split('?')[0];
+  const encryptedBackupName = path.basename(fileWithoutSignedParams, '.enc');
   const restoreFolderPath = path.join(this.config.backupsFolder, RESTORE_FOLDER);
   const encryptedBackupFilePath = path.join(restoreFolderPath, `${encryptedBackupName}.enc`);
   const compressedBackupFilePath = path.join(restoreFolderPath, `${encryptedBackupName}.db.gz`);

server/lib/gateway/gateway.init.js

@@ -1,5 +1,5 @@
 const logger = require('../../utils/logger');
-const { EVENTS, SYSTEM_VARIABLE_NAMES } = require('../../utils/constants');
+const { SYSTEM_VARIABLE_NAMES } = require('../../utils/constants');
 
 /**
  * @description Init Gladys Gateway.
@@ -24,8 +24,6 @@ async function init() {
         this.handleNewMessage.bind(this),
       );
       this.connected = true;
-      // try to backup, if needed
-      this.event.emit(EVENTS.GATEWAY.CHECK_IF_BACKUP_NEEDED);
 
       // check if google home is connected
       const value = await this.variable.getValue(
@@ -40,6 +38,20 @@ async function init() {
     this.connected = false;
   }
 
+  if (this.backupSchedule && this.backupSchedule.cancel) {
+    this.backupSchedule.cancel();
+  }
+  // schedule backup at midnight
+  const timezone = await this.variable.getValue(SYSTEM_VARIABLE_NAMES.TIMEZONE);
+
+  const rule = new this.schedule.RecurrenceRule();
+  rule.tz = timezone;
+  rule.hour = 0;
+  rule.minute = 0;
+  rule.second = 0;
+
+  this.backupSchedule = this.schedule.scheduleJob(rule, this.checkIfBackupNeeded.bind(this));
+
   if (process.env.NODE_ENV === 'production') {
     try {
       await this.getLatestGladysVersion();

server/lib/gateway/index.js

@@ -1,8 +1,10 @@
 const GladysGatewayClient = require('@gladysassistant/gladys-gateway-js');
 const WebCrypto = require('node-webcrypto-ossl');
+const schedule = require('node-schedule');
+
 const getConfig = require('../../utils/getConfig');
 const logger = require('../../utils/logger');
-const { EVENTS } = require('../../utils/constants');
+const { EVENTS, JOB_TYPES } = require('../../utils/constants');
 const { eventFunctionWrapper } = require('../../utils/functionsWrapper');
 
 const serverUrl = getConfig().gladysGatewayServerUrl;
@@ -29,23 +31,28 @@ const { restoreBackupEvent } = require('./gateway.restoreBackupEvent');
 const { saveUsersKeys } = require('./gateway.saveUsersKeys');
 const { refreshUserKeys } = require('./gateway.refreshUserKeys');
 
-const Gateway = function Gateway(variable, event, system, sequelize, config, user, stateManager, serviceManager) {
+const Gateway = function Gateway(variable, event, system, sequelize, config, user, stateManager, serviceManager, job) {
   this.variable = variable;
   this.event = event;
   this.system = system;
   this.sequelize = sequelize;
+  this.schedule = schedule;
   this.config = config;
   this.user = user;
   this.stateManager = stateManager;
   this.serviceManager = serviceManager;
+  this.job = job;
   this.connected = false;
   this.restoreInProgress = false;
   this.usersKeys = [];
   this.googleHomeConnected = false;
   this.forwardStateToGoogleHomeTimeouts = new Map();
   this.googleHomeForwardStateTimeout = 5 * 1000;
+  this.backupRandomInterval = 2 * 60 * 60 * 1000; // 2 hours
   this.GladysGatewayClient = GladysGatewayClient;
   this.gladysGatewayClient = new GladysGatewayClient({ cryptoLib, serverUrl, logger });
+  this.backup = this.job.wrapper(JOB_TYPES.GLADYS_GATEWAY_BACKUP, this.backup.bind(this));
+
   this.event.on(EVENTS.GATEWAY.CREATE_BACKUP, eventFunctionWrapper(this.backup.bind(this)));
   this.event.on(EVENTS.GATEWAY.CHECK_IF_BACKUP_NEEDED, eventFunctionWrapper(this.checkIfBackupNeeded.bind(this)));
   this.event.on(EVENTS.GATEWAY.RESTORE_BACKUP, eventFunctionWrapper(this.restoreBackupEvent.bind(this)));

server/lib/index.js

@@ -69,7 +69,7 @@ function Gladys(params = {}) {
   const scene = new Scene(stateManager, event, device, message, variable, house, calendar, http);
   const scheduler = new Scheduler(event);
   const weather = new Weather(service, event, message, house);
-  const gateway = new Gateway(variable, event, system, db.sequelize, config, user, stateManager, service);
+  const gateway = new Gateway(variable, event, system, db.sequelize, config, user, stateManager, service, job);
 
   const gladys = {
     version: '0.1.0', // todo, read package.json