diff --git a/front/src/actions/gateway.js b/front/src/actions/gateway.js index c6d339b742..fbd3e5aa3f 100644 --- a/front/src/actions/gateway.js +++ b/front/src/actions/gateway.js @@ -1,5 +1,6 @@ import get from 'get-value'; import update from 'immutability-helper'; +import { route } from 'preact-router'; import { RequestStatus, LoginStatus } from '../utils/consts'; import { validateEmail } from '../utils/validator'; import { ERROR_MESSAGES, SYSTEM_VARIABLE_NAMES } from '../../../server/utils/constants'; @@ -148,15 +149,12 @@ function createActions(store) { store.setState({ gatewayCreateBackupStatus: RequestStatus.Success }); - // we refresh backups - setTimeout(() => actions.getBackups(store.getState()), 1000); - setTimeout(() => actions.getBackups(store.getState()), 3000); - setTimeout(() => actions.getBackups(store.getState()), 8000); // we remove the backup status after 1 second setTimeout(() => { store.setState({ gatewayCreateBackupStatus: null }); + route('/dashboard/settings/jobs'); }, 1000); } catch (e) { store.setState({ diff --git a/front/src/config/i18n/en.json b/front/src/config/i18n/en.json index c07a3e54e3..dec46a103d 100644 --- a/front/src/config/i18n/en.json +++ b/front/src/config/i18n/en.json @@ -1400,7 +1400,8 @@ "jobTypes": { "monthly-device-state-aggregate": "Monthly sensors aggregation", "daily-device-state-aggregate": "Daily sensors aggregation", - "hourly-device-state-aggregate": "Hourly sensors aggregation" + "hourly-device-state-aggregate": "Hourly sensors aggregation", + "gladys-gateway-backup": "Gladys Plus backup" }, "jobErrors": { "purged-when-restarted": "Gladys Assistant restarted while this job was still running, so it was purged. It doesn't mean the job has failed, it's a normal behavior." diff --git a/front/src/config/i18n/fr.json b/front/src/config/i18n/fr.json index 7973904acb..d0334a034c 100644 --- a/front/src/config/i18n/fr.json +++ b/front/src/config/i18n/fr.json @@ -1400,7 +1400,8 @@ "jobTypes": { "monthly-device-state-aggregate": "Aggrégation donnée capteur mensuelle", "daily-device-state-aggregate": "Aggrégation donnée capteur journalière", - "hourly-device-state-aggregate": "Aggrégation donnée capteur horaire" + "hourly-device-state-aggregate": "Aggrégation donnée capteur horaire", + "gladys-gateway-backup": "Sauvegarde Gladys Plus" }, "jobErrors": { "purged-when-restarted": "Gladys Assistant a redémarré alors que cette tâche était en cours. Cela ne veut pas dire que cette tâche a échouée, c'est un comportement normal." diff --git a/server/api/controllers/gateway.controller.js b/server/api/controllers/gateway.controller.js index 759cd69fc9..ad6dbcf168 100644 --- a/server/api/controllers/gateway.controller.js +++ b/server/api/controllers/gateway.controller.js @@ -83,7 +83,7 @@ module.exports = function GatewayController(gladys) { * @apiGroup Gateway */ async function createBackup(req, res) { - gladys.event.emit(EVENTS.GATEWAY.CREATE_BACKUP, null); + gladys.event.emit(EVENTS.GATEWAY.CREATE_BACKUP); res.json({ success: true, }); diff --git a/server/config/scheduler-jobs.js b/server/config/scheduler-jobs.js index fcb7308e8e..ab479a8453 100644 --- a/server/config/scheduler-jobs.js +++ b/server/config/scheduler-jobs.js @@ -1,11 +1,6 @@ const { EVENTS } = require('../utils/constants'); const jobs = [ - { - name: 'backup-gateway', - frequencyInSeconds: 2 * 60 * 60, // we check every 2 hour if needed, but it will backup only once a day - event: EVENTS.GATEWAY.CHECK_IF_BACKUP_NEEDED, - }, { name: 'check-gladys-upgrade', frequencyInSeconds: 6 * 60 * 60, diff --git a/server/lib/gateway/gateway.backup.js b/server/lib/gateway/gateway.backup.js index 9a537275ab..fa95104007 100644 --- a/server/lib/gateway/gateway.backup.js +++ b/server/lib/gateway/gateway.backup.js @@ -1,33 +1,42 @@ const Sequelize = require('sequelize'); const path = require('path'); const fse = require('fs-extra'); -const fs = require('fs'); +const Promise = require('bluebird'); const fsPromise = require('fs').promises; -const FormData = require('form-data'); const retry = require('async-retry'); const db = require('../../models'); const logger = require('../../utils/logger'); const { exec } = require('../../utils/childProcess'); +const { readChunk } = require('../../utils/readChunk'); const { NotFoundError } = require('../../utils/coreErrors'); const BACKUP_NAME_BASE = 'gladys-db-backup'; -const RETRY_OPTIONS = { +const SQLITE_BACKUP_RETRY_OPTIONS = { retries: 3, factor: 2, minTimeout: 2000, }; +const UPLOAD_ONE_CHUNK_RETRY_OPTIONS = { + retries: 4, + factor: 2, + minTimeout: 50, +}; + /** * @description Create a backup and upload it to the Gateway + * @param {string} jobId - The job id. + * @returns {Promise} - Resolve when backup is finished. * @example * backup(); */ -async function backup() { +async function backup(jobId) { const encryptKey = await this.variable.getValue('GLADYS_GATEWAY_BACKUP_KEY'); if (encryptKey === null) { throw new NotFoundError('GLADYS_GATEWAY_BACKUP_KEY_NOT_FOUND'); } + const systemInfos = await this.system.getInfos(); const now = new Date(); const date = `${now.getFullYear()}-${now.getMonth() + 1}-${now.getDate()}-${now.getHours()}-${now.getMinutes()}-${now.getSeconds()}`; @@ -51,28 +60,83 @@ async function backup() { await exec(`sqlite3 ${this.config.storage} ".backup '${backupFilePath}'"`); logger.info(`Gateway backup: Unlocking Database`); }); - }, RETRY_OPTIONS); + }, SQLITE_BACKUP_RETRY_OPTIONS); + await this.job.updateProgress(jobId, 10); const fileInfos = await fsPromise.stat(backupFilePath); const fileSizeMB = Math.round(fileInfos.size / 1024 / 1024); logger.info(`Gateway backup : Success! File size is ${fileSizeMB}mb.`); // compress backup logger.info(`Gateway backup: Gzipping backup`); await exec(`gzip ${backupFilePath}`); + await this.job.updateProgress(jobId, 20); // encrypt backup logger.info(`Gateway backup: Encrypting backup`); await exec( `openssl enc -aes-256-cbc -pass pass:${encryptKey} -in ${compressedBackupFilePath} -out ${encryptedBackupFilePath}`, ); - // Read backup file in stream - const form = new FormData(); - form.append('upload', fs.createReadStream(encryptedBackupFilePath)); - // and upload it to the Gladys Gateway - logger.info(`Gateway backup: Uploading backup`); - await this.gladysGatewayClient.uploadBackup(form, (progressEvent) => { - logger.info(`Gladys Plus backup: Upload backup progress, ${progressEvent.loaded} / ${progressEvent.total}`); + await this.job.updateProgress(jobId, 30); + // Upload file to the Gladys Gateway + const encryptedFileInfos = await fsPromise.stat(encryptedBackupFilePath); + logger.info( + `Gateway backup: Uploading backup, size of encrypted backup = ${Math.round( + encryptedFileInfos.size / 1024 / 1024, + )}mb.`, + ); + const initializeBackupResponse = await this.gladysGatewayClient.initializeMultiPartBackup({ + file_size: encryptedFileInfos.size, }); - // done! - logger.info(`Gladys backup uploaded with success to Gladys Gateway. ${backupFileName}`); + try { + let numberOfchunksUploaded = 0; + const totalOfChunksToUpload = initializeBackupResponse.parts.length; + + const partsUploaded = await Promise.map( + initializeBackupResponse.parts, + async (part, index) => { + const startPosition = index * initializeBackupResponse.chunk_size; + const chunk = await readChunk(encryptedBackupFilePath, { + length: initializeBackupResponse.chunk_size, + startPosition, + }); + + // each chunk is retried + return retry(async () => { + const { headers } = await this.gladysGatewayClient.uploadOneBackupChunk( + part.signed_url, + chunk, + systemInfos.gladys_version, + ); + + numberOfchunksUploaded += 1; + + const percent = 30 + ((numberOfchunksUploaded * 100) / totalOfChunksToUpload) * 0.7; + + await this.job.updateProgress(jobId, percent); + + return { + PartNumber: part.part_number, + ETag: headers.etag.replace(/"/g, ''), + }; + }, UPLOAD_ONE_CHUNK_RETRY_OPTIONS); + }, + { concurrency: 2 }, + ); + await this.gladysGatewayClient.finalizeMultiPartBackup({ + file_key: initializeBackupResponse.file_key, + file_id: initializeBackupResponse.file_id, + parts: partsUploaded, + backup_id: initializeBackupResponse.backup_id, + }); + await this.job.updateProgress(jobId, 100); + // done! + logger.info(`Gladys backup uploaded with success to Gladys Gateway.`); + } catch (e) { + await this.gladysGatewayClient.abortMultiPartBackup({ + file_key: initializeBackupResponse.file_key, + file_id: initializeBackupResponse.file_id, + backup_id: initializeBackupResponse.backup_id, + }); + throw e; + } return { encryptedBackupFilePath, }; diff --git a/server/lib/gateway/gateway.checkIfBackupNeeded.js b/server/lib/gateway/gateway.checkIfBackupNeeded.js index b747d048a1..c883641aa9 100644 --- a/server/lib/gateway/gateway.checkIfBackupNeeded.js +++ b/server/lib/gateway/gateway.checkIfBackupNeeded.js @@ -1,4 +1,19 @@ const logger = require('../../utils/logger'); +const { EVENTS } = require('../../utils/constants'); + +/** + * @description Generate a number between 0 and max included. + * + * @param {number} max - The upper bound included. + * @returns {number} - Return a random value. + * @example + * + * const rand = generateRandomInteger(1000); + */ +function generateRandomInteger(max) { + return Math.floor(Math.random() * max) + 1; +} + /** * @description Check if a backup is needed * @example @@ -21,8 +36,15 @@ async function checkIfBackupNeeded() { } } if (shouldBackup) { - logger.info(`Trying to backup instance to Gladys Gateway`); - await this.backup(); + // To avoid being too brutal with the Gladys Gateway server + // we start backups at a random moment + // between now (0ms wait) and RAND_INTERVAL_IN_MS wait + const randomMoment = generateRandomInteger(this.backupRandomInterval); + logger.info(`Backup will be started in ${Math.round(randomMoment / 1000)} seconds`); + setTimeout(() => { + logger.info(`Starting backup!`); + this.event.emit(EVENTS.GATEWAY.CREATE_BACKUP); + }, randomMoment); } else { logger.info(`Not backing up instance to Gladys Gateway, last backup is recent.`); } diff --git a/server/lib/gateway/gateway.downloadBackup.js b/server/lib/gateway/gateway.downloadBackup.js index c3b7dec5fa..aa71138fee 100644 --- a/server/lib/gateway/gateway.downloadBackup.js +++ b/server/lib/gateway/gateway.downloadBackup.js @@ -20,7 +20,8 @@ async function downloadBackup(fileUrl) { throw new NotFoundError('GLADYS_GATEWAY_BACKUP_KEY_NOT_FOUND'); } // extract file name - const encryptedBackupName = path.basename(fileUrl, '.enc'); + const fileWithoutSignedParams = fileUrl.split('?')[0]; + const encryptedBackupName = path.basename(fileWithoutSignedParams, '.enc'); const restoreFolderPath = path.join(this.config.backupsFolder, RESTORE_FOLDER); const encryptedBackupFilePath = path.join(restoreFolderPath, `${encryptedBackupName}.enc`); const compressedBackupFilePath = path.join(restoreFolderPath, `${encryptedBackupName}.db.gz`); diff --git a/server/lib/gateway/gateway.init.js b/server/lib/gateway/gateway.init.js index 0dfc02eb72..c53dd28c4b 100644 --- a/server/lib/gateway/gateway.init.js +++ b/server/lib/gateway/gateway.init.js @@ -1,5 +1,5 @@ const logger = require('../../utils/logger'); -const { EVENTS, SYSTEM_VARIABLE_NAMES } = require('../../utils/constants'); +const { SYSTEM_VARIABLE_NAMES } = require('../../utils/constants'); /** * @description Init Gladys Gateway. @@ -24,8 +24,6 @@ async function init() { this.handleNewMessage.bind(this), ); this.connected = true; - // try to backup, if needed - this.event.emit(EVENTS.GATEWAY.CHECK_IF_BACKUP_NEEDED); // check if google home is connected const value = await this.variable.getValue( @@ -40,6 +38,20 @@ async function init() { this.connected = false; } + if (this.backupSchedule && this.backupSchedule.cancel) { + this.backupSchedule.cancel(); + } + // schedule backup at midnight + const timezone = await this.variable.getValue(SYSTEM_VARIABLE_NAMES.TIMEZONE); + + const rule = new this.schedule.RecurrenceRule(); + rule.tz = timezone; + rule.hour = 0; + rule.minute = 0; + rule.second = 0; + + this.backupSchedule = this.schedule.scheduleJob(rule, this.checkIfBackupNeeded.bind(this)); + if (process.env.NODE_ENV === 'production') { try { await this.getLatestGladysVersion(); diff --git a/server/lib/gateway/index.js b/server/lib/gateway/index.js index 12f2c07933..4d67b54624 100644 --- a/server/lib/gateway/index.js +++ b/server/lib/gateway/index.js @@ -1,8 +1,10 @@ const GladysGatewayClient = require('@gladysassistant/gladys-gateway-js'); const WebCrypto = require('node-webcrypto-ossl'); +const schedule = require('node-schedule'); + const getConfig = require('../../utils/getConfig'); const logger = require('../../utils/logger'); -const { EVENTS } = require('../../utils/constants'); +const { EVENTS, JOB_TYPES } = require('../../utils/constants'); const { eventFunctionWrapper } = require('../../utils/functionsWrapper'); const serverUrl = getConfig().gladysGatewayServerUrl; @@ -29,23 +31,28 @@ const { restoreBackupEvent } = require('./gateway.restoreBackupEvent'); const { saveUsersKeys } = require('./gateway.saveUsersKeys'); const { refreshUserKeys } = require('./gateway.refreshUserKeys'); -const Gateway = function Gateway(variable, event, system, sequelize, config, user, stateManager, serviceManager) { +const Gateway = function Gateway(variable, event, system, sequelize, config, user, stateManager, serviceManager, job) { this.variable = variable; this.event = event; this.system = system; this.sequelize = sequelize; + this.schedule = schedule; this.config = config; this.user = user; this.stateManager = stateManager; this.serviceManager = serviceManager; + this.job = job; this.connected = false; this.restoreInProgress = false; this.usersKeys = []; this.googleHomeConnected = false; this.forwardStateToGoogleHomeTimeouts = new Map(); this.googleHomeForwardStateTimeout = 5 * 1000; + this.backupRandomInterval = 2 * 60 * 60 * 1000; // 2 hours this.GladysGatewayClient = GladysGatewayClient; this.gladysGatewayClient = new GladysGatewayClient({ cryptoLib, serverUrl, logger }); + this.backup = this.job.wrapper(JOB_TYPES.GLADYS_GATEWAY_BACKUP, this.backup.bind(this)); + this.event.on(EVENTS.GATEWAY.CREATE_BACKUP, eventFunctionWrapper(this.backup.bind(this))); this.event.on(EVENTS.GATEWAY.CHECK_IF_BACKUP_NEEDED, eventFunctionWrapper(this.checkIfBackupNeeded.bind(this))); this.event.on(EVENTS.GATEWAY.RESTORE_BACKUP, eventFunctionWrapper(this.restoreBackupEvent.bind(this))); diff --git a/server/lib/index.js b/server/lib/index.js index 95c20f156a..33914a7e8f 100644 --- a/server/lib/index.js +++ b/server/lib/index.js @@ -69,7 +69,7 @@ function Gladys(params = {}) { const scene = new Scene(stateManager, event, device, message, variable, house, calendar, http); const scheduler = new Scheduler(event); const weather = new Weather(service, event, message, house); - const gateway = new Gateway(variable, event, system, db.sequelize, config, user, stateManager, service); + const gateway = new Gateway(variable, event, system, db.sequelize, config, user, stateManager, service, job); const gladys = { version: '0.1.0', // todo, read package.json diff --git a/server/package-lock.json b/server/package-lock.json index 2b6a024ac2..fd90475f8a 100644 --- a/server/package-lock.json +++ b/server/package-lock.json @@ -870,9 +870,9 @@ } }, "@gladysassistant/gladys-gateway-js": { - "version": "3.9.0", - "resolved": "https://registry.npmjs.org/@gladysassistant/gladys-gateway-js/-/gladys-gateway-js-3.9.0.tgz", - "integrity": "sha512-zAIrc48rAS32Yqf1jkaVGZaXWy5EpUzDP/dPRusoOyIYJ9j2Wm1nwlotqq/m9nf4uINqzfLhiYJ5PH3cTPF3kg==", + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/@gladysassistant/gladys-gateway-js/-/gladys-gateway-js-4.1.1.tgz", + "integrity": "sha512-9cwkdp0/G/3vTpw/huTeCIdo5IxL6mslqT8jV82s/opJxGPFWcRx3Dt+GAUXpmHEsVYqIXCWCY8372ZaR32LCw==", "requires": { "@ctrlpanel/pbkdf2": "^1.0.0", "array-buffer-to-hex": "^1.0.0", @@ -1879,14 +1879,14 @@ "integrity": "sha512-UxDxWn7dl97rKVeVS61vErvw086aCYhDLyvRQZ5Rk65rZKepaFdm53GeqXaKBuOhED4e9uWq34IC3TdSdJJ2Gw==" }, "@types/prop-types": { - "version": "15.7.4", - "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.4.tgz", - "integrity": "sha512-rZ5drC/jWjrArrS8BR6SIr4cWpW09RNTYt9AMZo3Jwwif+iacXAqgVjm0B0Bv/S1jhDXKHqRVNCbACkJ89RAnQ==" + "version": "15.7.5", + "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.5.tgz", + "integrity": "sha512-JCB8C6SnDoQf0cNycqd/35A7MjcnK+ZTqE7judS6o7utxUCg6imJg3QK2qzHKszlTjcj2cn+NwMB2i96ubpj7w==" }, "@types/react": { - "version": "16.14.24", - "resolved": "https://registry.npmjs.org/@types/react/-/react-16.14.24.tgz", - "integrity": "sha512-e7U2WC8XQP/xfR7bwhOhNFZKPTfW1ph+MiqtudKb8tSV8RyCsovQx2sNVtKoOryjxFKpHPPC/yNiGfdeVM5Gyw==", + "version": "16.14.26", + "resolved": "https://registry.npmjs.org/@types/react/-/react-16.14.26.tgz", + "integrity": "sha512-c/5CYyciOO4XdFcNhZW1O2woVx86k4T+DO2RorHZL7EhitkNQgSD/SgpdZJAUJa/qjVgOmTM44gHkAdZSXeQuQ==", "requires": { "@types/prop-types": "*", "@types/scheduler": "*", @@ -3072,7 +3072,8 @@ "dependencies": { "ansi-regex": { "version": "5.0.0", - "resolved": "", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", "dev": true }, "ansi-styles": { @@ -6704,6 +6705,14 @@ "parse-json": "^2.2.0", "pify": "^2.0.0", "strip-bom": "^3.0.0" + }, + "dependencies": { + "pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", + "dev": true + } } }, "loader-runner": { @@ -7150,7 +7159,8 @@ "dependencies": { "ansi-regex": { "version": "5.0.0", - "resolved": "", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", "dev": true }, "ansi-styles": { @@ -7972,7 +7982,8 @@ "dependencies": { "ansi-regex": { "version": "5.0.0", - "resolved": "", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", "dev": true }, "ansi-styles": { @@ -8761,6 +8772,14 @@ "dev": true, "requires": { "pify": "^2.0.0" + }, + "dependencies": { + "pify": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", + "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", + "dev": true + } } }, "pathval": { @@ -8799,10 +8818,9 @@ "dev": true }, "pify": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/pify/-/pify-2.3.0.tgz", - "integrity": "sha1-7RQaasBDqEnqWISY59yosVMw6Qw=", - "dev": true + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/pify/-/pify-5.0.0.tgz", + "integrity": "sha512-eW/gHNMlxdSP6dmG6uJip6FXN0EQBwm2clYYd8Wul42Cwu/DK8HEftzsapcNdYe2MfLiIwZqsDk2RDEsTE79hA==" }, "pkg-dir": { "version": "4.2.0", @@ -10062,7 +10080,8 @@ }, "ansi-regex": { "version": "5.0.0", - "resolved": "", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", "dev": true }, "fast-deep-equal": { @@ -10912,7 +10931,8 @@ "dependencies": { "ansi-regex": { "version": "5.0.0", - "resolved": "", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", "dev": true }, "is-fullwidth-code-point": { diff --git a/server/package.json b/server/package.json index 3608f385a1..af6de2dd2b 100644 --- a/server/package.json +++ b/server/package.json @@ -73,7 +73,7 @@ "supertest": "^3.4.2" }, "dependencies": { - "@gladysassistant/gladys-gateway-js": "^3.9.0", + "@gladysassistant/gladys-gateway-js": "^4.1.1", "@hapi/joi": "^17.1.0", "@hapi/joi-date": "^2.0.1", "async-retry": "^1.3.3", @@ -99,6 +99,7 @@ "node-schedule": "^1.3.2", "node-webcrypto-ossl": "^1.0.48", "path-to-regexp": "^3.0.0", + "pify": "^5.0.0", "queue": "^6.0.0", "semver": "^6.1.1", "sequelize": "^6.18.0", diff --git a/server/test/lib/gateway/GladysGatewayClientMock.test.js b/server/test/lib/gateway/GladysGatewayClientMock.test.js index 03ed9ae037..4738dd74f1 100644 --- a/server/test/lib/gateway/GladysGatewayClientMock.test.js +++ b/server/test/lib/gateway/GladysGatewayClientMock.test.js @@ -18,8 +18,24 @@ const GladysGatewayClientMock = function GladysGatewayClientMock() { ecdsaPublicKeyJwk: {}, }), instanceConnect: fake.resolves({}), - uploadBackup: fake.resolves({ - success: true, + initializeMultiPartBackup: fake.resolves({ + file_id: 'c1075a77-0553-495f-8646-116c599f13bc', + file_key: 'b838be4b-c399-495b-ae0b-23247e71743c.enc', + backup_id: 'b5f5e38e-d0e6-469f-bee9-b2ae54f731c7', + chunk_size: 20 * 1024 * 1024, + parts: [ + { + part_number: 1, + signed_url: 'https://test.test.com', + }, + ], + }), + finalizeMultiPartBackup: fake.resolves({ backup: {}, signed_url: 'https://test.com' }), + abortMultiPartBackup: fake.resolves({ backup: {} }), + uploadOneBackupChunk: fake.resolves({ + headers: { + etag: '"this-is-the-etag"', + }, }), getBackups: fake.resolves([ { diff --git a/server/test/lib/gateway/gateway.test.js b/server/test/lib/gateway/gateway.test.js index d96383c7f4..0cdd545f68 100644 --- a/server/test/lib/gateway/gateway.test.js +++ b/server/test/lib/gateway/gateway.test.js @@ -15,7 +15,7 @@ const Gateway = proxyquire('../../../lib/gateway', { }); const getConfig = require('../../../utils/getConfig'); -const { EVENTS } = require('../../../utils/constants'); +const { EVENTS, SYSTEM_VARIABLE_NAMES } = require('../../../utils/constants'); const sequelize = { close: fake.resolves(null), @@ -34,13 +34,22 @@ const system = { const config = getConfig(); +const job = { + wrapper: (type, func) => { + return async () => { + return func(); + }; + }, + updateProgress: fake.resolves({}), +}; + describe('gateway', () => { describe('gateway.login', () => { const variable = { getValue: fake.resolves(null), setValue: fake.resolves(null), }; - const gateway = new Gateway(variable, event, system, sequelize, config); + const gateway = new Gateway(variable, event, system, sequelize, config, {}, {}, {}, job); it('should login to gladys gateway', async () => { const loginResults = await gateway.login('tony.stark@gladysassistant.com', 'warmachine123'); expect(loginResults).to.have.property('two_factor_token'); @@ -67,14 +76,20 @@ describe('gateway', () => { }, ]; const variable = { - getValue: fake.resolves(JSON.stringify(userKeys)), + getValue: (name) => { + if (name === SYSTEM_VARIABLE_NAMES.TIMEZONE) { + return 'Europe/Paris'; + } + return JSON.stringify(userKeys); + }, setValue: fake.resolves(null), }; - const gateway = new Gateway(variable, event, system, sequelize, config); + const gateway = new Gateway(variable, event, system, sequelize, config, {}, {}, {}, job); it('should login two factor to gladys gateway', async () => { await gateway.init(); expect(gateway.connected).to.equal(true); expect(gateway.usersKeys).to.deep.equal(userKeys); + expect(gateway.backupSchedule).to.not.equal(undefined); }); }); @@ -85,17 +100,32 @@ describe('gateway', () => { getValue: fake.resolves('key'), setValue: fake.resolves(null), }; - const gateway = new Gateway(variable, event, system, sequelize, config); + const gateway = new Gateway(variable, event, system, sequelize, config, {}, {}, {}, job); await gateway.login('tony.stark@gladysassistant.com', 'warmachine123'); await gateway.backup(); - assert.calledOnce(gateway.gladysGatewayClient.uploadBackup); + assert.calledOnce(gateway.gladysGatewayClient.initializeMultiPartBackup); + assert.calledOnce(gateway.gladysGatewayClient.uploadOneBackupChunk); + assert.calledOnce(gateway.gladysGatewayClient.finalizeMultiPartBackup); + }); + it('should start and abort backup', async () => { + const variable = { + getValue: fake.resolves('key'), + setValue: fake.resolves(null), + }; + const gateway = new Gateway(variable, event, system, sequelize, config, {}, {}, {}, job); + await gateway.login('tony.stark@gladysassistant.com', 'warmachine123'); + gateway.gladysGatewayClient.uploadOneBackupChunk = fake.rejects('error'); + const promise = gateway.backup(); + await assertChai.isRejected(promise); + assert.calledOnce(gateway.gladysGatewayClient.initializeMultiPartBackup); + assert.calledOnce(gateway.gladysGatewayClient.abortMultiPartBackup); }); it('should backup gladys with lots of insert at the same time', async function Test() { const variable = { getValue: fake.resolves('key'), setValue: fake.resolves(null), }; - const gateway = new Gateway(variable, event, system, sequelize, config); + const gateway = new Gateway(variable, event, system, sequelize, config, {}, {}, {}, job); const promisesDevices = []; const promises = []; await gateway.login('tony.stark@gladysassistant.com', 'warmachine123'); @@ -121,28 +151,16 @@ describe('gateway', () => { } await Promise.all(promisesDevices); await Promise.all(promises); - assert.calledOnce(gateway.gladysGatewayClient.uploadBackup); - }); - - it('should backup gladys and test progress bar', async () => { - const variable = { - getValue: fake.resolves('key'), - setValue: fake.resolves(null), - }; - const gateway = new Gateway(variable, event, system, sequelize, config); - await gateway.login('tony.stark@gladysassistant.com', 'warmachine123'); - gateway.gladysGatewayClient.uploadBackup = async (form, func) => { - func({ loaded: 1, total: 100 }); - await Promise.delay(100); - }; - await gateway.backup(); + assert.calledOnce(gateway.gladysGatewayClient.initializeMultiPartBackup); + assert.calledOnce(gateway.gladysGatewayClient.uploadOneBackupChunk); + assert.calledOnce(gateway.gladysGatewayClient.finalizeMultiPartBackup); }); it('should not backup, no backup key found', async () => { const variable = { getValue: fake.resolves(null), setValue: fake.resolves(null), }; - const gateway = new Gateway(variable, event, system, sequelize, config); + const gateway = new Gateway(variable, event, system, sequelize, config, {}, {}, {}, job); await gateway.login('tony.stark@gladysassistant.com', 'warmachine123'); const promise = gateway.backup(); return assertChai.isRejected(promise, 'GLADYS_GATEWAY_BACKUP_KEY_NOT_FOUND'); @@ -152,7 +170,7 @@ describe('gateway', () => { getValue: fake.resolves('key'), setValue: fake.resolves(null), }; - const gateway = new Gateway(variable, event, system, sequelize, config); + const gateway = new Gateway(variable, event, system, sequelize, config, {}, {}, {}, job); await gateway.login('tony.stark@gladysassistant.com', 'warmachine123'); const backups = await gateway.getBackups(); expect(backups).to.deep.equal([ @@ -169,13 +187,40 @@ describe('gateway', () => { }); }); + describe('gateway.checkIfBackupNeeded', async function Describe() { + this.timeout(20000); + it('should check if backup is needed and execute backup', async () => { + const variable = { + getValue: fake.resolves('key'), + setValue: fake.resolves(null), + }; + const eventFake = { + emit: fake.returns(null), + on: fake.returns(null), + }; + const gateway = new Gateway(variable, eventFake, system, sequelize, config, {}, {}, {}, job); + gateway.backupRandomInterval = 10; + await gateway.login('tony.stark@gladysassistant.com', 'warmachine123'); + gateway.connected = true; + await gateway.checkIfBackupNeeded(); + assert.calledOnce(gateway.gladysGatewayClient.getBackups); + // wait 50ms and see if backup was called + await new Promise((resolve) => { + setTimeout(() => { + assert.calledOnce(eventFake.emit); + resolve(); + }, 50); + }); + }); + }); + describe('gateway.downloadBackup', () => { it('should restore a backup', async () => { const variable = { getValue: fake.resolves('key'), setValue: fake.resolves(null), }; - const gateway = new Gateway(variable, event, system, sequelize, config); + const gateway = new Gateway(variable, event, system, sequelize, config, {}, {}, {}, job); await gateway.login('tony.stark@gladysassistant.com', 'warmachine123'); const { encryptedBackupFilePath } = await gateway.backup(); const { backupFilePath } = await gateway.downloadBackup(encryptedBackupFilePath); @@ -187,7 +232,7 @@ describe('gateway', () => { getValue: fake.resolves('key'), setValue: fake.resolves(null), }; - const gateway = new Gateway(variable, event, system, sequelize, config); + const gateway = new Gateway(variable, event, system, sequelize, config, {}, {}, {}, job); await gateway.login('tony.stark@gladysassistant.com', 'warmachine123'); await gateway.restoreBackupEvent('this-path-does-not-exist'); }); @@ -196,7 +241,7 @@ describe('gateway', () => { getValue: fake.resolves('key'), setValue: fake.resolves(null), }; - const gateway = new Gateway(variable, event, system, sequelize, config); + const gateway = new Gateway(variable, event, system, sequelize, config, {}, {}, {}, job); await gateway.login('tony.stark@gladysassistant.com', 'warmachine123'); const emptyFile = path.join(__dirname, 'this_file_is_not_a_valid_db.dbfile'); const promise = gateway.restoreBackup(emptyFile); @@ -207,7 +252,7 @@ describe('gateway', () => { getValue: fake.resolves('key'), setValue: fake.resolves(null), }; - const gateway = new Gateway(variable, event, system, sequelize, config); + const gateway = new Gateway(variable, event, system, sequelize, config, {}, {}, {}, job); await gateway.login('tony.stark@gladysassistant.com', 'warmachine123'); const emptyFile = path.join(__dirname, 'this_file_has_no_user_table.dbfile'); const promise = gateway.restoreBackup(emptyFile); @@ -218,7 +263,7 @@ describe('gateway', () => { getValue: fake.resolves('key'), setValue: fake.resolves(null), }; - const gateway = new Gateway(variable, event, system, sequelize, config); + const gateway = new Gateway(variable, event, system, sequelize, config, {}, {}, {}, job); await gateway.login('tony.stark@gladysassistant.com', 'warmachine123'); const emptyFile = path.join(__dirname, 'this_db_has_no_users.dbfile'); const promise = gateway.restoreBackup(emptyFile); @@ -232,7 +277,7 @@ describe('gateway', () => { getValue: fake.resolves('key'), setValue: fake.resolves(null), }; - const gateway = new Gateway(variable, event, system, sequelize, config); + const gateway = new Gateway(variable, event, system, sequelize, config, {}, {}, {}, job); const version = await gateway.getLatestGladysVersion(); expect(version).to.have.property('name'); expect(version).to.have.property('created_at'); @@ -245,7 +290,7 @@ describe('gateway', () => { getValue: fake.resolves('[]'), setValue: fake.resolves(null), }; - const gateway = new Gateway(variable, event, system, sequelize, config); + const gateway = new Gateway(variable, event, system, sequelize, config, {}, {}, {}, job); const users = await gateway.getUsersKeys(); expect(users).to.deep.equal([ { @@ -275,7 +320,7 @@ describe('gateway', () => { getValue: fake.resolves(JSON.stringify(oldUsers)), setValue: fake.resolves(null), }; - const gateway = new Gateway(variable, event, system, sequelize, config); + const gateway = new Gateway(variable, event, system, sequelize, config, {}, {}, {}, job); const users = await gateway.getUsersKeys(); expect(users).to.deep.equal([ { @@ -305,7 +350,7 @@ describe('gateway', () => { getValue: fake.resolves(JSON.stringify(oldUsers)), setValue: fake.resolves(null), }; - const gateway = new Gateway(variable, event, system, sequelize, config); + const gateway = new Gateway(variable, event, system, sequelize, config, {}, {}, {}, job); const users = await gateway.getUsersKeys(); expect(users).to.deep.equal([ { @@ -328,14 +373,14 @@ describe('gateway', () => { setValue: fake.resolves(null), destroy: fake.resolves(null), }; - const gateway = new Gateway(variable, event, system, sequelize, config); + const gateway = new Gateway(variable, event, system, sequelize, config, {}, {}, {}, job); await gateway.login('tony.stark@gladysassistant.com', 'warmachine123'); await gateway.disconnect(); }); }); describe('gateway.forwardWebsockets', () => { it('should forward a websocket message when connected', () => { - const gateway = new Gateway({}, event, system, sequelize, config); + const gateway = new Gateway({}, event, system, sequelize, config, {}, {}, {}, job); gateway.connected = true; const websocketMessage = { @@ -346,7 +391,7 @@ describe('gateway', () => { assert.calledWith(gateway.gladysGatewayClient.newEventInstance, websocketMessage.type, websocketMessage.payload); }); it('should prevent forwarding a websocket message when not connected', () => { - const gateway = new Gateway({}, event, system, sequelize, config); + const gateway = new Gateway({}, event, system, sequelize, config, {}, {}, {}, job); const websocketMessage = { type: 'zwave.new-node', @@ -362,7 +407,7 @@ describe('gateway', () => { const variable = { setValue: fake.resolves(null), }; - const gateway = new Gateway(variable, event, system, sequelize, config); + const gateway = new Gateway(variable, event, system, sequelize, config, {}, {}, {}, job); await gateway.login('tony.stark@gladysassistant.com', 'warmachine123'); return new Promise((resolve, reject) => { @@ -391,7 +436,7 @@ describe('gateway', () => { id: '0cd30aef-9c4e-4a23-88e3-3547971296e5', }), }; - const gateway = new Gateway(variable, eventGateway, system, sequelize, config, {}, stateManager); + const gateway = new Gateway(variable, eventGateway, system, sequelize, config, {}, stateManager, {}, job); await gateway.login('tony.stark@gladysassistant.com', 'warmachine123'); gateway.usersKeys = [ @@ -431,7 +476,7 @@ describe('gateway', () => { get: fake.returns(null), }; const eventGateway = new EventEmitter(); - const gateway = new Gateway(variable, eventGateway, system, sequelize, config, {}, stateManager); + const gateway = new Gateway(variable, eventGateway, system, sequelize, config, {}, stateManager, {}, job); await gateway.login('tony.stark@gladysassistant.com', 'warmachine123'); gateway.usersKeys = [ { @@ -476,7 +521,7 @@ describe('gateway', () => { }), }; const eventGateway = new EventEmitter(); - const gateway = new Gateway(variable, eventGateway, system, sequelize, config, {}, stateManager); + const gateway = new Gateway(variable, eventGateway, system, sequelize, config, {}, stateManager, {}, job); await gateway.login('tony.stark@gladysassistant.com', 'warmachine123'); gateway.usersKeys = [ { @@ -531,7 +576,7 @@ describe('gateway', () => { emit: fake.returns(), on: fake.returns(), }; - const gateway = new Gateway(variable, eventGateway, system, sequelize, config, {}, stateManager); + const gateway = new Gateway(variable, eventGateway, system, sequelize, config, {}, stateManager, {}, job); await gateway.login('tony.stark@gladysassistant.com', 'warmachine123'); gateway.usersKeys = [ { @@ -586,7 +631,7 @@ describe('gateway', () => { emit: fake.returns(), on: fake.returns(), }; - const gateway = new Gateway(variable, eventGateway, system, sequelize, config, {}, stateManager); + const gateway = new Gateway(variable, eventGateway, system, sequelize, config, {}, stateManager, {}, job); await gateway.login('tony.stark@gladysassistant.com', 'warmachine123'); gateway.usersKeys = [ { @@ -656,7 +701,17 @@ describe('gateway', () => { emit: fake.returns(), on: fake.returns(), }; - const gateway = new Gateway(variable, eventGateway, system, sequelize, config, {}, stateManager, serviceManager); + const gateway = new Gateway( + variable, + eventGateway, + system, + sequelize, + config, + {}, + stateManager, + serviceManager, + job, + ); await gateway.login('tony.stark@gladysassistant.com', 'warmachine123'); gateway.usersKeys = [ { @@ -808,7 +863,7 @@ describe('gateway', () => { return feature; }, }; - const gateway = new Gateway(variable, event, system, sequelize, config, {}, stateManager); + const gateway = new Gateway(variable, event, system, sequelize, config, {}, stateManager, {}, job); await gateway.login('tony.stark@gladysassistant.com', 'warmachine123'); gateway.googleHomeForwardStateTimeout = 1; gateway.connected = true; @@ -869,7 +924,7 @@ describe('gateway', () => { return feature; }, }; - const gateway = new Gateway(variable, event, system, sequelize, config, {}, stateManager); + const gateway = new Gateway(variable, event, system, sequelize, config, {}, stateManager, {}, job); await gateway.login('tony.stark@gladysassistant.com', 'warmachine123'); gateway.googleHomeForwardStateTimeout = 1; gateway.connected = true; diff --git a/server/utils/constants.js b/server/utils/constants.js index 506c31a12d..931913d85a 100644 --- a/server/utils/constants.js +++ b/server/utils/constants.js @@ -734,6 +734,7 @@ const JOB_TYPES = { HOURLY_DEVICE_STATE_AGGREGATE: 'hourly-device-state-aggregate', DAILY_DEVICE_STATE_AGGREGATE: 'daily-device-state-aggregate', MONTHLY_DEVICE_STATE_AGGREGATE: 'monthly-device-state-aggregate', + GLADYS_GATEWAY_BACKUP: 'gladys-gateway-backup', }; const JOB_STATUS = { diff --git a/server/utils/readChunk.js b/server/utils/readChunk.js new file mode 100644 index 0000000000..5f81f32f98 --- /dev/null +++ b/server/utils/readChunk.js @@ -0,0 +1,42 @@ +const { promisify } = require('util'); +const fs = require('fs'); +const { Buffer } = require('buffer'); +const pify = require('pify'); + +const fsReadP = pify(fs.read, { multiArgs: true }); +const fsOpenP = promisify(fs.open); +const fsCloseP = promisify(fs.close); + +/** + * @description Read one chunk of a file + * + * @param {string} filePath - The path of the file to read. + * @param {Object} options - Options. + * @returns {Promise} Return the chunk of the file. + * + * @example const data = await readChunk('./file.txt', {length: 10, startPosition: 10}); + */ +async function readChunk(filePath, { length, startPosition }) { + const fileDescriptor = await fsOpenP(filePath, 'r'); + + try { + // eslint-disable-next-line prefer-const + let [bytesRead, buffer] = await fsReadP(fileDescriptor, { + buffer: Buffer.alloc(length), + length, + position: startPosition, + }); + + if (bytesRead < length) { + buffer = buffer.slice(0, bytesRead); + } + + return buffer; + } finally { + await fsCloseP(fileDescriptor); + } +} + +module.exports = { + readChunk, +};