Merge branch 'master' into withings-only

front/src/config/i18n/en.json

@@ -1581,7 +1581,9 @@
       "monthly-device-state-aggregate": "Monthly sensors aggregation",
       "daily-device-state-aggregate": "Daily sensors aggregation",
       "hourly-device-state-aggregate": "Hourly sensors aggregation",
-      "gladys-gateway-backup": "Gladys Plus backup"
+      "gladys-gateway-backup": "Gladys Plus backup",
+      "device-state-purge-single-feature": "Single device feature states clean",
+      "vacuum": "Database cleaning"
     },
     "jobErrors": {
       "purged-when-restarted": "Gladys Assistant restarted while this job was still running, so it was purged. It doesn't mean the job has failed, it's a normal behavior."
@@ -1615,6 +1617,10 @@
     "timezoneText": "The timezone is used in scheduled scene.",
     "deviceStateRetentionTime": "Keep Device State History",
     "deviceStateRetentionTimeDescription": "How long Gladys will keep your device states in database.",
+    "vacuumDatabaseTitle": "Database cleaning",
+    "vacuumDatabaseDescription": "By clicking on this button, Gladys will start cleaning the database to free up space. This does not delete any data, it just physically remove old deleted data. Be careful, this task can take some time to run, and during that time Gladys will not be available.",
+    "vacuumDatabaseButton": "Start database cleaning",
+    "vacuumDatabaseStarted": "Database cleaning has started... Gladys will not be available until this job finishes.",
     "containerState": {
       "created": "Created",
       "restarting": "Restarting",

front/src/config/i18n/fr.json

@@ -1581,7 +1581,9 @@
       "monthly-device-state-aggregate": "Aggrégation donnée capteur mensuelle",
       "daily-device-state-aggregate": "Aggrégation donnée capteur journalière",
       "hourly-device-state-aggregate": "Aggrégation donnée capteur horaire",
-      "gladys-gateway-backup": "Sauvegarde Gladys Plus"
+      "gladys-gateway-backup": "Sauvegarde Gladys Plus",
+      "device-state-purge-single-feature": "Nettoyage des états d'un appareil",
+      "vacuum": "Nettoyage de la base de donnée"
     },
     "jobErrors": {
       "purged-when-restarted": "Gladys Assistant a redémarré alors que cette tâche était en cours. Cela ne veut pas dire que cette tâche a échouée, c'est un comportement normal."
@@ -1615,6 +1617,10 @@
     "timezoneText": "Le fuseau horaire est utilisé dans les scènes programmées.",
     "deviceStateRetentionTime": "Conserver l'historique des états des appareils",
     "deviceStateRetentionTimeDescription": "La période pendant laquelle Gladys gardera en base de données les états des appareils.",
+    "vacuumDatabaseTitle": "Nettoyer la base de donnée",
+    "vacuumDatabaseDescription": "En cliquant sur ce bouton, Gladys lancera un nettoyage de la base de donnée, ce qui libère de l'espace disque. Cette tâche ne supprime pas de données, elle re-arrange juste la donnée existante sur le disque et vide des vieilles lignes qui ne sont plus utilisées. Attention, cette opération peut prendre un certain temps sur les grosses bases de données, et pendant ce temps Gladys sera indisponible. Ne lancez pas cette commande si vous avez besoin de Gladys dans la prochaine heure.",
+    "vacuumDatabaseButton": "Lancer le nettoyage",
+    "vacuumDatabaseStarted": "Le nettoyage a commencé... Gladys sera indisponible jusqu'à la fin du nettoyage.",
     "containerState": {
       "created": "Créé",
       "restarting": "Redémarrage",

front/src/routes/settings/settings-system/SettingsSystemPage.jsx

@@ -200,6 +200,26 @@ const SystemPage = ({ children, ...props }) => (
                 </label>
               </div>
             </form>
+            <form class="mt-4">
+              <label>
+                <Text id="systemSettings.vacuumDatabaseTitle" />
+              </label>
+              <p>
+                <small>
+                  <Text id="systemSettings.vacuumDatabaseDescription" />
+                </small>
+              </p>
+              <p>
+                {props.vacuumStarted && (
+                  <div class="alert alert-info">
+                    <Text id="systemSettings.vacuumDatabaseStarted" />
+                  </div>
+                )}
+                <button onClick={props.vacuumDatabase} class="btn btn-primary">
+                  <Text id="systemSettings.vacuumDatabaseButton" />
+                </button>
+              </p>
+            </form>
           </div>
         </div>
       </div>

front/src/routes/settings/settings-system/index.js

@@ -26,6 +26,18 @@ class SettingsSystem extends Component {
     }
   };
 
+  vacuumDatabase = async e => {
+    e.preventDefault();
+    this.setState({
+      vacuumStarted: true
+    });
+    try {
+      await this.props.httpClient.post('/api/v1/system/vacuum');
+    } catch (e) {
+      console.error(e);
+    }
+  };
+
   getTimezone = async () => {
     try {
       const { value } = await this.props.httpClient.get(`/api/v1/variable/${SYSTEM_VARIABLE_NAMES.TIMEZONE}`);
@@ -86,7 +98,14 @@ class SettingsSystem extends Component {
     clearInterval(this.refreshPingIntervalId);
   }
 
-  render(props, { selectedTimezone, deviceStateHistoryInDays }) {
+  constructor(props) {
+    super(props);
+    this.state = {
+      vacuumStarted: false
+    };
+  }
+
+  render(props, { selectedTimezone, deviceStateHistoryInDays, vacuumStarted }) {
     const isDocker = get(props, 'systemInfos.is_docker');
     const upgradeDownloadInProgress = props.downloadUpgradeStatus === RequestStatus.Getting;
     const upgradeDownloadFinished = props.downloadUpgradeStatus === RequestStatus.Success;
@@ -104,6 +123,8 @@ class SettingsSystem extends Component {
         selectedTimezone={selectedTimezone}
         deviceStateHistoryInDays={deviceStateHistoryInDays}
         updateDeviceStateHistory={this.updateDeviceStateHistory}
+        vacuumDatabase={this.vacuumDatabase}
+        vacuumStarted={vacuumStarted}
       />
     );
   }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "gladys",
-  "version": "4.10.1",
+  "version": "4.10.2",
   "description": "A privacy-first, open-source home assistant",
   "main": "index.js",
   "engines": {

server/api/controllers/system.controller.js

@@ -71,12 +71,26 @@ module.exports = function SystemController(gladys) {
     gladys.system.shutdown();
   }
 
+  /**
+   * @api {post} /api/v1/system/vacuum
+   * @apiName vacuumSystem
+   * @apiGroup System
+   */
+  async function vacuum(req, res) {
+    gladys.event.emit(EVENTS.SYSTEM.VACUUM);
+    res.json({
+      success: true,
+      message: 'Vacuum started, system might be unresponsive for a while',
+    });
+  }
+
   return Object.freeze({
     downloadUpgrade: asyncMiddleware(downloadUpgrade),
     getSystemInfos: asyncMiddleware(getSystemInfos),
     getDiskSpace: asyncMiddleware(getDiskSpace),
     getContainers: asyncMiddleware(getContainers),
     shutdown: asyncMiddleware(shutdown),
     getUpgradeDownloadStatus: asyncMiddleware(getUpgradeDownloadStatus),
+    vacuum: asyncMiddleware(vacuum),
   });
 };

server/api/routes.js

@@ -496,6 +496,11 @@ function getRoutes(gladys) {
       authenticated: true,
       controller: systemController.getUpgradeDownloadStatus,
     },
+    'post /api/v1/system/vacuum': {
+      authenticated: true,
+      admin: true,
+      controller: systemController.vacuum,
+    },
     // user
     'post /api/v1/user': {
       authenticated: true,

server/lib/device/device.create.js

@@ -2,7 +2,6 @@ const Promise = require('bluebird');
 const { BadParameters } = require('../../utils/coreErrors');
 const db = require('../../models');
 const { EVENTS } = require('../../utils/constants');
-const logger = require('../../utils/logger');
 
 const getByExternalId = async (externalId) => {
   return db.Device.findOne({
@@ -154,17 +153,13 @@ async function create(device) {
   });
 
   // We purge states of all device features that were marked as "keep_history = false"
-  await Promise.each(deviceFeaturesIdsToPurge, async (deviceFeaturesIdToPurge) => {
-    await this.purgeStatesByFeatureId(deviceFeaturesIdToPurge);
+  // We do this asynchronously with an event, so it doesn't block the current request
+  // Also, the function called will delete as slowly as possible the event
+  // To make sure that Gladys is not locked during this time
+  deviceFeaturesIdsToPurge.forEach((deviceFeatureIdToPurge) => {
+    this.eventManager.emit(EVENTS.DEVICE.PURGE_STATES_SINGLE_FEATURE, deviceFeatureIdToPurge);
   });
 
-  if (deviceFeaturesIdsToPurge.length > 0) {
-    // If we don't run a VACUUM, the database file size will stay the same
-    // Read: https://www.sqlite.org/lang_vacuum.html
-    logger.info('Running VACUUM command to free up space.');
-    await db.sequelize.query('VACUUM;');
-  }
-
   // we get the whole device from the DB to avoid
   // having a partial final object
   const newDevice = (await getByExternalId(device.external_id)).get({ plain: true });

server/lib/device/device.purgeStatesByFeatureId.js

@@ -1,22 +1,105 @@
+const { QueryTypes } = require('sequelize');
+const Promise = require('bluebird');
 const db = require('../../models');
 const logger = require('../../utils/logger');
 
 /**
  * @description Purge device states of a specific feature
  * @param {string} deviceFeatureId - Id of a device feature.
+ * @param {string} jobId - Id of the job.
  * @returns {Promise} Resolve when finished.
  * @example
  * device.purgeStatesByFeatureId('d47b481b-a7be-4224-9850-313cdb8a4065');
  */
-async function purgeStatesByFeatureId(deviceFeatureId) {
+async function purgeStatesByFeatureId(deviceFeatureId, jobId) {
   logger.info(`Purging states of device feature ${deviceFeatureId}`);
-  const queryInterface = db.sequelize.getQueryInterface();
-  await queryInterface.bulkDelete('t_device_feature_state', {
-    device_feature_id: deviceFeatureId,
+
+  const numberOfDeviceFeatureStateToDelete = await db.DeviceFeatureState.count({
+    where: {
+      device_feature_id: deviceFeatureId,
+    },
+  });
+
+  const numberOfDeviceFeatureStateAggregateToDelete = await db.DeviceFeatureStateAggregate.count({
+    where: {
+      device_feature_id: deviceFeatureId,
+    },
   });
-  await queryInterface.bulkDelete('t_device_feature_state_aggregate', {
-    device_feature_id: deviceFeatureId,
+
+  logger.info(
+    `Purging "${deviceFeatureId}": ${numberOfDeviceFeatureStateToDelete} states & ${numberOfDeviceFeatureStateAggregateToDelete} aggregates to delete.`,
+  );
+
+  const numberOfIterationsStates = Math.ceil(
+    numberOfDeviceFeatureStateToDelete / this.STATES_TO_PURGE_PER_DEVICE_FEATURE_CLEAN_BATCH,
+  );
+  const iterator = [...Array(numberOfIterationsStates)];
+
+  const numberOfIterationsStatesAggregates = Math.ceil(
+    numberOfDeviceFeatureStateAggregateToDelete / this.STATES_TO_PURGE_PER_DEVICE_FEATURE_CLEAN_BATCH,
+  );
+  const iteratorAggregates = [...Array(numberOfIterationsStatesAggregates)];
+
+  const total = numberOfIterationsStates + numberOfIterationsStatesAggregates;
+  let currentBatch = 0;
+  let currentProgressPercent = 0;
+
+  // We only save progress to DB if it changed
+  // Because saving progress is expensive (DB write + Websocket call)
+  const updateProgressIfNeeded = async () => {
+    currentBatch += 1;
+    const newProgressPercent = Math.round((currentBatch * 100) / total);
+    if (currentProgressPercent !== newProgressPercent) {
+      currentProgressPercent = newProgressPercent;
+      await this.job.updateProgress(jobId, currentProgressPercent);
+    }
+  };
+
+  await Promise.each(iterator, async () => {
+    await db.sequelize.query(
+      `
+      DELETE FROM t_device_feature_state WHERE id IN (
+        SELECT id FROM t_device_feature_state
+        WHERE device_feature_id = :id
+        LIMIT :limit
+      );
+    `,
+      {
+        replacements: {
+          id: deviceFeatureId,
+          limit: this.STATES_TO_PURGE_PER_DEVICE_FEATURE_CLEAN_BATCH,
+        },
+        type: QueryTypes.SELECT,
+      },
+    );
+    await updateProgressIfNeeded();
+    await Promise.delay(this.WAIT_TIME_BETWEEN_DEVICE_FEATURE_CLEAN_BATCH);
+  });
+
+  await Promise.each(iteratorAggregates, async () => {
+    await db.sequelize.query(
+      `
+      DELETE FROM t_device_feature_state_aggregate WHERE id IN (
+        SELECT id FROM t_device_feature_state_aggregate
+        WHERE device_feature_id = :id
+        LIMIT :limit
+      );
+    `,
+      {
+        replacements: {
+          id: deviceFeatureId,
+          limit: this.STATES_TO_PURGE_PER_DEVICE_FEATURE_CLEAN_BATCH,
+        },
+        type: QueryTypes.SELECT,
+      },
+    );
+    await updateProgressIfNeeded();
+    await Promise.delay(this.WAIT_TIME_BETWEEN_DEVICE_FEATURE_CLEAN_BATCH);
   });
+  return {
+    numberOfDeviceFeatureStateToDelete,
+    numberOfDeviceFeatureStateAggregateToDelete,
+  };
 }
 
 module.exports = {

server/lib/device/index.js

@@ -1,4 +1,4 @@
-const { EVENTS } = require('../../utils/constants');
+const { EVENTS, JOB_TYPES } = require('../../utils/constants');
 const { eventFunctionWrapper } = require('../../utils/functionsWrapper');
 
 // Categories of DeviceFeatures
@@ -51,12 +51,20 @@ const DeviceManager = function DeviceManager(
   this.variable = variable;
   this.job = job;
 
+  this.STATES_TO_PURGE_PER_DEVICE_FEATURE_CLEAN_BATCH = 1000;
+  this.WAIT_TIME_BETWEEN_DEVICE_FEATURE_CLEAN_BATCH = 100;
+
   // initialize all types of device feature categories
   this.camera = new CameraManager(this.stateManager, messageManager, eventManager, this);
   this.lightManager = new LightManager(eventManager, messageManager, this);
   this.temperatureSensorManager = new TemperatureSensorManager(eventManager, messageManager, this);
   this.humiditySensorManager = new HumiditySensorManager(eventManager, messageManager, this);
 
+  this.purgeStatesByFeatureId = this.job.wrapper(
+    JOB_TYPES.DEVICE_STATES_PURGE_SINGLE_FEATURE,
+    this.purgeStatesByFeatureId.bind(this),
+  );
+
   this.devicesByPollFrequency = {};
   // listen to events
   this.eventManager.on(EVENTS.DEVICE.NEW_STATE, this.newStateEvent.bind(this));
@@ -68,6 +76,10 @@ const DeviceManager = function DeviceManager(
     EVENTS.DEVICE.CALCULATE_HOURLY_AGGREGATE,
     eventFunctionWrapper(this.onHourlyDeviceAggregateEvent.bind(this)),
   );
+  this.eventManager.on(
+    EVENTS.DEVICE.PURGE_STATES_SINGLE_FEATURE,
+    eventFunctionWrapper(this.purgeStatesByFeatureId.bind(this)),
+  );
 };
 
 DeviceManager.prototype.add = add;

server/lib/index.js

@@ -55,7 +55,7 @@ function Gladys(params = {}) {
   const area = new Area(event);
   const dashboard = new Dashboard();
   const stateManager = new StateManager(event);
-  const system = new System(db.sequelize, event, config);
+  const system = new System(db.sequelize, event, config, job);
   const http = new Http(system);
   const house = new House(event, stateManager);
   const room = new Room(brain);

server/lib/job/job.wrapper.js

@@ -14,8 +14,9 @@ function wrapper(type, func) {
     let job;
     try {
       job = await this.start(type);
-      await func(...args, job.id);
+      const res = await func(...args, job.id);
       await this.finish(job.id, JOB_STATUS.SUCCESS);
+      return res;
     } catch (error) {
       if (job) {
         const data = {