From 269d31aa7a7f720dc230f497da602c0853cb5356 Mon Sep 17 00:00:00 2001 From: Pierre-Gilles Leymarie Date: Fri, 13 Sep 2024 14:30:08 +0200 Subject: [PATCH 1/4] Add nightly job to purge old messages (#2119) --- .github/workflows/docker-dev-build.yml | 4 +- .github/workflows/docker-master-test.yml | 2 +- .github/workflows/docker-pr-build.yml | 6 +-- .github/workflows/docker-release-build.yml | 4 +- server/config/scheduler-jobs.js | 5 +++ server/lib/message/index.js | 6 ++- server/lib/message/message.purge.js | 29 +++++++++++++ server/test/lib/message/message.purge.test.js | 41 +++++++++++++++++++ server/utils/constants.js | 1 + 9 files changed, 89 insertions(+), 9 deletions(-) create mode 100644 server/lib/message/message.purge.js create mode 100644 server/test/lib/message/message.purge.test.js diff --git a/.github/workflows/docker-dev-build.yml b/.github/workflows/docker-dev-build.yml index d82b8f6185..60f0342999 100644 --- a/.github/workflows/docker-dev-build.yml +++ b/.github/workflows/docker-dev-build.yml @@ -34,7 +34,7 @@ jobs: run: | npm run build - name: ↗️ Upload build artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: static path: front/build @@ -64,7 +64,7 @@ jobs: with: version: latest - name: ↙️ Download build artifact - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: static path: static diff --git a/.github/workflows/docker-master-test.yml b/.github/workflows/docker-master-test.yml index e3e174f4ee..07087243c8 100644 --- a/.github/workflows/docker-master-test.yml +++ b/.github/workflows/docker-master-test.yml @@ -62,7 +62,7 @@ jobs: env: RELATIVE_CI_KEY: ${{ secrets.RELATIVE_CI_KEY }} - name: ↗️ Upload build artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: static path: front/build diff --git a/.github/workflows/docker-pr-build.yml b/.github/workflows/docker-pr-build.yml index 99021bdbb8..cccb00d5d1 100644 --- a/.github/workflows/docker-pr-build.yml +++ b/.github/workflows/docker-pr-build.yml @@ -144,11 +144,11 @@ jobs: run: | npm run build-with-stats - name: ↗️ Upload webpack stats artifact - uses: relative-ci/agent-upload-artifact-action@v1 + uses: relative-ci/agent-upload-artifact-action@v2 with: webpackStatsFile: ./front/stats.json - name: ↗️ Upload build artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: static path: front/build @@ -168,7 +168,7 @@ jobs: with: version: latest - name: ↙️ Download build artifact - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: static path: static diff --git a/.github/workflows/docker-release-build.yml b/.github/workflows/docker-release-build.yml index 6d71cbbf04..852495f396 100644 --- a/.github/workflows/docker-release-build.yml +++ b/.github/workflows/docker-release-build.yml @@ -114,7 +114,7 @@ jobs: run: | npm run build - name: ↗️ Upload build artifact - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: static path: front/build @@ -148,7 +148,7 @@ jobs: with: version: v0.9.1 - name: ↙️ Download build artifact - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4 with: name: static path: static diff --git a/server/config/scheduler-jobs.js b/server/config/scheduler-jobs.js index 0823d4683e..04f6c37a55 100644 --- a/server/config/scheduler-jobs.js +++ b/server/config/scheduler-jobs.js @@ -16,6 +16,11 @@ const jobs = [ rule: '0 0 22 * * *', // every day at 22:00 event: EVENTS.JOB.PURGE_OLD_JOBS, }, + { + name: 'daily-purge-of-old-messages', + rule: '0 0 23 * * *', // every day at 23:00 + event: EVENTS.MESSAGE.PURGE_OLD_MESSAGES, + }, { name: 'check-device-batteries', rule: '0 0 9 * * 6', // At 09:00 AM, only on Saturday diff --git a/server/lib/message/index.js b/server/lib/message/index.js index d836f3dfb4..ef351bdf60 100644 --- a/server/lib/message/index.js +++ b/server/lib/message/index.js @@ -2,9 +2,11 @@ const { EVENTS } = require('../../utils/constants'); const { create } = require('./message.create'); const { get } = require('./message.get'); const { reply } = require('./message.reply'); +const { purge } = require('./message.purge'); const { handleEvent } = require('./message.handleEvent'); const { replyByIntent } = require('./message.replyByIntent'); const { sendToUser } = require('./message.sendToUser'); +const { eventFunctionWrapper } = require('../../utils/functionsWrapper'); const MessageHandler = function MessageHandler(event, brain, service, state, variable) { this.event = event; @@ -12,13 +14,15 @@ const MessageHandler = function MessageHandler(event, brain, service, state, var this.service = service; this.state = state; this.variable = variable; - event.on(EVENTS.MESSAGE.NEW, (message) => this.handleEvent(message)); + this.event.on(EVENTS.MESSAGE.NEW, (message) => this.handleEvent(message)); + this.event.on(EVENTS.MESSAGE.PURGE_OLD_MESSAGES, eventFunctionWrapper(this.purge.bind(this))); }; MessageHandler.prototype.create = create; MessageHandler.prototype.get = get; MessageHandler.prototype.handleEvent = handleEvent; MessageHandler.prototype.reply = reply; +MessageHandler.prototype.purge = purge; MessageHandler.prototype.replyByIntent = replyByIntent; MessageHandler.prototype.sendToUser = sendToUser; diff --git a/server/lib/message/message.purge.js b/server/lib/message/message.purge.js new file mode 100644 index 0000000000..396e755fb3 --- /dev/null +++ b/server/lib/message/message.purge.js @@ -0,0 +1,29 @@ +const { Op } = require('sequelize'); +const db = require('../../models'); +const logger = require('../../utils/logger'); + +const DAYS_TO_KEEP = 15; + +/** + * @public + * @description Purge. + * @returns {Promise} Resolve. + * @example + * gladys.message.purge(); + */ +async function purge() { + const deleteBeforeDate = new Date(new Date().getTime() - DAYS_TO_KEEP * 24 * 60 * 60 * 1000); + logger.info(`Deleting all messages created before = ${deleteBeforeDate}`); + await db.Message.destroy({ + where: { + created_at: { + [Op.lte]: deleteBeforeDate, + }, + }, + }); + logger.info('Messages purged!'); +} + +module.exports = { + purge, +}; diff --git a/server/test/lib/message/message.purge.test.js b/server/test/lib/message/message.purge.test.js new file mode 100644 index 0000000000..3514110145 --- /dev/null +++ b/server/test/lib/message/message.purge.test.js @@ -0,0 +1,41 @@ +const { expect } = require('chai'); +const EventEmitter = require('events'); +const db = require('../../../models'); +const MessageHandler = require('../../../lib/message'); + +describe('message.purge', () => { + const eventEmitter = new EventEmitter(); + const messageHandler = new MessageHandler(eventEmitter); + it('should purge messages', async () => { + await db.Message.truncate(); + await db.Message.create({ + id: '2e3dccb0-fe8e-4e26-96c7-13041a1a3852', + sender_id: '0cd30aef-9c4e-4a23-88e3-3547971296e5', + receiver_id: null, + file: null, + text: 'This is an old message', + is_read: true, + created_at: new Date('2019-02-12T07:49:07.556Z'), + }); + await db.Message.create({ + id: 'b9a395df-d1d6-4905-a29f-2f110e028ea5', + sender_id: '0cd30aef-9c4e-4a23-88e3-3547971296e5', + receiver_id: null, + file: null, + text: 'this is a recent message', + is_read: true, + created_at: new Date(), + }); + await messageHandler.purge(); + const rows = await db.Message.findAll({ + attributes: ['id', 'text'], + raw: true, + }); + expect(rows).to.deep.equal([ + { + id: 'b9a395df-d1d6-4905-a29f-2f110e028ea5', + text: 'this is a recent message', + }, + ]); + }); +}); diff --git a/server/utils/constants.js b/server/utils/constants.js index 95eae8e42a..9e8a7aa702 100644 --- a/server/utils/constants.js +++ b/server/utils/constants.js @@ -241,6 +241,7 @@ const EVENTS = { MESSAGE: { NEW: 'message.new', NEW_FOR_OPEN_AI: 'message.new-for-open-ai', + PURGE_OLD_MESSAGES: 'message.purge-old-messages', }, SYSTEM: { DOWNLOAD_UPGRADE: 'system.download-upgrade', From 344ad9b8ca3078d9292dd95f2dd7b9172bc6ebbe Mon Sep 17 00:00:00 2001 From: chasebowman-contrast Date: Fri, 13 Sep 2024 06:38:54 -0600 Subject: [PATCH 2/4] Prevent user from updating his own role (#2115) --- server/api/controllers/user.controller.js | 1 + 1 file changed, 1 insertion(+) diff --git a/server/api/controllers/user.controller.js b/server/api/controllers/user.controller.js index 6aa147f501..4a7bdcdce7 100644 --- a/server/api/controllers/user.controller.js +++ b/server/api/controllers/user.controller.js @@ -131,6 +131,7 @@ module.exports = function UserController(gladys) { * @apiGroup User */ async function updateMySelf(req, res, next) { + delete req.body.role; const newUser = await gladys.user.update(req.user.id, req.body); res.json(newUser); } From 48c3786eb0d68e223464573636c46441c2ee3f89 Mon Sep 17 00:00:00 2001 From: Bertrand d'Aure Date: Fri, 13 Sep 2024 14:49:29 +0200 Subject: [PATCH 3/4] Caldav integration: Handle new until date for recurring events (#2107) --- server/lib/calendar/calendar.destroyEvents.js | 28 ++++++++++++---- .../calendar/calendar.syncUserCalendars.js | 14 ++++++++ server/services/caldav/package-lock.json | 33 +++++++++++++++++++ server/services/caldav/package.json | 1 + .../test/lib/calendar/calendar.event.test.js | 24 +++++++++++++- .../lib/calendar/syncUserCalendars.test.js | 16 +++++++++ 6 files changed, 108 insertions(+), 8 deletions(-) diff --git a/server/lib/calendar/calendar.destroyEvents.js b/server/lib/calendar/calendar.destroyEvents.js index f2acb6067e..532beb7202 100644 --- a/server/lib/calendar/calendar.destroyEvents.js +++ b/server/lib/calendar/calendar.destroyEvents.js @@ -1,17 +1,31 @@ +const { Op } = require('sequelize'); const db = require('../../models'); /** * @description Delete events from a calendar. * @param {string} calendarId - Calendar id to empty. + * @param {object} options - Options of the query. * @example - * gladys.calendar.destroyEvents('0dc03aef-4a23-9c4e-88e3-5437971269e5'); + * gladys.calendar.destroyEvents('0dc03aef-4a23-9c4e-88e3-5437971269e5', {url: '/calendar/event.ics'}); */ -async function destroyEvents(calendarId) { - await db.CalendarEvent.destroy({ - where: { - calendar_id: calendarId, - }, - }); +async function destroyEvents(calendarId, options = {}) { + const where = { + calendar_id: calendarId, + }; + + if (options.url) { + where.url = { + [Op.eq]: options.url, + }; + } + + if (options.from) { + where.start = { + [Op.gte]: new Date(options.from), + }; + } + + await db.CalendarEvent.destroy({ where }); } module.exports = { diff --git a/server/services/caldav/lib/calendar/calendar.syncUserCalendars.js b/server/services/caldav/lib/calendar/calendar.syncUserCalendars.js index 5b71c98edd..b38b83542e 100644 --- a/server/services/caldav/lib/calendar/calendar.syncUserCalendars.js +++ b/server/services/caldav/lib/calendar/calendar.syncUserCalendars.js @@ -1,4 +1,5 @@ const Promise = require('bluebird'); +const get = require('get-value'); const logger = require('../../../../utils/logger'); const { ServiceNotConfiguredError, NotFoundError } = require('../../../../utils/coreErrors'); @@ -108,6 +109,19 @@ async function syncUserCalendars(userId) { throw new NotFoundError('CALDAV_FAILED_REQUEST_EVENTS'); } + await Promise.map( + jsonEvents, + async (jsonEvent) => { + if (get(jsonEvent, 'rrule.options.until') && jsonEvent.href) { + await this.gladys.calendar.destroyEvents(calendarToUpdate.id, { + url: jsonEvent.href, + from: get(jsonEvent, 'rrule.options.until'), + }); + } + }, + { concurrency: 5 }, + ); + const formatedEvents = this.formatEvents(jsonEvents, calendarToUpdate); let insertedOrUpdatedEvent = 0; diff --git a/server/services/caldav/package-lock.json b/server/services/caldav/package-lock.json index 5ff5378e92..11532d5f57 100644 --- a/server/services/caldav/package-lock.json +++ b/server/services/caldav/package-lock.json @@ -19,6 +19,7 @@ "bluebird": "^3.7.0", "dav-request": "^1.8.0", "dayjs": "^1.11.10", + "get-value": "^3.0.1", "ical": "^0.8.0", "xmldom": "^0.6.0" } @@ -79,6 +80,17 @@ "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.10.tgz", "integrity": "sha512-vjAczensTgRcqDERK0SR2XMwsF/tSvnvlv6VcF2GIhg6Sx4yOIt/irsr1RDJsKiIyBzJDpCoXiWWq28MqH2cnQ==" }, + "node_modules/get-value": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/get-value/-/get-value-3.0.1.tgz", + "integrity": "sha512-mKZj9JLQrwMBtj5wxi6MH8Z5eSKaERpAwjg43dPtlGI1ZVEgH/qC7T8/6R2OBSUA+zzHBZgICsVJaEIV2tKTDA==", + "dependencies": { + "isobject": "^3.0.1" + }, + "engines": { + "node": ">=6.0" + } + }, "node_modules/ical": { "version": "0.8.0", "resolved": "https://registry.npmjs.org/ical/-/ical-0.8.0.tgz", @@ -87,6 +99,14 @@ "rrule": "2.4.1" } }, + "node_modules/isobject": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", + "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/luxon": { "version": "1.28.0", "resolved": "https://registry.npmjs.org/luxon/-/luxon-1.28.0.tgz", @@ -174,6 +194,14 @@ "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.10.tgz", "integrity": "sha512-vjAczensTgRcqDERK0SR2XMwsF/tSvnvlv6VcF2GIhg6Sx4yOIt/irsr1RDJsKiIyBzJDpCoXiWWq28MqH2cnQ==" }, + "get-value": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/get-value/-/get-value-3.0.1.tgz", + "integrity": "sha512-mKZj9JLQrwMBtj5wxi6MH8Z5eSKaERpAwjg43dPtlGI1ZVEgH/qC7T8/6R2OBSUA+zzHBZgICsVJaEIV2tKTDA==", + "requires": { + "isobject": "^3.0.1" + } + }, "ical": { "version": "0.8.0", "resolved": "https://registry.npmjs.org/ical/-/ical-0.8.0.tgz", @@ -182,6 +210,11 @@ "rrule": "2.4.1" } }, + "isobject": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", + "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==" + }, "luxon": { "version": "1.28.0", "resolved": "https://registry.npmjs.org/luxon/-/luxon-1.28.0.tgz", diff --git a/server/services/caldav/package.json b/server/services/caldav/package.json index b6d1448a35..793a6a7794 100644 --- a/server/services/caldav/package.json +++ b/server/services/caldav/package.json @@ -16,6 +16,7 @@ "bluebird": "^3.7.0", "dav-request": "^1.8.0", "dayjs": "^1.11.10", + "get-value": "^3.0.1", "ical": "^0.8.0", "xmldom": "^0.6.0" } diff --git a/server/test/lib/calendar/calendar.event.test.js b/server/test/lib/calendar/calendar.event.test.js index eff161ce85..a4d1bc40a3 100644 --- a/server/test/lib/calendar/calendar.event.test.js +++ b/server/test/lib/calendar/calendar.event.test.js @@ -52,8 +52,8 @@ describe('calendar.destroy', () => { }); describe('calendar.destroyEvents', () => { - const calendar = new Calendar(); it("should destroy all calendar's event", async () => { + const calendar = new Calendar(); await calendar.destroyEvents('07ec2599-3221-4d6c-ac56-41443973201b'); const allCalendarEvents = await calendar.getEvents( '0cd30aef-9c4e-4a23-88e3-3547971296e5', @@ -61,6 +61,28 @@ describe('calendar.destroyEvents', () => { ); assert.deepEqual(allCalendarEvents, []); }); + + it("should destroy calendar's events by url", async () => { + const calendar = new Calendar(); + await calendar.destroyEvents('07ec2599-3221-4d6c-ac56-41443973201b', { + url: '/remote.php/dav/calendars/tony/personal/eee42d70-24f2-4c18-949d-822f3f72594c.ics', + }); + const allCalendarEvents = await calendar.getEvents( + '0cd30aef-9c4e-4a23-88e3-3547971296e5', + '07ec2599-3221-4d6c-ac56-41443973201b', + ); + expect(allCalendarEvents.length).eq(1); + }); + + it("should destroy calendar's events starting after date", async () => { + const calendar = new Calendar(); + await calendar.destroyEvents('07ec2599-3221-4d6c-ac56-41443973201b', { from: '2019-03-10 07:49:07.556 +00:00' }); + const allCalendarEvents = await calendar.getEvents( + '0cd30aef-9c4e-4a23-88e3-3547971296e5', + '07ec2599-3221-4d6c-ac56-41443973201b', + ); + expect(allCalendarEvents.length).eq(1); + }); }); describe('calendar.getEvents', () => { diff --git a/server/test/services/caldav/lib/calendar/syncUserCalendars.test.js b/server/test/services/caldav/lib/calendar/syncUserCalendars.test.js index 89b2d63168..50b0cc365e 100644 --- a/server/test/services/caldav/lib/calendar/syncUserCalendars.test.js +++ b/server/test/services/caldav/lib/calendar/syncUserCalendars.test.js @@ -22,6 +22,7 @@ describe('CalDAV sync', () => { syncUserCalendars, formatCalendars, formatEvents, + formatRecurringEvents: sinon.stub().returns([]), requestCalendars: sinon.stub(), requestChanges: sinon.stub(), requestEventsData: sinon.stub(), @@ -37,6 +38,7 @@ describe('CalDAV sync', () => { .stub() .withArgs('event-to-delete') .resolves(), + destroyEvents: sinon.stub().resolves(), }, variable: { getValue: sinon.stub(), @@ -181,6 +183,19 @@ describe('CalDAV sync', () => { start: new Date('2018-06-08 00:00:00.000 +00:00'), location: null, }, + { + type: 'VEVENT', + uid: '3a98f1eb-e8e9-4f09-8454-353e92f9ff0d', + summary: 'Evenement 4 to update with rrule', + start: new Date('2018-06-08 00:00:00.000 +00:00'), + location: null, + href: '/home/personal/event-4.ics', + rrule: { + options: { + until: '2019-06-08 00:00:00.000 +00:00', + }, + }, + }, ]); sync.gladys.calendar.getEvents @@ -251,6 +266,7 @@ describe('CalDAV sync', () => { expect(sync.gladys.calendar.update.callCount).to.equal(1); expect(sync.gladys.calendar.getEvents.callCount).to.equal(4); expect(sync.gladys.calendar.destroyEvent.callCount).to.equal(1); + expect(sync.gladys.calendar.destroyEvents.callCount).to.equal(1); }); it('should failed if no CALDAV_HOST', async () => { From 1f7617c2fb47e4a52d81d790f0efd163f6d2a2ff Mon Sep 17 00:00:00 2001 From: Pierre-Gilles Leymarie Date: Fri, 13 Sep 2024 14:50:11 +0200 Subject: [PATCH 4/4] 4.45.1 --- package-lock.json | 4 ++-- package.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package-lock.json b/package-lock.json index 92e8e8efc8..4cf6a5916a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "gladys", - "version": "4.45.0", + "version": "4.45.1", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "gladys", - "version": "4.45.0", + "version": "4.45.1", "hasInstallScript": true, "license": "Apache-2.0", "devDependencies": { diff --git a/package.json b/package.json index f951cce2a0..d4ed969ba8 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "gladys", - "version": "4.45.0", + "version": "4.45.1", "description": "A privacy-first, open-source home assistant", "main": "index.js", "engines": {