Improve reliability of caldav service (#975)

* Add Promise.map + try catch around single insert of event * change all Promise.all in Promise.map * event selector should be unique * Fix eslint * Fix tests * Add test on updateEvent
GladysAssistant · Nov 27, 2020 · abb17b3 · abb17b3
1 parent d14b141
commit abb17b3
Show file tree

Hide file tree

Showing 4 changed files with 108 additions and 52 deletions.
diff --git a/server/services/caldav/lib/calendar/calendar.formaters.js b/server/services/caldav/lib/calendar/calendar.formaters.js
@@ -1,5 +1,3 @@
-const { slugify } = require('../../../../utils/slugify');
-
 // From : https://github.com/peterbraden/ical.js/blob/master/example_rrule.js
 /**
  * @description Format recurring events.
@@ -86,8 +84,8 @@ function formatRecurringEvents(event, gladysCalendar) {
 
     if (showRecurrence === true) {
       const newEvent = {
-        external_id: `${event.uid}${startDate.format('YYYY-MM-DD')}`,
-        selector: slugify(`${recurrenceTitle} ${startDate.format('YYYY-MM-DD')}`),
+        external_id: `${event.uid}${startDate.format('YYYY-MM-DD-HH-mm')}`,
+        selector: `${event.uid}${startDate.format('YYYY-MM-DD-HH-mm')}`,
         name: recurrenceTitle,
         location: event.location,
         url: event.href,
@@ -132,7 +130,7 @@ function formatEvents(caldavEvents, gladysCalendar) {
     if (typeof caldavEvent.rrule === 'undefined') {
       const newEvent = {
         external_id: caldavEvent.uid,
-        selector: slugify(`${caldavEvent.summary} ${this.moment(caldavEvent.start).format('YYYY-MM-DD')}`),
+        selector: caldavEvent.uid,
         name: caldavEvent.summary,
         location: caldavEvent.location,
         url: caldavEvent.href,

diff --git a/server/services/caldav/lib/calendar/calendar.syncUserCalendars.js b/server/services/caldav/lib/calendar/calendar.syncUserCalendars.js
@@ -39,8 +39,10 @@ async function syncUserCalendars(userId) {
 
   // Format all fetched calendars
   const formatedCalendars = this.formatCalendars(davCalendars, userId);
-  const calendarsToUpdate = await Promise.all(
-    formatedCalendars.map(async (formatedCalendar) => {
+
+  const calendarsToUpdate = await Promise.map(
+    formatedCalendars,
+    async (formatedCalendar) => {
       const gladysCalendar = await this.gladys.calendar.get(userId, { externalId: formatedCalendar.external_id });
       // Create calendar if it does not already exist in database
       if (gladysCalendar.length === 0) {
@@ -55,7 +57,8 @@ async function syncUserCalendars(userId) {
         return gladysCalendar[0];
       }
       return null;
-    }),
+    },
+    { concurrency: 1 },
   );
 
   await Promise.map(
@@ -69,8 +72,10 @@ async function syncUserCalendars(userId) {
         logger.error(e);
         throw new NotFoundError('CALDAV_FAILED_REQUEST_CHANGES');
       }
-      await Promise.all(
-        eventsToUpdate.map(async (eventToUpdate) => {
+
+      await Promise.map(
+        eventsToUpdate,
+        async (eventToUpdate) => {
           // Delete existing event if pops is empty
           if (JSON.stringify(eventToUpdate.props) === JSON.stringify({})) {
             const eventToDelete = await this.gladys.calendar.getEvents(userId, { url: eventToUpdate.href });
@@ -80,7 +85,8 @@ async function syncUserCalendars(userId) {
             return null;
           }
           return eventToUpdate;
-        }),
+        },
+        { concurrency: 1 },
       );
 
       if (
@@ -101,19 +107,29 @@ async function syncUserCalendars(userId) {
 
       const formatedEvents = this.formatEvents(jsonEvents, calendarToUpdate);
 
-      const savedEvents = await Promise.all(
-        formatedEvents.map(async (formatedEvent) => {
+      let insertedOrUpdatedEvent = 0;
+
+      await Promise.map(
+        formatedEvents,
+        async (formatedEvent) => {
           const gladysEvents = await this.gladys.calendar.getEvents(userId, { externalId: formatedEvent.external_id });
-          // Create event if it does not already exist in database
-          if (gladysEvents.length === 0) {
-            return this.gladys.calendar.createEvent(calendarToUpdate.selector, formatedEvent);
-          }
+          try {
+            // Create event if it does not already exist in database
+            if (gladysEvents.length === 0) {
+              await this.gladys.calendar.createEvent(calendarToUpdate.selector, formatedEvent);
+            } else {
+              // Else update existing event
+              await this.gladys.calendar.updateEvent(gladysEvents[0].selector, formatedEvent);
+            }
 
-          // Else update existing event
-          return this.gladys.calendar.updateEvent(gladysEvents[0].selector, formatedEvent);
-        }),
+            insertedOrUpdatedEvent += 1;
+          } catch (e) {
+            logger.error(e);
+          }
+        },
+        { concurrency: 1 },
       );
-      logger.info(`CalDAV : ${savedEvents.length} events updated for calendar ${calendarToUpdate.name}.`);
+      logger.info(`CalDAV : ${insertedOrUpdatedEvent} events updated for calendar ${calendarToUpdate.name}.`);
     },
     { concurrency: 1 },
   );

diff --git a/server/test/services/caldav/lib/calendar/formaters.test.js b/server/test/services/caldav/lib/calendar/formaters.test.js
@@ -101,7 +101,7 @@ describe('CalDAV formaters', () => {
     expectedEvents = [
       {
         external_id: 'e52c11e3-af8a-48c7-9f54-de7aba373c46',
-        selector: 'event-1-2019-02-25',
+        selector: 'e52c11e3-af8a-48c7-9f54-de7aba373c46',
         name: 'Event 1',
         location: 'Paris',
         start: '2019-02-25T10:00:00.000Z',
@@ -111,7 +111,7 @@ describe('CalDAV formaters', () => {
       },
       {
         external_id: '71c01038-2231-4dee-a230-6820fdb1136e',
-        selector: 'event-2-2019-04-01',
+        selector: '71c01038-2231-4dee-a230-6820fdb1136e',
         name: 'Event 2',
         location: 'Toulouse',
         start: '2019-04-01T00:00:00.000Z',
@@ -121,8 +121,8 @@ describe('CalDAV formaters', () => {
         url: 'https://caldav.host/home/event2.ics',
       },
       {
-        external_id: '29f76a08-5439-4e04-bc1f-a67c32b47c802019-09-27',
-        selector: 'anniversaire-pepper-2019-09-27',
+        external_id: '29f76a08-5439-4e04-bc1f-a67c32b47c802019-09-27-00-00',
+        selector: '29f76a08-5439-4e04-bc1f-a67c32b47c802019-09-27-00-00',
         name: 'Anniversaire Pepper',
         location: 'Paris',
         calendar_id: '1fe8f557-2685-4b6b-8f05-238184f6b701',
@@ -132,8 +132,8 @@ describe('CalDAV formaters', () => {
         url: 'https://caldav.host.com/home/recur-event2',
       },
       {
-        external_id: '29f76a08-5439-4e04-bc1f-a67c32b47c802020-09-27',
-        selector: 'anniversaire-pepper-2020-09-27',
+        external_id: '29f76a08-5439-4e04-bc1f-a67c32b47c802020-09-27-00-00',
+        selector: '29f76a08-5439-4e04-bc1f-a67c32b47c802020-09-27-00-00',
         name: 'Anniversaire Pepper',
         location: 'Paris',
         calendar_id: '1fe8f557-2685-4b6b-8f05-238184f6b701',
@@ -145,11 +145,11 @@ describe('CalDAV formaters', () => {
       {
         calendar_id: '1fe8f557-2685-4b6b-8f05-238184f6b701',
         end: '2021-09-28T00:00:00.000Z',
-        external_id: '29f76a08-5439-4e04-bc1f-a67c32b47c802021-09-27',
+        external_id: '29f76a08-5439-4e04-bc1f-a67c32b47c802021-09-27-00-00',
         full_day: true,
         location: 'Paris',
         name: 'Anniversaire Pepper',
-        selector: 'anniversaire-pepper-2021-09-27',
+        selector: '29f76a08-5439-4e04-bc1f-a67c32b47c802021-09-27-00-00',
         start: '2021-09-27T00:00:00.000Z',
         url: 'https://caldav.host.com/home/recur-event2',
       },
@@ -205,8 +205,8 @@ describe('CalDAV formaters', () => {
     expectedRecurrEvents = [
       [
         {
-          external_id: 'fdc2bf57-0adb-4300-8287-4a9b34dc37862019-06-01',
-          selector: 'cours-de-tennis-2019-06-01',
+          external_id: 'fdc2bf57-0adb-4300-8287-4a9b34dc37862019-06-01-09-00',
+          selector: 'fdc2bf57-0adb-4300-8287-4a9b34dc37862019-06-01-09-00',
           name: 'Cours de tennis',
           location: 'Stade Roland-Garros',
           calendar_id: '1fe8f557-2685-4b6b-8f05-238184f6b701',
@@ -219,8 +219,8 @@ describe('CalDAV formaters', () => {
       ],
       [
         {
-          external_id: '29f76a08-5439-4e04-bc1f-a67c32b47c802019-09-27',
-          selector: 'anniversaire-pepper-2019-09-27',
+          external_id: '29f76a08-5439-4e04-bc1f-a67c32b47c802019-09-27-00-00',
+          selector: '29f76a08-5439-4e04-bc1f-a67c32b47c802019-09-27-00-00',
           name: 'Anniversaire Pepper',
           location: 'Paris',
           calendar_id: '1fe8f557-2685-4b6b-8f05-238184f6b701',
@@ -230,8 +230,8 @@ describe('CalDAV formaters', () => {
           url: 'https://caldav.host.com/home/recur-event2',
         },
         {
-          external_id: '29f76a08-5439-4e04-bc1f-a67c32b47c802020-09-27',
-          selector: 'anniversaire-pepper-2020-09-27',
+          external_id: '29f76a08-5439-4e04-bc1f-a67c32b47c802020-09-27-00-00',
+          selector: '29f76a08-5439-4e04-bc1f-a67c32b47c802020-09-27-00-00',
           name: 'Anniversaire Pepper',
           location: 'Paris',
           calendar_id: '1fe8f557-2685-4b6b-8f05-238184f6b701',

diff --git a/server/test/services/caldav/lib/calendar/syncUserCalendars.test.js b/server/test/services/caldav/lib/calendar/syncUserCalendars.test.js
@@ -26,6 +26,7 @@ describe('CalDAV sync', () => {
         calendar: {
           create: sinon.stub(),
           createEvent: sinon.stub(),
+          updateEvent: sinon.stub(),
           get: sinon.stub(),
           update: sinon.stub().resolves(),
           getEvents: sinon.stub(),
@@ -160,33 +161,74 @@ describe('CalDAV sync', () => {
         start: new Date('2018-06-08 00:00:00.000 +00:00'),
         location: null,
       },
+      {
+        type: 'VEVENT',
+        uid: '49193db9-f666-4947-8ce6-3357ce3b7166',
+        summary: 'Evenement 1 duplicate to test errors',
+        start: new Date('2018-06-08 00:00:00.000 +00:00'),
+        location: null,
+      },
+      {
+        type: 'VEVENT',
+        uid: '9daca4e5-80dc-4b3e-8b15-a26e19e35ea5',
+        summary: 'Evenement 3 to update',
+        start: new Date('2018-06-08 00:00:00.000 +00:00'),
+        location: null,
+      },
     ]);
 
     sync.gladys.calendar.getEvents
       .withArgs(userId, { externalId: '49193db9-f666-4947-8ce6-3357ce3b7166' })
       .resolves([])
+      .withArgs(userId, { externalId: '9daca4e5-80dc-4b3e-8b15-a26e19e35ea5' })
+      .resolves([
+        {
+          selector: '9daca4e5-80dc-4b3e-8b15-a26e19e35ea5',
+          external_id: '9daca4e5-80dc-4b3e-8b15-a26e19e35ea5',
+          name: 'Evenement 3 to update',
+        },
+      ])
       .withArgs(userId, { url: 'https://caldav.host.com/home/professional/event-3.ics' })
       .resolves([
         {
           selector: 'event-to-delete',
         },
       ]);
 
-    sync.gladys.calendar.createEvent.resolves({
-      dataValues: {
-        id: '22396073-3fe6-49a6-bcd7-566281862b02',
-        calendar_id: '402dd55b-6e06-4a7c-8164-ba3e4641c71b',
-        name: 'Evenement 1',
-        selector: 'evenement-1-2018-06-08',
-        external_id: '49193db9-f666-4947-8ce6-3357ce3b7166',
-        location: null,
-        start: '2018-06-08 00:00:00.000 +00:00',
-        end: '2018-06-09 00:00:00.000 +00:00',
-        url: 'https://caldav.host.com/home/personal/event-1.ics',
-        full_day: '1',
-        created_at: '2020-02-11 21:04:56.090 +00:00',
-        updated_at: '2020-02-11 21:04:56.090 +00:00',
-      },
+    sync.gladys.calendar.createEvent
+      .onFirstCall()
+      .resolves({
+        dataValues: {
+          id: '22396073-3fe6-49a6-bcd7-566281862b02',
+          calendar_id: '402dd55b-6e06-4a7c-8164-ba3e4641c71b',
+          name: 'Evenement 1',
+          selector: 'evenement-1-2018-06-08',
+          external_id: '49193db9-f666-4947-8ce6-3357ce3b7166',
+          location: null,
+          start: '2018-06-08 00:00:00.000 +00:00',
+          end: '2018-06-09 00:00:00.000 +00:00',
+          url: 'https://caldav.host.com/home/personal/event-1.ics',
+          full_day: '1',
+          created_at: '2020-02-11 21:04:56.090 +00:00',
+          updated_at: '2020-02-11 21:04:56.090 +00:00',
+        },
+      })
+      .onSecondCall()
+      .rejects('ALREADY_EXIST');
+
+    sync.gladys.calendar.updateEvent.onFirstCall().resolves({
+      id: '078149ff-50f4-4f48-b2da-03dc06af0835',
+      calendar_id: '402dd55b-6e06-4a7c-8164-ba3e4641c71b',
+      selector: '9daca4e5-80dc-4b3e-8b15-a26e19e35ea5',
+      external_id: '9daca4e5-80dc-4b3e-8b15-a26e19e35ea5',
+      name: 'Evenement 3 to update',
+      location: null,
+      start: '2018-06-08 00:00:00.000 +00:00',
+      end: '2018-06-09 00:00:00.000 +00:00',
+      url: 'https://caldav.host.com/home/personal/event-1.ics',
+      full_day: '1',
+      created_at: '2020-02-11 21:04:56.090 +00:00',
+      updated_at: '2020-02-11 21:04:56.090 +00:00',
     });
 
     await sync.syncUserCalendars(userId);
@@ -198,10 +240,10 @@ describe('CalDAV sync', () => {
     expect(sync.requestEventsData.callCount).to.equal(1);
 
     expect(sync.gladys.calendar.create.callCount).to.equal(1);
-    expect(sync.gladys.calendar.createEvent.callCount).to.equal(1);
+    expect(sync.gladys.calendar.createEvent.callCount).to.equal(2);
     expect(sync.gladys.calendar.get.callCount).to.equal(3);
     expect(sync.gladys.calendar.update.callCount).to.equal(1);
-    expect(sync.gladys.calendar.getEvents.callCount).to.equal(2);
+    expect(sync.gladys.calendar.getEvents.callCount).to.equal(4);
     expect(sync.gladys.calendar.destroyEvent.callCount).to.equal(1);
   });