From ab4f5278b0222eed121492ec3e181900f81cee73 Mon Sep 17 00:00:00 2001 From: Kevin Nderitu Date: Tue, 16 Mar 2021 19:02:35 +0300 Subject: [PATCH 01/48] Added a migration --- .../index.js | 26 ++++++++++++++++ ...6-populate-mimetype-on-attachments.spec.js | 30 +++++++++++++++++++ .../specs/fixtures.js | 3 ++ 3 files changed, 59 insertions(+) create mode 100644 app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js create mode 100644 app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js create mode 100644 app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/fixtures.js diff --git a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js new file mode 100644 index 0000000000..0aeb7b0a6b --- /dev/null +++ b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js @@ -0,0 +1,26 @@ +import request from 'shared/JSONRequest'; + +export default { + delta: 36, + + name: 'populate-mimetype-to-attachment', + + description: 'Populates mimetype of an attachment from a url ', + + async up(db) { + process.stdout.write(`${this.name}...\r\n`); + const cursor = await db.collection('files').find(); + + // eslint-disable-next-line no-await-in-loop + while (await cursor.hasNext()) { + const file = cursor.next(); + if (file.url && !file.mimetype) { + // eslint-disable-next-line no-await-in-loop + const response = await request.head(file.url); + const mimetype = response.headers.get('content-type') || undefined; + + db.collection('files').updateOne({ _id: file._id }, { $set: { mimetype } }); + } + } + }, +}; diff --git a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js new file mode 100644 index 0000000000..8e0bdbc73f --- /dev/null +++ b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js @@ -0,0 +1,30 @@ +import testingDB from 'api/utils/testing_db'; +import request from 'shared/JSONRequest'; +import migration from '../index.js'; +import fixtures from './fixtures.js'; + +describe('migration populate-mimetype-on-attachments', () => { + let headRequestMock; + beforeEach(async () => { + spyOn(process.stdout, 'write'); + headRequestMock = spyOn(request, 'head'); + await testingDB.clearAllAndLoad(fixtures); + }); + + afterAll(async () => { + headRequestMock.mockRestore(); + await testingDB.disconnect(); + }); + + it('should have a delta number', () => { + expect(migration.delta).toBe(36); + }); + + it('should populate mimetype with Content-Type', async () => { + headRequestMock.mockReturnValue( + Promise.resolve({ headers: { 'Content-Type': 'application/pdf' } }) + ); + await migration.up(testingDB.mongodb); + expect(request.head).toHaveBeenCalled(); + }); +}); diff --git a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/fixtures.js b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/fixtures.js new file mode 100644 index 0000000000..64aaf47ee6 --- /dev/null +++ b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/fixtures.js @@ -0,0 +1,3 @@ +export default { + files: [{ url: 'some/file/path.jpg' }], +}; From 2cf7bd105537538234fb27788533c0cf214a95ac Mon Sep 17 00:00:00 2001 From: Kevin Nderitu Date: Wed, 17 Mar 2021 16:55:43 +0300 Subject: [PATCH 02/48] Added tests to the migration --- .../index.js | 5 ++- ...6-populate-mimetype-on-attachments.spec.js | 41 +++++++++++++++++-- .../specs/fixtures.js | 2 +- 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js index 0aeb7b0a6b..039dc2c13e 100644 --- a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js +++ b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js @@ -9,11 +9,12 @@ export default { async up(db) { process.stdout.write(`${this.name}...\r\n`); - const cursor = await db.collection('files').find(); + const cursor = await db.collection('files').find({}); // eslint-disable-next-line no-await-in-loop while (await cursor.hasNext()) { - const file = cursor.next(); + // eslint-disable-next-line no-await-in-loop + const file = await cursor.next(); if (file.url && !file.mimetype) { // eslint-disable-next-line no-await-in-loop const response = await request.head(file.url); diff --git a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js index 8e0bdbc73f..382418f06b 100644 --- a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js +++ b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js @@ -21,10 +21,45 @@ describe('migration populate-mimetype-on-attachments', () => { }); it('should populate mimetype with Content-Type', async () => { - headRequestMock.mockReturnValue( - Promise.resolve({ headers: { 'Content-Type': 'application/pdf' } }) + await testingDB.clearAllAndLoad(fixtures); + const headers = { + get: jest + .fn() + .mockReturnValueOnce('application/pdf') + .mockReturnValueOnce('mimetype2'), + }; + headRequestMock.and.returnValue( + Promise.resolve({ + headers, + }) ); await migration.up(testingDB.mongodb); - expect(request.head).toHaveBeenCalled(); + expect(request.head).toHaveBeenCalledWith(fixtures.files[0].url); + expect(request.head).toHaveBeenCalledWith(fixtures.files[1].url); + expect(headers.get).toHaveBeenCalledWith('content-type'); + + const files = await testingDB.mongodb + .collection('files') + .find({}) + .toArray(); + + expect(files[0].mimetype).toEqual('application/pdf'); + expect(files[1].mimetype).toEqual('mimetype2'); + }); + + it('should not change the value of mimetype if it already exists', async () => { + const fixturesWithMimetype = { + files: [ + { + url: 'some/url/item.jpg', + mimetype: 'application/pdf', + }, + ], + }; + await testingDB.clearAllAndLoad(fixturesWithMimetype); + + const file = await testingDB.mongodb.collection('files').findOne({}); + + expect(file.mimetype).toEqual(fixturesWithMimetype.files[0].mimetype); }); }); diff --git a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/fixtures.js b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/fixtures.js index 64aaf47ee6..ddb3e522b1 100644 --- a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/fixtures.js +++ b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/fixtures.js @@ -1,3 +1,3 @@ export default { - files: [{ url: 'some/file/path.jpg' }], + files: [{ url: 'some/file/path.jpg' }, { url: 'some/other/path.jpg' }], }; From 4fb9c8630aa6f3a9150f912d357c8c3a50d8ef5b Mon Sep 17 00:00:00 2001 From: Kevin Nderitu Date: Thu, 18 Mar 2021 09:53:38 +0300 Subject: [PATCH 03/48] Added a case where the attachment is local --- .../index.js | 9 +++- ...6-populate-mimetype-on-attachments.spec.js | 54 ++++++++++++++++++- 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js index 039dc2c13e..0510918bc6 100644 --- a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js +++ b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js @@ -1,4 +1,6 @@ import request from 'shared/JSONRequest'; +import { attachmentsPath } from 'api/files/filesystem'; +import { execSync } from 'child_process'; export default { delta: 36, @@ -19,7 +21,12 @@ export default { // eslint-disable-next-line no-await-in-loop const response = await request.head(file.url); const mimetype = response.headers.get('content-type') || undefined; - + db.collection('files').updateOne({ _id: file._id }, { $set: { mimetype } }); + } + if (file.filename && file.type === 'attachment' && !file.mimetype) { + const mimetype = execSync(`file --mime-type -b "${attachmentsPath(file.filename)}"`) + .toString() + .trim(); db.collection('files').updateOne({ _id: file._id }, { $set: { mimetype } }); } } diff --git a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js index 382418f06b..c1a8f78ee5 100644 --- a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js +++ b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js @@ -1,18 +1,27 @@ import testingDB from 'api/utils/testing_db'; import request from 'shared/JSONRequest'; +import * as attachmentMethods from 'api/files/filesystem'; +import childProcess from 'child_process'; import migration from '../index.js'; import fixtures from './fixtures.js'; describe('migration populate-mimetype-on-attachments', () => { let headRequestMock; + let attachmentPathMock; + let execSyncMock; + beforeEach(async () => { spyOn(process.stdout, 'write'); headRequestMock = spyOn(request, 'head'); + attachmentPathMock = spyOn(attachmentMethods, 'attachmentsPath'); + execSyncMock = spyOn(childProcess, 'execSync'); await testingDB.clearAllAndLoad(fixtures); }); afterAll(async () => { headRequestMock.mockRestore(); + attachmentPathMock.mockRestore(); + execSyncMock.mockRestore(); await testingDB.disconnect(); }); @@ -47,7 +56,7 @@ describe('migration populate-mimetype-on-attachments', () => { expect(files[1].mimetype).toEqual('mimetype2'); }); - it('should not change the value of mimetype if it already exists', async () => { + it('should not change the value of mimetype if it already exists in external attachments', async () => { const fixturesWithMimetype = { files: [ { @@ -62,4 +71,47 @@ describe('migration populate-mimetype-on-attachments', () => { expect(file.mimetype).toEqual(fixturesWithMimetype.files[0].mimetype); }); + + it('should not change if value of mimetype already exists in internal attachments', async () => { + const fixturesWithFilenames = { + files: [ + { + filename: 'somename.pdf', + mimetype: 'application/pdf', + type: 'attachment', + }, + ], + }; + await testingDB.clearAllAndLoad(fixturesWithFilenames); + attachmentPathMock.and.returnValue('/some/path/to/file.pdf'); + execSyncMock.and.returnValue('application/pdf'); + await migration.up(testingDB.mongodb); + + const file = await testingDB.mongodb.collection('files').findOne({}); + expect(file.mimetype).toEqual(fixturesWithFilenames.files[0].mimetype); + }); + + it('should update mimetype if filename exists in internal attachments', async () => { + const fixturesWithFilenames = { + files: [ + { + filename: 'somename.pdf', + type: 'attachment', + }, + ], + }; + await testingDB.clearAllAndLoad(fixturesWithFilenames); + execSyncMock.and.returnValue('application/pdf'); + attachmentPathMock.and.returnValue('/some/path/to/file.pdf'); + await migration.up(testingDB.mongodb); + + const file = await testingDB.mongodb.collection('files').findOne({}); + expect(file.mimetype).toEqual('application/pdf'); + expect(attachmentMethods.attachmentsPath).toHaveBeenCalledWith( + fixturesWithFilenames.files[0].filename + ); + expect(childProcess.execSync).toHaveBeenCalledWith( + 'file --mime-type -b "/some/path/to/file.pdf"' + ); + }); }); From eee3b43dfffc501de4932114732f3cc3a023a977 Mon Sep 17 00:00:00 2001 From: Kevin Nderitu Date: Thu, 18 Mar 2021 09:57:24 +0300 Subject: [PATCH 04/48] Addeda test making sure mimetype is not updated if type is not attachment --- .../36-populate-mimetype-on-attachments.spec.js | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js index c1a8f78ee5..bae0607f3c 100644 --- a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js +++ b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js @@ -114,4 +114,21 @@ describe('migration populate-mimetype-on-attachments', () => { 'file --mime-type -b "/some/path/to/file.pdf"' ); }); + it('should not update mimetype if type is not attachment in internal attachments', async () => { + const fixturesWithFilenames = { + files: [ + { + filename: 'somename.pdf', + type: 'document', + }, + ], + }; + await testingDB.clearAllAndLoad(fixturesWithFilenames); + execSyncMock.and.returnValue('application/pdf'); + attachmentPathMock.and.returnValue('/some/path/to/file.pdf'); + await migration.up(testingDB.mongodb); + + expect(attachmentMethods.attachmentsPath).not.toHaveBeenCalledWith(); + expect(childProcess.execSync).not.toHaveBeenCalledWith(); + }); }); From 6aec884aa3edd11488085d8b0ed313071aaff9de Mon Sep 17 00:00:00 2001 From: Kevin Nderitu Date: Fri, 19 Mar 2021 11:09:27 +0300 Subject: [PATCH 05/48] Updated tests --- .../specs/36-populate-mimetype-on-attachments.spec.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js index bae0607f3c..5b36525019 100644 --- a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js +++ b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js @@ -128,7 +128,9 @@ describe('migration populate-mimetype-on-attachments', () => { attachmentPathMock.and.returnValue('/some/path/to/file.pdf'); await migration.up(testingDB.mongodb); - expect(attachmentMethods.attachmentsPath).not.toHaveBeenCalledWith(); - expect(childProcess.execSync).not.toHaveBeenCalledWith(); + expect(attachmentMethods.attachmentsPath).not.toHaveBeenCalledWith( + fixturesWithFilenames.files[0].filename + ); + expect(childProcess.execSync).not.toHaveBeenCalledWith('/some/path/to/file.pdf'); }); }); From 1763aa49d32588421fb2c903140557117d4e862c Mon Sep 17 00:00:00 2001 From: Kevin Nderitu Date: Mon, 22 Mar 2021 15:29:43 +0300 Subject: [PATCH 06/48] Removed fixtures file --- .../specs/36-populate-mimetype-on-attachments.spec.js | 5 +++-- .../36-populate-mimetype-on-attachments/specs/fixtures.js | 3 --- 2 files changed, 3 insertions(+), 5 deletions(-) delete mode 100644 app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/fixtures.js diff --git a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js index 5b36525019..80670e790d 100644 --- a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js +++ b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js @@ -3,7 +3,6 @@ import request from 'shared/JSONRequest'; import * as attachmentMethods from 'api/files/filesystem'; import childProcess from 'child_process'; import migration from '../index.js'; -import fixtures from './fixtures.js'; describe('migration populate-mimetype-on-attachments', () => { let headRequestMock; @@ -15,7 +14,6 @@ describe('migration populate-mimetype-on-attachments', () => { headRequestMock = spyOn(request, 'head'); attachmentPathMock = spyOn(attachmentMethods, 'attachmentsPath'); execSyncMock = spyOn(childProcess, 'execSync'); - await testingDB.clearAllAndLoad(fixtures); }); afterAll(async () => { @@ -30,6 +28,9 @@ describe('migration populate-mimetype-on-attachments', () => { }); it('should populate mimetype with Content-Type', async () => { + const fixtures = { + files: [{ url: 'some/file/path.jpg' }, { url: 'some/other/path.jpg' }], + } await testingDB.clearAllAndLoad(fixtures); const headers = { get: jest diff --git a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/fixtures.js b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/fixtures.js deleted file mode 100644 index ddb3e522b1..0000000000 --- a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/fixtures.js +++ /dev/null @@ -1,3 +0,0 @@ -export default { - files: [{ url: 'some/file/path.jpg' }, { url: 'some/other/path.jpg' }], -}; From f2b39cbf6220f4bc7ac4bcb3bc26a43bcc5f6b7f Mon Sep 17 00:00:00 2001 From: Kevin Nderitu Date: Mon, 22 Mar 2021 15:47:10 +0300 Subject: [PATCH 07/48] Added an else for readability --- .../migrations/36-populate-mimetype-on-attachments/index.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js index 0510918bc6..10926a95dd 100644 --- a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js +++ b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js @@ -22,8 +22,7 @@ export default { const response = await request.head(file.url); const mimetype = response.headers.get('content-type') || undefined; db.collection('files').updateOne({ _id: file._id }, { $set: { mimetype } }); - } - if (file.filename && file.type === 'attachment' && !file.mimetype) { + } else if (file.filename && file.type === 'attachment' && !file.mimetype) { const mimetype = execSync(`file --mime-type -b "${attachmentsPath(file.filename)}"`) .toString() .trim(); From 2413b95f58e2c1b10e087e522d09c3012c94da1d Mon Sep 17 00:00:00 2001 From: Kevin Nderitu Date: Tue, 16 Mar 2021 19:02:35 +0300 Subject: [PATCH 08/48] Added a migration --- .../index.js | 26 ++++++++++++++++ ...6-populate-mimetype-on-attachments.spec.js | 30 +++++++++++++++++++ .../specs/fixtures.js | 3 ++ 3 files changed, 59 insertions(+) create mode 100644 app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js create mode 100644 app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js create mode 100644 app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/fixtures.js diff --git a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js new file mode 100644 index 0000000000..0aeb7b0a6b --- /dev/null +++ b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js @@ -0,0 +1,26 @@ +import request from 'shared/JSONRequest'; + +export default { + delta: 36, + + name: 'populate-mimetype-to-attachment', + + description: 'Populates mimetype of an attachment from a url ', + + async up(db) { + process.stdout.write(`${this.name}...\r\n`); + const cursor = await db.collection('files').find(); + + // eslint-disable-next-line no-await-in-loop + while (await cursor.hasNext()) { + const file = cursor.next(); + if (file.url && !file.mimetype) { + // eslint-disable-next-line no-await-in-loop + const response = await request.head(file.url); + const mimetype = response.headers.get('content-type') || undefined; + + db.collection('files').updateOne({ _id: file._id }, { $set: { mimetype } }); + } + } + }, +}; diff --git a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js new file mode 100644 index 0000000000..8e0bdbc73f --- /dev/null +++ b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js @@ -0,0 +1,30 @@ +import testingDB from 'api/utils/testing_db'; +import request from 'shared/JSONRequest'; +import migration from '../index.js'; +import fixtures from './fixtures.js'; + +describe('migration populate-mimetype-on-attachments', () => { + let headRequestMock; + beforeEach(async () => { + spyOn(process.stdout, 'write'); + headRequestMock = spyOn(request, 'head'); + await testingDB.clearAllAndLoad(fixtures); + }); + + afterAll(async () => { + headRequestMock.mockRestore(); + await testingDB.disconnect(); + }); + + it('should have a delta number', () => { + expect(migration.delta).toBe(36); + }); + + it('should populate mimetype with Content-Type', async () => { + headRequestMock.mockReturnValue( + Promise.resolve({ headers: { 'Content-Type': 'application/pdf' } }) + ); + await migration.up(testingDB.mongodb); + expect(request.head).toHaveBeenCalled(); + }); +}); diff --git a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/fixtures.js b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/fixtures.js new file mode 100644 index 0000000000..64aaf47ee6 --- /dev/null +++ b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/fixtures.js @@ -0,0 +1,3 @@ +export default { + files: [{ url: 'some/file/path.jpg' }], +}; From a470aa32566545ccd62efb4facd44c4b48f307bd Mon Sep 17 00:00:00 2001 From: Kevin Nderitu Date: Wed, 17 Mar 2021 16:55:43 +0300 Subject: [PATCH 09/48] Added tests to the migration --- .../index.js | 5 ++- ...6-populate-mimetype-on-attachments.spec.js | 41 +++++++++++++++++-- .../specs/fixtures.js | 2 +- 3 files changed, 42 insertions(+), 6 deletions(-) diff --git a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js index 0aeb7b0a6b..039dc2c13e 100644 --- a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js +++ b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js @@ -9,11 +9,12 @@ export default { async up(db) { process.stdout.write(`${this.name}...\r\n`); - const cursor = await db.collection('files').find(); + const cursor = await db.collection('files').find({}); // eslint-disable-next-line no-await-in-loop while (await cursor.hasNext()) { - const file = cursor.next(); + // eslint-disable-next-line no-await-in-loop + const file = await cursor.next(); if (file.url && !file.mimetype) { // eslint-disable-next-line no-await-in-loop const response = await request.head(file.url); diff --git a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js index 8e0bdbc73f..382418f06b 100644 --- a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js +++ b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js @@ -21,10 +21,45 @@ describe('migration populate-mimetype-on-attachments', () => { }); it('should populate mimetype with Content-Type', async () => { - headRequestMock.mockReturnValue( - Promise.resolve({ headers: { 'Content-Type': 'application/pdf' } }) + await testingDB.clearAllAndLoad(fixtures); + const headers = { + get: jest + .fn() + .mockReturnValueOnce('application/pdf') + .mockReturnValueOnce('mimetype2'), + }; + headRequestMock.and.returnValue( + Promise.resolve({ + headers, + }) ); await migration.up(testingDB.mongodb); - expect(request.head).toHaveBeenCalled(); + expect(request.head).toHaveBeenCalledWith(fixtures.files[0].url); + expect(request.head).toHaveBeenCalledWith(fixtures.files[1].url); + expect(headers.get).toHaveBeenCalledWith('content-type'); + + const files = await testingDB.mongodb + .collection('files') + .find({}) + .toArray(); + + expect(files[0].mimetype).toEqual('application/pdf'); + expect(files[1].mimetype).toEqual('mimetype2'); + }); + + it('should not change the value of mimetype if it already exists', async () => { + const fixturesWithMimetype = { + files: [ + { + url: 'some/url/item.jpg', + mimetype: 'application/pdf', + }, + ], + }; + await testingDB.clearAllAndLoad(fixturesWithMimetype); + + const file = await testingDB.mongodb.collection('files').findOne({}); + + expect(file.mimetype).toEqual(fixturesWithMimetype.files[0].mimetype); }); }); diff --git a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/fixtures.js b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/fixtures.js index 64aaf47ee6..ddb3e522b1 100644 --- a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/fixtures.js +++ b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/fixtures.js @@ -1,3 +1,3 @@ export default { - files: [{ url: 'some/file/path.jpg' }], + files: [{ url: 'some/file/path.jpg' }, { url: 'some/other/path.jpg' }], }; From cee74b270f505860a959aa2ba38e8ee392c0c260 Mon Sep 17 00:00:00 2001 From: Kevin Nderitu Date: Thu, 18 Mar 2021 09:53:38 +0300 Subject: [PATCH 10/48] Added a case where the attachment is local --- .../index.js | 9 +++- ...6-populate-mimetype-on-attachments.spec.js | 54 ++++++++++++++++++- 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js index 039dc2c13e..0510918bc6 100644 --- a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js +++ b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js @@ -1,4 +1,6 @@ import request from 'shared/JSONRequest'; +import { attachmentsPath } from 'api/files/filesystem'; +import { execSync } from 'child_process'; export default { delta: 36, @@ -19,7 +21,12 @@ export default { // eslint-disable-next-line no-await-in-loop const response = await request.head(file.url); const mimetype = response.headers.get('content-type') || undefined; - + db.collection('files').updateOne({ _id: file._id }, { $set: { mimetype } }); + } + if (file.filename && file.type === 'attachment' && !file.mimetype) { + const mimetype = execSync(`file --mime-type -b "${attachmentsPath(file.filename)}"`) + .toString() + .trim(); db.collection('files').updateOne({ _id: file._id }, { $set: { mimetype } }); } } diff --git a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js index 382418f06b..c1a8f78ee5 100644 --- a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js +++ b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js @@ -1,18 +1,27 @@ import testingDB from 'api/utils/testing_db'; import request from 'shared/JSONRequest'; +import * as attachmentMethods from 'api/files/filesystem'; +import childProcess from 'child_process'; import migration from '../index.js'; import fixtures from './fixtures.js'; describe('migration populate-mimetype-on-attachments', () => { let headRequestMock; + let attachmentPathMock; + let execSyncMock; + beforeEach(async () => { spyOn(process.stdout, 'write'); headRequestMock = spyOn(request, 'head'); + attachmentPathMock = spyOn(attachmentMethods, 'attachmentsPath'); + execSyncMock = spyOn(childProcess, 'execSync'); await testingDB.clearAllAndLoad(fixtures); }); afterAll(async () => { headRequestMock.mockRestore(); + attachmentPathMock.mockRestore(); + execSyncMock.mockRestore(); await testingDB.disconnect(); }); @@ -47,7 +56,7 @@ describe('migration populate-mimetype-on-attachments', () => { expect(files[1].mimetype).toEqual('mimetype2'); }); - it('should not change the value of mimetype if it already exists', async () => { + it('should not change the value of mimetype if it already exists in external attachments', async () => { const fixturesWithMimetype = { files: [ { @@ -62,4 +71,47 @@ describe('migration populate-mimetype-on-attachments', () => { expect(file.mimetype).toEqual(fixturesWithMimetype.files[0].mimetype); }); + + it('should not change if value of mimetype already exists in internal attachments', async () => { + const fixturesWithFilenames = { + files: [ + { + filename: 'somename.pdf', + mimetype: 'application/pdf', + type: 'attachment', + }, + ], + }; + await testingDB.clearAllAndLoad(fixturesWithFilenames); + attachmentPathMock.and.returnValue('/some/path/to/file.pdf'); + execSyncMock.and.returnValue('application/pdf'); + await migration.up(testingDB.mongodb); + + const file = await testingDB.mongodb.collection('files').findOne({}); + expect(file.mimetype).toEqual(fixturesWithFilenames.files[0].mimetype); + }); + + it('should update mimetype if filename exists in internal attachments', async () => { + const fixturesWithFilenames = { + files: [ + { + filename: 'somename.pdf', + type: 'attachment', + }, + ], + }; + await testingDB.clearAllAndLoad(fixturesWithFilenames); + execSyncMock.and.returnValue('application/pdf'); + attachmentPathMock.and.returnValue('/some/path/to/file.pdf'); + await migration.up(testingDB.mongodb); + + const file = await testingDB.mongodb.collection('files').findOne({}); + expect(file.mimetype).toEqual('application/pdf'); + expect(attachmentMethods.attachmentsPath).toHaveBeenCalledWith( + fixturesWithFilenames.files[0].filename + ); + expect(childProcess.execSync).toHaveBeenCalledWith( + 'file --mime-type -b "/some/path/to/file.pdf"' + ); + }); }); From 7f573c0f4a4d22a894ee1c065360e9e58160611e Mon Sep 17 00:00:00 2001 From: Kevin Nderitu Date: Thu, 18 Mar 2021 09:57:24 +0300 Subject: [PATCH 11/48] Addeda test making sure mimetype is not updated if type is not attachment --- .../36-populate-mimetype-on-attachments.spec.js | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js index c1a8f78ee5..bae0607f3c 100644 --- a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js +++ b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js @@ -114,4 +114,21 @@ describe('migration populate-mimetype-on-attachments', () => { 'file --mime-type -b "/some/path/to/file.pdf"' ); }); + it('should not update mimetype if type is not attachment in internal attachments', async () => { + const fixturesWithFilenames = { + files: [ + { + filename: 'somename.pdf', + type: 'document', + }, + ], + }; + await testingDB.clearAllAndLoad(fixturesWithFilenames); + execSyncMock.and.returnValue('application/pdf'); + attachmentPathMock.and.returnValue('/some/path/to/file.pdf'); + await migration.up(testingDB.mongodb); + + expect(attachmentMethods.attachmentsPath).not.toHaveBeenCalledWith(); + expect(childProcess.execSync).not.toHaveBeenCalledWith(); + }); }); From fc2386cc6f29fce998933f920a6cd05ce728d7ae Mon Sep 17 00:00:00 2001 From: Kevin Nderitu Date: Fri, 19 Mar 2021 11:09:27 +0300 Subject: [PATCH 12/48] Updated tests --- .../specs/36-populate-mimetype-on-attachments.spec.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js index bae0607f3c..5b36525019 100644 --- a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js +++ b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js @@ -128,7 +128,9 @@ describe('migration populate-mimetype-on-attachments', () => { attachmentPathMock.and.returnValue('/some/path/to/file.pdf'); await migration.up(testingDB.mongodb); - expect(attachmentMethods.attachmentsPath).not.toHaveBeenCalledWith(); - expect(childProcess.execSync).not.toHaveBeenCalledWith(); + expect(attachmentMethods.attachmentsPath).not.toHaveBeenCalledWith( + fixturesWithFilenames.files[0].filename + ); + expect(childProcess.execSync).not.toHaveBeenCalledWith('/some/path/to/file.pdf'); }); }); From 385127fbc46cd22ae4c4fd82d18ffc2c4ded2589 Mon Sep 17 00:00:00 2001 From: Kevin Nderitu Date: Mon, 22 Mar 2021 15:29:43 +0300 Subject: [PATCH 13/48] Removed fixtures file --- .../specs/36-populate-mimetype-on-attachments.spec.js | 5 +++-- .../36-populate-mimetype-on-attachments/specs/fixtures.js | 3 --- 2 files changed, 3 insertions(+), 5 deletions(-) delete mode 100644 app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/fixtures.js diff --git a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js index 5b36525019..80670e790d 100644 --- a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js +++ b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js @@ -3,7 +3,6 @@ import request from 'shared/JSONRequest'; import * as attachmentMethods from 'api/files/filesystem'; import childProcess from 'child_process'; import migration from '../index.js'; -import fixtures from './fixtures.js'; describe('migration populate-mimetype-on-attachments', () => { let headRequestMock; @@ -15,7 +14,6 @@ describe('migration populate-mimetype-on-attachments', () => { headRequestMock = spyOn(request, 'head'); attachmentPathMock = spyOn(attachmentMethods, 'attachmentsPath'); execSyncMock = spyOn(childProcess, 'execSync'); - await testingDB.clearAllAndLoad(fixtures); }); afterAll(async () => { @@ -30,6 +28,9 @@ describe('migration populate-mimetype-on-attachments', () => { }); it('should populate mimetype with Content-Type', async () => { + const fixtures = { + files: [{ url: 'some/file/path.jpg' }, { url: 'some/other/path.jpg' }], + } await testingDB.clearAllAndLoad(fixtures); const headers = { get: jest diff --git a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/fixtures.js b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/fixtures.js deleted file mode 100644 index ddb3e522b1..0000000000 --- a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/fixtures.js +++ /dev/null @@ -1,3 +0,0 @@ -export default { - files: [{ url: 'some/file/path.jpg' }, { url: 'some/other/path.jpg' }], -}; From 25e42deb77078e409525dc7f6899e6a49e148201 Mon Sep 17 00:00:00 2001 From: Kevin Nderitu Date: Mon, 22 Mar 2021 15:47:10 +0300 Subject: [PATCH 14/48] Added an else for readability --- .../migrations/36-populate-mimetype-on-attachments/index.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js index 0510918bc6..10926a95dd 100644 --- a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js +++ b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js @@ -22,8 +22,7 @@ export default { const response = await request.head(file.url); const mimetype = response.headers.get('content-type') || undefined; db.collection('files').updateOne({ _id: file._id }, { $set: { mimetype } }); - } - if (file.filename && file.type === 'attachment' && !file.mimetype) { + } else if (file.filename && file.type === 'attachment' && !file.mimetype) { const mimetype = execSync(`file --mime-type -b "${attachmentsPath(file.filename)}"`) .toString() .trim(); From 7adaaa7c7cc2005de9f448a1eaa870f5d17688b6 Mon Sep 17 00:00:00 2001 From: Kevin Nderitu Date: Mon, 22 Mar 2021 16:05:21 +0300 Subject: [PATCH 15/48] Added a mixed test --- ...6-populate-mimetype-on-attachments.spec.js | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js index 80670e790d..86d1d6062d 100644 --- a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js +++ b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js @@ -134,4 +134,34 @@ describe('migration populate-mimetype-on-attachments', () => { ); expect(childProcess.execSync).not.toHaveBeenCalledWith('/some/path/to/file.pdf'); }); + it('should not use local attachment if url field is present', async () => { + const mixedFixtures = { + files: [ + { + filename: 'somename.pdf', + type: 'attachment', + url: '/some/url/to/file.something', + }, + ], + }; + await testingDB.clearAllAndLoad(mixedFixtures); + const headers = { + get: jest + .fn() + .mockReturnValueOnce('application/pdf') + .mockReturnValueOnce('mimetype2'), + }; + headRequestMock.and.returnValue( + Promise.resolve({ + headers, + }) + ); + execSyncMock.and.returnValue('application/pdf'); + attachmentPathMock.and.returnValue('/some/path/to/file.pdf'); + await migration.up(testingDB.mongodb); + + expect(attachmentMethods.attachmentsPath).not.toHaveBeenCalled(); + expect(childProcess.execSync).not.toHaveBeenCalled(); + expect(request.head).toHaveBeenCalledWith(mixedFixtures.files[0].url); + }); }); From e159e2a8d355f2097f677656f395abafbfa0b09d Mon Sep 17 00:00:00 2001 From: Kevin Nderitu Date: Mon, 22 Mar 2021 16:06:56 +0300 Subject: [PATCH 16/48] Fixed syntax error --- .../specs/36-populate-mimetype-on-attachments.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js index 86d1d6062d..dc2dac7a35 100644 --- a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js +++ b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js @@ -30,7 +30,7 @@ describe('migration populate-mimetype-on-attachments', () => { it('should populate mimetype with Content-Type', async () => { const fixtures = { files: [{ url: 'some/file/path.jpg' }, { url: 'some/other/path.jpg' }], - } + }; await testingDB.clearAllAndLoad(fixtures); const headers = { get: jest From 21e94367e213071470e56161f90f79001fac781a Mon Sep 17 00:00:00 2001 From: Kevin Nderitu Date: Tue, 23 Mar 2021 12:40:58 +0300 Subject: [PATCH 17/48] Added mime-type lib to get the mimetype of files --- .../index.js | 9 ++++---- ...6-populate-mimetype-on-attachments.spec.js | 22 +++++++++---------- package.json | 1 + yarn.lock | 12 ++++++++++ 4 files changed, 28 insertions(+), 16 deletions(-) diff --git a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js index 10926a95dd..e08c30f56a 100644 --- a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js +++ b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js @@ -1,13 +1,14 @@ import request from 'shared/JSONRequest'; import { attachmentsPath } from 'api/files/filesystem'; -import { execSync } from 'child_process'; +// import { execSync } from 'child_process'; +import mime from 'mime-types'; export default { delta: 36, name: 'populate-mimetype-to-attachment', - description: 'Populates mimetype of an attachment from a url ', + description: 'Populates mimetype of an attachment from a url', async up(db) { process.stdout.write(`${this.name}...\r\n`); @@ -23,9 +24,7 @@ export default { const mimetype = response.headers.get('content-type') || undefined; db.collection('files').updateOne({ _id: file._id }, { $set: { mimetype } }); } else if (file.filename && file.type === 'attachment' && !file.mimetype) { - const mimetype = execSync(`file --mime-type -b "${attachmentsPath(file.filename)}"`) - .toString() - .trim(); + const mimetype = mime.lookup(attachmentsPath(file.filename)); db.collection('files').updateOne({ _id: file._id }, { $set: { mimetype } }); } } diff --git a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js index dc2dac7a35..27fb21a758 100644 --- a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js +++ b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js @@ -2,24 +2,26 @@ import testingDB from 'api/utils/testing_db'; import request from 'shared/JSONRequest'; import * as attachmentMethods from 'api/files/filesystem'; import childProcess from 'child_process'; +import mime from 'mime-types'; import migration from '../index.js'; describe('migration populate-mimetype-on-attachments', () => { let headRequestMock; let attachmentPathMock; let execSyncMock; + let mimeMock; beforeEach(async () => { spyOn(process.stdout, 'write'); headRequestMock = spyOn(request, 'head'); attachmentPathMock = spyOn(attachmentMethods, 'attachmentsPath'); - execSyncMock = spyOn(childProcess, 'execSync'); + mimeMock = spyOn(mime, 'lookup'); }); afterAll(async () => { headRequestMock.mockRestore(); attachmentPathMock.mockRestore(); - execSyncMock.mockRestore(); + mimeMock.mockRestore(); await testingDB.disconnect(); }); @@ -85,7 +87,7 @@ describe('migration populate-mimetype-on-attachments', () => { }; await testingDB.clearAllAndLoad(fixturesWithFilenames); attachmentPathMock.and.returnValue('/some/path/to/file.pdf'); - execSyncMock.and.returnValue('application/pdf'); + mimeMock.and.returnValue('application/pdf'); await migration.up(testingDB.mongodb); const file = await testingDB.mongodb.collection('files').findOne({}); @@ -102,7 +104,7 @@ describe('migration populate-mimetype-on-attachments', () => { ], }; await testingDB.clearAllAndLoad(fixturesWithFilenames); - execSyncMock.and.returnValue('application/pdf'); + mimeMock.and.returnValue('application/pdf'); attachmentPathMock.and.returnValue('/some/path/to/file.pdf'); await migration.up(testingDB.mongodb); @@ -111,9 +113,7 @@ describe('migration populate-mimetype-on-attachments', () => { expect(attachmentMethods.attachmentsPath).toHaveBeenCalledWith( fixturesWithFilenames.files[0].filename ); - expect(childProcess.execSync).toHaveBeenCalledWith( - 'file --mime-type -b "/some/path/to/file.pdf"' - ); + expect(mime.lookup).toHaveBeenCalledWith('/some/path/to/file.pdf'); }); it('should not update mimetype if type is not attachment in internal attachments', async () => { const fixturesWithFilenames = { @@ -125,14 +125,14 @@ describe('migration populate-mimetype-on-attachments', () => { ], }; await testingDB.clearAllAndLoad(fixturesWithFilenames); - execSyncMock.and.returnValue('application/pdf'); + mimeMock.and.returnValue('application/pdf'); attachmentPathMock.and.returnValue('/some/path/to/file.pdf'); await migration.up(testingDB.mongodb); expect(attachmentMethods.attachmentsPath).not.toHaveBeenCalledWith( fixturesWithFilenames.files[0].filename ); - expect(childProcess.execSync).not.toHaveBeenCalledWith('/some/path/to/file.pdf'); + expect(mime.lookup).not.toHaveBeenCalledWith('/some/path/to/file.pdf'); }); it('should not use local attachment if url field is present', async () => { const mixedFixtures = { @@ -156,12 +156,12 @@ describe('migration populate-mimetype-on-attachments', () => { headers, }) ); - execSyncMock.and.returnValue('application/pdf'); + mimeMock.and.returnValue('application/pdf'); attachmentPathMock.and.returnValue('/some/path/to/file.pdf'); await migration.up(testingDB.mongodb); expect(attachmentMethods.attachmentsPath).not.toHaveBeenCalled(); - expect(childProcess.execSync).not.toHaveBeenCalled(); + expect(mime.lookup).not.toHaveBeenCalled(); expect(request.head).toHaveBeenCalledWith(mixedFixtures.files[0].url); }); }); diff --git a/package.json b/package.json index 2136c743f7..dfae563e7f 100644 --- a/package.json +++ b/package.json @@ -112,6 +112,7 @@ "mark.js": "^8.11.1", "markdown-it": "11.0.0", "markdown-it-container": "3.0.0", + "mime-types": "^2.1.29", "moment": "2.27.0", "moment-timezone": "0.5.31", "mongodb": "^3.6.2", diff --git a/yarn.lock b/yarn.lock index 20b664de43..e466648f41 100644 --- a/yarn.lock +++ b/yarn.lock @@ -9428,6 +9428,11 @@ mime-db@1.45.0: resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.45.0.tgz#cceeda21ccd7c3a745eba2decd55d4b73e7879ea" integrity sha512-CkqLUxUk15hofLoLyljJSrukZi8mAtgd+yE5uO4tqRZsdsAJKv0O+rFMhVDRJgozy+yG6md5KwuXhD4ocIoP+w== +mime-db@1.46.0: + version "1.46.0" + resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.46.0.tgz#6267748a7f799594de3cbc8cde91def349661cee" + integrity sha512-svXaP8UQRZ5K7or+ZmfNhg2xX3yKDMUzqadsSqi4NCH/KomcH75MAMYAGVlvXn4+b/xOPhS3I2uHKRUzvjY7BQ== + mime-db@~1.33.0: version "1.33.0" resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.33.0.tgz#a3492050a5cb9b63450541e39d9788d2272783db" @@ -9439,6 +9444,13 @@ mime-types@^2.1.12, mime-types@~2.1.19: dependencies: mime-db "1.45.0" +mime-types@^2.1.29: + version "2.1.29" + resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.29.tgz#1d4ab77da64b91f5f72489df29236563754bb1b2" + integrity sha512-Y/jMt/S5sR9OaqteJtslsFZKWOIIqMACsJSiHghlCAyhf7jfVYjKBmLiX8OgpWeW+fjJ2b+Az69aPFPkUOY6xQ== + dependencies: + mime-db "1.46.0" + mime-types@~2.1.17, mime-types@~2.1.18: version "2.1.18" resolved "https://registry.yarnpkg.com/mime-types/-/mime-types-2.1.18.tgz#6f323f60a83d11146f831ff11fd66e2fe5503bb8" From e06be466a275d81a87a988963fb3c97479200633 Mon Sep 17 00:00:00 2001 From: Kevin Nderitu Date: Tue, 23 Mar 2021 12:50:35 +0300 Subject: [PATCH 18/48] Removed comments and unused variables --- .../migrations/36-populate-mimetype-on-attachments/index.js | 1 - .../specs/36-populate-mimetype-on-attachments.spec.js | 2 -- 2 files changed, 3 deletions(-) diff --git a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js index e08c30f56a..d6d9402908 100644 --- a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js +++ b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js @@ -1,6 +1,5 @@ import request from 'shared/JSONRequest'; import { attachmentsPath } from 'api/files/filesystem'; -// import { execSync } from 'child_process'; import mime from 'mime-types'; export default { diff --git a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js index 27fb21a758..a9c9cb97ca 100644 --- a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js +++ b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/specs/36-populate-mimetype-on-attachments.spec.js @@ -1,14 +1,12 @@ import testingDB from 'api/utils/testing_db'; import request from 'shared/JSONRequest'; import * as attachmentMethods from 'api/files/filesystem'; -import childProcess from 'child_process'; import mime from 'mime-types'; import migration from '../index.js'; describe('migration populate-mimetype-on-attachments', () => { let headRequestMock; let attachmentPathMock; - let execSyncMock; let mimeMock; beforeEach(async () => { From 83d31ae2d5a6f227568a5117a45d7e0e12800a01 Mon Sep 17 00:00:00 2001 From: Daneryl Date: Wed, 24 Mar 2021 08:49:37 +0100 Subject: [PATCH 19/48] build should exit with 1 on any command fails --- webpack/build.sh | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/webpack/build.sh b/webpack/build.sh index 2a22672acc..07ab4877b0 100755 --- a/webpack/build.sh +++ b/webpack/build.sh @@ -1,13 +1,15 @@ #!/bin/bash +set -euo pipefail +shopt -s inherit_errexit export NODE_ENV=production -SCRIPTPATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +SCRIPTPATH="$(cd "$(dirname "$0")" >/dev/null 2>&1; pwd -P)" cd "$SCRIPTPATH"/../ rm -rf ./prod/* yarn webpack --config ./webpack.production.config.js --progress --profile --colors -yarn babel -D -d prod/app --extensions .js,.ts,.tsx --ignore **/specs/* app +yarn babel -D -d prod/app --extensions .js,.ts,.tsx --ignore ./**/specs/* app yarn babel -D -d prod/ message.js yarn babel -D -d prod/database --extensions .js,.ts,.tsx database From 9a64551d2478b73d469ec86198fa241a045e43a8 Mon Sep 17 00:00:00 2001 From: Kevin Nderitu Date: Wed, 24 Mar 2021 16:34:52 +0300 Subject: [PATCH 20/48] Added undefined as default mimetype --- .../migrations/36-populate-mimetype-on-attachments/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js index d6d9402908..c0cb887e03 100644 --- a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js +++ b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js @@ -23,7 +23,7 @@ export default { const mimetype = response.headers.get('content-type') || undefined; db.collection('files').updateOne({ _id: file._id }, { $set: { mimetype } }); } else if (file.filename && file.type === 'attachment' && !file.mimetype) { - const mimetype = mime.lookup(attachmentsPath(file.filename)); + const mimetype = mime.lookup(attachmentsPath(file.filename)) || undefined; db.collection('files').updateOne({ _id: file._id }, { $set: { mimetype } }); } } From de45075154d67b6a951272aa4a117ba2df527e58 Mon Sep 17 00:00:00 2001 From: Kevin Nderitu Date: Fri, 19 Mar 2021 17:19:15 +0300 Subject: [PATCH 21/48] Added a style to fix copy from layout --- .../Documents/components/DocumentSidePanel.js | 27 ++++++++++--------- 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/app/react/Documents/components/DocumentSidePanel.js b/app/react/Documents/components/DocumentSidePanel.js index 6c346f5744..fb965850d8 100644 --- a/app/react/Documents/components/DocumentSidePanel.js +++ b/app/react/Documents/components/DocumentSidePanel.js @@ -377,18 +377,21 @@ export class DocumentSidePanel extends Component { if (docBeingEdited && this.state.copyFrom) { return ( <> - - +
+ + + +
); } From e4b974a8bf71e8fafbcd6ffd7e27d432fc67d644 Mon Sep 17 00:00:00 2001 From: Kevin Nderitu Date: Fri, 19 Mar 2021 17:24:51 +0300 Subject: [PATCH 22/48] Removed unnecessary fragment --- .../Documents/components/DocumentSidePanel.js | 31 +++++++++---------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/app/react/Documents/components/DocumentSidePanel.js b/app/react/Documents/components/DocumentSidePanel.js index fb965850d8..86007f5702 100644 --- a/app/react/Documents/components/DocumentSidePanel.js +++ b/app/react/Documents/components/DocumentSidePanel.js @@ -376,23 +376,20 @@ export class DocumentSidePanel extends Component { {(() => { if (docBeingEdited && this.state.copyFrom) { return ( - <> -
- - - -
- +
+ + +
); } if (docBeingEdited) { From 825a8abd166a5229829e52dcc2156bd097c75132 Mon Sep 17 00:00:00 2001 From: Kevin Nderitu Date: Fri, 19 Mar 2021 17:19:15 +0300 Subject: [PATCH 23/48] Added a style to fix copy from layout --- .../Documents/components/DocumentSidePanel.js | 31 ++++++++++--------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/app/react/Documents/components/DocumentSidePanel.js b/app/react/Documents/components/DocumentSidePanel.js index 86007f5702..fb965850d8 100644 --- a/app/react/Documents/components/DocumentSidePanel.js +++ b/app/react/Documents/components/DocumentSidePanel.js @@ -376,20 +376,23 @@ export class DocumentSidePanel extends Component { {(() => { if (docBeingEdited && this.state.copyFrom) { return ( -
- - -
+ <> +
+ + + +
+ ); } if (docBeingEdited) { From 01402929b04ff0fea447b64f257e4aaa406f1364 Mon Sep 17 00:00:00 2001 From: Kevin Nderitu Date: Fri, 19 Mar 2021 17:24:51 +0300 Subject: [PATCH 24/48] Removed unnecessary fragment --- .../Documents/components/DocumentSidePanel.js | 31 +++++++++---------- 1 file changed, 14 insertions(+), 17 deletions(-) diff --git a/app/react/Documents/components/DocumentSidePanel.js b/app/react/Documents/components/DocumentSidePanel.js index fb965850d8..86007f5702 100644 --- a/app/react/Documents/components/DocumentSidePanel.js +++ b/app/react/Documents/components/DocumentSidePanel.js @@ -376,23 +376,20 @@ export class DocumentSidePanel extends Component { {(() => { if (docBeingEdited && this.state.copyFrom) { return ( - <> -
- - - -
- +
+ + +
); } if (docBeingEdited) { From ecd00f80f42c593acbd8326982cccd9cd3b5c8e7 Mon Sep 17 00:00:00 2001 From: Kevin Nderitu Date: Wed, 24 Mar 2021 16:14:01 +0300 Subject: [PATCH 25/48] Added a seperate style class --- app/react/Documents/components/DocumentSidePanel.js | 2 +- app/react/Documents/components/scss/toc.scss | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/app/react/Documents/components/DocumentSidePanel.js b/app/react/Documents/components/DocumentSidePanel.js index 86007f5702..9142b14b69 100644 --- a/app/react/Documents/components/DocumentSidePanel.js +++ b/app/react/Documents/components/DocumentSidePanel.js @@ -376,7 +376,7 @@ export class DocumentSidePanel extends Component { {(() => { if (docBeingEdited && this.state.copyFrom) { return ( -
+
Date: Thu, 25 Mar 2021 13:40:31 +0300 Subject: [PATCH 26/48] Added border --- app/react/Documents/components/scss/toc.scss | 3 +++ 1 file changed, 3 insertions(+) diff --git a/app/react/Documents/components/scss/toc.scss b/app/react/Documents/components/scss/toc.scss index cd1296c7b6..bcfa17cb42 100644 --- a/app/react/Documents/components/scss/toc.scss +++ b/app/react/Documents/components/scss/toc.scss @@ -21,4 +21,7 @@ .side-panel-container { display: flex; + .copy-from { + border-left: 1px solid #d7d7dc; + } } From c1fdb4d6a75ec5cbf82c7a3140d6d5a09886af31 Mon Sep 17 00:00:00 2001 From: Kevin Nderitu Date: Thu, 25 Mar 2021 15:38:19 +0300 Subject: [PATCH 27/48] Update now happens if mimetype is not undefined --- .../36-populate-mimetype-on-attachments/index.js | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js index c0cb887e03..db2e54dd8a 100644 --- a/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js +++ b/app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js @@ -21,11 +21,18 @@ export default { // eslint-disable-next-line no-await-in-loop const response = await request.head(file.url); const mimetype = response.headers.get('content-type') || undefined; - db.collection('files').updateOne({ _id: file._id }, { $set: { mimetype } }); + // eslint-disable-next-line no-await-in-loop + await this.updateFile(db, file, mimetype); } else if (file.filename && file.type === 'attachment' && !file.mimetype) { const mimetype = mime.lookup(attachmentsPath(file.filename)) || undefined; - db.collection('files').updateOne({ _id: file._id }, { $set: { mimetype } }); + // eslint-disable-next-line no-await-in-loop + await this.updateFile(db, file, mimetype); } } }, + async updateFile(db, file, mimetype) { + if (mimetype) { + await db.collection('files').updateOne({ _id: file._id }, { $set: { mimetype } }); + } + }, }; From eeeb2d068ea79ec1a75926f7ba7ff2146544db10 Mon Sep 17 00:00:00 2001 From: Daneryl Date: Tue, 23 Mar 2021 12:36:05 +0100 Subject: [PATCH 28/48] silence stdr on migration 33 --- .../specs/33-character-count-to-absolute-position.spec.js | 1 + 1 file changed, 1 insertion(+) diff --git a/app/api/migrations/migrations/33-character-count-to-absolute-position/specs/33-character-count-to-absolute-position.spec.js b/app/api/migrations/migrations/33-character-count-to-absolute-position/specs/33-character-count-to-absolute-position.spec.js index 65c071e3da..4a1e454a74 100644 --- a/app/api/migrations/migrations/33-character-count-to-absolute-position/specs/33-character-count-to-absolute-position.spec.js +++ b/app/api/migrations/migrations/33-character-count-to-absolute-position/specs/33-character-count-to-absolute-position.spec.js @@ -20,6 +20,7 @@ import migration from '../index.js'; describe('conversion of character count to absolute position', () => { beforeEach(done => { spyOn(process.stdout, 'write'); + spyOn(process.stderr, 'write'); spyOn(errorLog, 'error'); config.defaultTenant.uploadedDocuments = __dirname; testingDB From dfc1b05b63be17df50d6339c2bb48540185358ab Mon Sep 17 00:00:00 2001 From: Daneryl Date: Tue, 23 Mar 2021 12:55:46 +0100 Subject: [PATCH 29/48] removed react deprecations warnings on tests --- app/setUpJestClient.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/app/setUpJestClient.js b/app/setUpJestClient.js index 84cd9e4dc3..de7e9af379 100644 --- a/app/setUpJestClient.js +++ b/app/setUpJestClient.js @@ -4,6 +4,14 @@ const Adapter = require('enzyme-adapter-react-16'); configure({ adapter: new Adapter() }); +const warn = console.warn.bind(console); +console.warn = function(message) { + if (message.match('UNSAFE_')) { + return; + } + warn(message); +}; + const error = console.error.bind(console); console.error = function(message) { if (message.match('/api/i18n/systemKeys')) { From 726e038e53407140319699c5e00f47e6b5f2f3c6 Mon Sep 17 00:00:00 2001 From: Daneryl Date: Tue, 23 Mar 2021 12:56:31 +0100 Subject: [PATCH 30/48] settings routes.spec using its own elastic index --- app/api/settings/specs/routes.spec.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/api/settings/specs/routes.spec.ts b/app/api/settings/specs/routes.spec.ts index 30324f82a5..25c774c0b6 100644 --- a/app/api/settings/specs/routes.spec.ts +++ b/app/api/settings/specs/routes.spec.ts @@ -23,7 +23,8 @@ describe('Settings routes', () => { beforeEach(async () => { spyOn(search, 'indexEntities').and.returnValue(Promise.resolve()); - await db.clearAllAndLoad(fixtures); + const elasticIndex = 'settings_index'; + await db.clearAllAndLoad(fixtures, elasticIndex); }); afterAll(async () => db.disconnect()); From 0c3a81846619f98b2ab76e9956b48a30d6823f7d Mon Sep 17 00:00:00 2001 From: Daneryl Date: Tue, 23 Mar 2021 12:59:19 +0100 Subject: [PATCH 31/48] templates.spec now uses its own elastic index --- app/api/templates/specs/templates.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/templates/specs/templates.spec.js b/app/api/templates/specs/templates.spec.js index 7a2ed9a319..5533f2c98f 100644 --- a/app/api/templates/specs/templates.spec.js +++ b/app/api/templates/specs/templates.spec.js @@ -21,7 +21,7 @@ import fixtures, { } from './fixtures.js'; describe('templates', () => { - const elasticIndex = 'index'; + const elasticIndex = 'templates_spec_index'; beforeEach(async () => { spyOn(translations, 'addContext').and.returnValue(Promise.resolve()); From f2405175a971b45c2c9cf9c1e4f26f0e23e02218 Mon Sep 17 00:00:00 2001 From: Daneryl Date: Tue, 23 Mar 2021 13:11:52 +0100 Subject: [PATCH 32/48] removed not helpfull output from errorLog.spec --- app/api/log/specs/errorLog.spec.js | 1 + 1 file changed, 1 insertion(+) diff --git a/app/api/log/specs/errorLog.spec.js b/app/api/log/specs/errorLog.spec.js index ffa1eee27a..87f67394b3 100644 --- a/app/api/log/specs/errorLog.spec.js +++ b/app/api/log/specs/errorLog.spec.js @@ -50,6 +50,7 @@ describe('errorLog', () => { process.env.LOGS_DIR = './some_dir'; const anErrorLog = createErrorLog(); + spyOn(anErrorLog.transports[1], 'log'); expect(anErrorLog.transports[0].dirname).toBe('./some_dir'); expect(anErrorLog.transports[0].filename).toBe('error.log'); From b5096b7c8853ff79a295319bc5984ac5aae54101 Mon Sep 17 00:00:00 2001 From: Daneryl Date: Tue, 23 Mar 2021 13:39:49 +0100 Subject: [PATCH 33/48] prevent unhandled rejections --- app/react/Forms/components/specs/Captcha.spec.js | 5 +++-- app/setUpJestClient.js | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/app/react/Forms/components/specs/Captcha.spec.js b/app/react/Forms/components/specs/Captcha.spec.js index 8bf595b76a..2d38cad33a 100644 --- a/app/react/Forms/components/specs/Captcha.spec.js +++ b/app/react/Forms/components/specs/Captcha.spec.js @@ -16,6 +16,7 @@ describe('Captcha', () => { }); const render = () => { + spyOn(api, 'get').and.returnValue(Promise.resolve({ json: { svg: '', id: 2 } })); component = shallow(); }; @@ -37,9 +38,9 @@ describe('Captcha', () => { }; render(); expect(component.find('div div').props().dangerouslySetInnerHTML).toEqual({ __html: '' }); - spyOn(api, 'get').and.returnValue(Promise.resolve({ json: { svg: 'captchasvg', id: 2 } })); - await refreshCaptcha(); + api.get.and.returnValue(Promise.resolve({ json: { svg: 'captchasvg', id: 2 } })); + await refreshCaptcha(); expect(component.find('div div').props().dangerouslySetInnerHTML).toEqual({ __html: 'captchasvg', }); diff --git a/app/setUpJestClient.js b/app/setUpJestClient.js index de7e9af379..534867dad6 100644 --- a/app/setUpJestClient.js +++ b/app/setUpJestClient.js @@ -22,6 +22,10 @@ console.error = function(message) { process.env.__testingEnvironment = true; +process.on('unhandledRejection', err => { + fail(err); +}); + jasmine.createSpyObj = (name, methodNames) => { let names = methodNames; if (Array.isArray(name)) { From 3d0f7a61910fe751f632067ff0a72646f329b85d Mon Sep 17 00:00:00 2001 From: Daneryl Date: Tue, 23 Mar 2021 13:51:38 +0100 Subject: [PATCH 34/48] remove deprecation warnings, await for promise --- .../migrations/34-move-attachments/index.js | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/app/api/migrations/migrations/34-move-attachments/index.js b/app/api/migrations/migrations/34-move-attachments/index.js index 5e1bb59c21..047f24d818 100644 --- a/app/api/migrations/migrations/34-move-attachments/index.js +++ b/app/api/migrations/migrations/34-move-attachments/index.js @@ -16,13 +16,15 @@ export default { await Promise.all( entity.attachments.map(async ({ filename, originalname }) => - db.collection('files').update( + db.collection('files').updateMany( { filename }, { - entity: entity.sharedId, - filename, - originalname, - type: 'attachment', + $set: { + entity: entity.sharedId, + filename, + originalname, + type: 'attachment', + }, }, { upsert: true } ) @@ -32,7 +34,7 @@ export default { process.stdout.write(`-> processed: ${index} \r`); index += 1; } - db.collection('entities').update({}, { $unset: { attachments: 1 } }, { multi: true }); + await db.collection('entities').updateMany({}, { $unset: { attachments: 1 } }, { multi: true }); process.stdout.write('\r\n'); }, }; From 640d12237c68cbe12a443693973056d02acf778b Mon Sep 17 00:00:00 2001 From: Daneryl Date: Tue, 23 Mar 2021 15:17:05 +0100 Subject: [PATCH 35/48] ensure order of attachments on the spec --- .../specs/34-move-attachments.spec.js | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/app/api/migrations/migrations/34-move-attachments/specs/34-move-attachments.spec.js b/app/api/migrations/migrations/34-move-attachments/specs/34-move-attachments.spec.js index 95b7e037a8..3ded470a37 100644 --- a/app/api/migrations/migrations/34-move-attachments/specs/34-move-attachments.spec.js +++ b/app/api/migrations/migrations/34-move-attachments/specs/34-move-attachments.spec.js @@ -22,14 +22,10 @@ describe('migration move-attachments', () => { const attachments = await testingDB.mongodb .collection('files') .find({ type: 'attachment' }) + .sort({ originalname: 1 }) .toArray(); expect(attachments).toEqual([ - expect.objectContaining({ - originalname: 'The chain', - filename: 'thechain.mp3', - entity: 'fleet_wood', - }), expect.objectContaining({ originalname: 'Dont let me down', filename: 'dontletmedown.mp3', @@ -40,6 +36,11 @@ describe('migration move-attachments', () => { filename: 'strangemagic.mp3', entity: 'electric_light_orchestra', }), + expect.objectContaining({ + originalname: 'The chain', + filename: 'thechain.mp3', + entity: 'fleet_wood', + }), ]); }); From f793dbad28955cc497a6809552c75a7c1f858a75 Mon Sep 17 00:00:00 2001 From: Daneryl Date: Tue, 23 Mar 2021 15:58:19 +0100 Subject: [PATCH 36/48] avoid file name collisions --- app/api/csv/specs/csvLoaderLanguages.spec.ts | 16 ++++++++-------- .../csv/typeParsers/specs/relationship.spec.js | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/app/api/csv/specs/csvLoaderLanguages.spec.ts b/app/api/csv/specs/csvLoaderLanguages.spec.ts index 1f014496e9..7f07de8453 100644 --- a/app/api/csv/specs/csvLoaderLanguages.spec.ts +++ b/app/api/csv/specs/csvLoaderLanguages.spec.ts @@ -40,7 +40,7 @@ describe('csvLoader languages', () => { 'testLanguages.zip' ); const csv = path.join(__dirname, 'zipData/testLanguages.zip'); - spyOn(filesystem, 'generateFileName').and.callFake(file => `generated${file.originalname}`); + spyOn(filesystem, 'generateFileName').and.callFake(file => `generatedLang${file.originalname}`); await loader.load(csv, template1Id, { language: 'en', user: {} }); imported = await entities.get(); @@ -50,8 +50,8 @@ describe('csvLoader languages', () => { const generatedImages = (await files.get({})).map(u => u._id.toString()); await filesystem.deleteFiles([ - filesystem.uploadsPath('generated1.pdf'), - filesystem.uploadsPath('generated2.pdf'), + filesystem.uploadsPath('generatedLang1.pdf'), + filesystem.uploadsPath('generatedLang2.pdf'), filesystem.uploadsPath(`${generatedImages[0]}.jpg`), filesystem.uploadsPath(`${generatedImages[1]}.jpg`), filesystem.uploadsPath(`${generatedImages[2]}.jpg`), @@ -84,7 +84,7 @@ describe('csvLoader languages', () => { it('should import translated files', async () => { const importedFiles = await files.get({ type: 'document' }); - expect(importedFiles.map(f => f.filename)).toEqual(['generated2.pdf', 'generated1.pdf']); + expect(importedFiles.map(f => f.filename)).toEqual(['generatedLang2.pdf', 'generatedLang1.pdf']); }); it('should import attachment files', async () => { @@ -98,19 +98,19 @@ describe('csvLoader languages', () => { expect(enAttachments).toEqual([ expect.objectContaining({ - filename: 'generated1.pdf', + filename: 'generatedLang1.pdf', }), expect.objectContaining({ - filename: 'generated2.pdf', + filename: 'generatedLang2.pdf', }), ]); expect(esAttachments).toEqual([ expect.objectContaining({ - filename: 'generated1.pdf', + filename: 'generatedLang1.pdf', }), expect.objectContaining({ - filename: 'generated2.pdf', + filename: 'generatedLang2.pdf', }), ]); }); diff --git a/app/api/csv/typeParsers/specs/relationship.spec.js b/app/api/csv/typeParsers/specs/relationship.spec.js index 585dc9ecfb..45debf1742 100644 --- a/app/api/csv/typeParsers/specs/relationship.spec.js +++ b/app/api/csv/typeParsers/specs/relationship.spec.js @@ -47,8 +47,6 @@ describe('relationship', () => { templateProp ); - afterAll(async () => db.disconnect()); - value2 = await typeParsers.relationship( { relationship_prop: 'value1|value2', language: 'en' }, templateProp @@ -73,6 +71,8 @@ describe('relationship', () => { entitiesRelated = await entities.get({ template: templateToRelateId, language: 'en' }); }); + afterAll(async () => db.disconnect()); + it('should create entities and return the ids', async () => { expect(entitiesRelated[0].title).toBe('value1'); expect(entitiesRelated[1].title).toBe('value3'); From a62414cdb97ea06100ccc4f930ce61fdd7d5c9c5 Mon Sep 17 00:00:00 2001 From: Daneryl Date: Tue, 23 Mar 2021 16:13:22 +0100 Subject: [PATCH 37/48] close open handles on specs --- app/api/auth/specs/captchaMiddleware.spec.js | 2 ++ app/api/search/specs/routes.spec.js | 2 ++ app/api/socketio/specs/socketClusterMode.spec.ts | 3 ++- app/api/templates/specs/utils.spec.ts | 2 ++ app/api/usergroups/specs/userGroups.spec.ts | 2 ++ 5 files changed, 10 insertions(+), 1 deletion(-) diff --git a/app/api/auth/specs/captchaMiddleware.spec.js b/app/api/auth/specs/captchaMiddleware.spec.js index 4dd1d94e3f..8cf18a3460 100644 --- a/app/api/auth/specs/captchaMiddleware.spec.js +++ b/app/api/auth/specs/captchaMiddleware.spec.js @@ -28,6 +28,8 @@ describe('captchaMiddleware', () => { .catch(catchErrors(done)); }); + afterAll(async () => db.disconnect()); + it('should return an error when there is no captcha in the request', async () => { const middleWare = captchaMiddleware(); await middleWare(req, res, next); diff --git a/app/api/search/specs/routes.spec.js b/app/api/search/specs/routes.spec.js index d558253832..a86f3dab4f 100644 --- a/app/api/search/specs/routes.spec.js +++ b/app/api/search/specs/routes.spec.js @@ -92,6 +92,8 @@ describe('search routes', () => { describe('/api/search_snippets', () => { const app = setUpApp(searchRoutes); + afterAll(async () => testingDB.disconnect()); + it('should have a validation schema', async () => { await testingDB.clearAllAndLoad({ settings: [ diff --git a/app/api/socketio/specs/socketClusterMode.spec.ts b/app/api/socketio/specs/socketClusterMode.spec.ts index 41756f95b7..594aea5c36 100644 --- a/app/api/socketio/specs/socketClusterMode.spec.ts +++ b/app/api/socketio/specs/socketClusterMode.spec.ts @@ -196,7 +196,8 @@ describe('socket middlewares setup', () => { }); it('should not fail when not sending a cookie', async () => { - await connectSocket(port, 'tenant5'); + const socket5 = await connectSocket(port, 'tenant5'); await requestTestRoute('tenant5', '/api/onlySender'); + socket5.disconnect(); }); }); diff --git a/app/api/templates/specs/utils.spec.ts b/app/api/templates/specs/utils.spec.ts index 286f21b232..c8f12aea12 100644 --- a/app/api/templates/specs/utils.spec.ts +++ b/app/api/templates/specs/utils.spec.ts @@ -14,6 +14,8 @@ describe('templates utils', () => { await db.clearAllAndLoad({}); }); + afterAll(async () => db.disconnect()); + describe('name generation', () => { describe('default name generation', () => { it('should sanitize the labels and append the type', async () => { diff --git a/app/api/usergroups/specs/userGroups.spec.ts b/app/api/usergroups/specs/userGroups.spec.ts index 7a7e77bf89..e6b6f65812 100644 --- a/app/api/usergroups/specs/userGroups.spec.ts +++ b/app/api/usergroups/specs/userGroups.spec.ts @@ -10,6 +10,8 @@ describe('userGroups', () => { await db.clearAllAndLoad(fixtures); }); + afterAll(async () => db.disconnect()); + describe('get', () => { it('should return populated user groups from model', async () => { const groups = await userGroups.get({}, '', { sort: { name: 1 } }); From d9aabf60af571d76cd4823a83cd496c1c4c57dbb Mon Sep 17 00:00:00 2001 From: Daneryl Date: Tue, 23 Mar 2021 20:55:58 +0100 Subject: [PATCH 38/48] isolate testing uploads path from each test --- app/api/csv/specs/csvLoaderLanguages.spec.ts | 7 +++-- app/api/csv/specs/csvLoaderZip.spec.ts | 2 +- app/api/files/filesystem.ts | 29 +++++++++++++++++-- app/api/files/specs/jsRoutes.spec.js | 10 ++++--- app/api/files/specs/publicRoutes.spec.ts | 2 +- app/api/files/specs/uploadRoutes.spec.ts | 3 +- app/api/sync/specs/uploadRoute.spec.ts | 4 +-- app/api/thesauri/specs/routes.spec.ts | 2 +- app/api/utils/async-fs.js | 1 + .../utils/specs/staticFilesMiddleware.spec.ts | 4 +-- app/api/utils/testing_db.ts | 14 ++------- 11 files changed, 48 insertions(+), 30 deletions(-) diff --git a/app/api/csv/specs/csvLoaderLanguages.spec.ts b/app/api/csv/specs/csvLoaderLanguages.spec.ts index 7f07de8453..1190003264 100644 --- a/app/api/csv/specs/csvLoaderLanguages.spec.ts +++ b/app/api/csv/specs/csvLoaderLanguages.spec.ts @@ -25,7 +25,7 @@ describe('csvLoader languages', () => { beforeAll(async () => { await db.clearAllAndLoad(fixtures); - filesystem.setupTestUploadedPaths(); + await filesystem.setupTestUploadedPaths('csvLoader'); spyOn(search, 'indexEntities').and.returnValue(Promise.resolve()); const { languages = [] } = await settings.get(); @@ -84,7 +84,10 @@ describe('csvLoader languages', () => { it('should import translated files', async () => { const importedFiles = await files.get({ type: 'document' }); - expect(importedFiles.map(f => f.filename)).toEqual(['generatedLang2.pdf', 'generatedLang1.pdf']); + expect(importedFiles.map(f => f.filename)).toEqual([ + 'generatedLang2.pdf', + 'generatedLang1.pdf', + ]); }); it('should import attachment files', async () => { diff --git a/app/api/csv/specs/csvLoaderZip.spec.ts b/app/api/csv/specs/csvLoaderZip.spec.ts index 3fdc64fc59..413caf054f 100644 --- a/app/api/csv/specs/csvLoaderZip.spec.ts +++ b/app/api/csv/specs/csvLoaderZip.spec.ts @@ -26,7 +26,7 @@ describe('csvLoader zip file', () => { const zip = path.join(__dirname, '/zipData/test.zip'); const loader = new CSVLoader(); await db.clearAllAndLoad(fixtures); - filesystem.setupTestUploadedPaths(); + await filesystem.setupTestUploadedPaths('csvLoaderZip'); await createTestingZip( [ path.join(__dirname, '/zipData/test.csv'), diff --git a/app/api/files/filesystem.ts b/app/api/files/filesystem.ts index 3bded01ec3..c320f772e9 100644 --- a/app/api/files/filesystem.ts +++ b/app/api/files/filesystem.ts @@ -47,12 +47,34 @@ async function deleteFiles(files: FilePath[]) { const testingUploadPaths = { uploadedDocuments: `${__dirname}/specs/uploads/`, attachments: `${__dirname}/specs/uploads/`, - customUploads: `${__dirname}/specs/customUploads/`, + customUploads: `${__dirname}/specs/uploads/`, temporalFiles: `${__dirname}/specs/uploads/`, }; -const setupTestUploadedPaths = () => { - testingTenants.changeCurrentTenant(testingUploadPaths); +const createDirIfNotExists = async (dirPath: string) => { + try { + await asyncFS.mkdir(dirPath); + } catch (e) { + if (!e.message.match(/file already exists/)) { + throw e; + } + } +}; + +const generateUploadsPath = async (subPath: string) => { + if (subPath) { + await createDirIfNotExists(`${__dirname}/specs/uploads/${subPath}`); + } + return { + uploadedDocuments: `${__dirname}/specs/uploads/${subPath}`, + attachments: `${__dirname}/specs/uploads/${subPath}`, + customUploads: `${__dirname}/specs/uploads/${subPath}`, + temporalFiles: `${__dirname}/specs/uploads/${subPath}`, + }; +}; + +const setupTestUploadedPaths = async (subFolder: string = '') => { + testingTenants.changeCurrentTenant(await generateUploadsPath(subFolder)); }; const deleteUploadedFiles = async (files: FileType[]) => @@ -124,6 +146,7 @@ export { setupTestUploadedPaths, testingUploadPaths, deleteUploadedFiles, + createDirIfNotExists, deleteFiles, deleteFile, generateFileName, diff --git a/app/api/files/specs/jsRoutes.spec.js b/app/api/files/specs/jsRoutes.spec.js index 2323fc46d3..58f3a6ae05 100644 --- a/app/api/files/specs/jsRoutes.spec.js +++ b/app/api/files/specs/jsRoutes.spec.js @@ -14,6 +14,7 @@ import { fixtures, templateId } from './fixtures'; import instrumentRoutes from '../../utils/instrumentRoutes'; import uploadRoutes from '../jsRoutes.js'; import errorLog from '../../log/errorLog'; +import { createDirIfNotExists } from '../filesystem'; const mockExport = jest.fn(); jest.mock('api/csv/csvExporter', () => @@ -26,8 +27,9 @@ describe('upload routes', () => { let req; let file; - const deleteAllFiles = cb => { - const directory = `${__dirname}/uploads/`; + const deleteAllFiles = async cb => { + const directory = `${__dirname}/uploads/upload_routes`; + await createDirIfNotExists(directory); const dontDeleteFiles = [ 'import.zip', 'eng.pdf', @@ -51,8 +53,8 @@ describe('upload routes', () => { }); }; - beforeEach(done => { - deleteAllFiles(() => { + beforeEach(async done => { + await deleteAllFiles(() => { spyOn(search, 'delete').and.returnValue(Promise.resolve()); spyOn(search, 'indexEntities').and.returnValue(Promise.resolve()); routes = instrumentRoutes(uploadRoutes); diff --git a/app/api/files/specs/publicRoutes.spec.ts b/app/api/files/specs/publicRoutes.spec.ts index c01680a183..e97dd6236f 100644 --- a/app/api/files/specs/publicRoutes.spec.ts +++ b/app/api/files/specs/publicRoutes.spec.ts @@ -43,7 +43,7 @@ describe('public routes', () => { spyOn(Date, 'now').and.returnValue(1000); spyOn(errorLog, 'error'); await db.clearAllAndLoad(fixtures); - setupTestUploadedPaths(); + await setupTestUploadedPaths(); }); afterAll(async () => db.disconnect()); diff --git a/app/api/files/specs/uploadRoutes.spec.ts b/app/api/files/specs/uploadRoutes.spec.ts index 3cb15c61dd..8abde0ea97 100644 --- a/app/api/files/specs/uploadRoutes.spec.ts +++ b/app/api/files/specs/uploadRoutes.spec.ts @@ -35,8 +35,7 @@ describe('upload routes', () => { spyOn(Date, 'now').and.returnValue(1000); spyOn(errorLog, 'error'); //just to avoid annoying console output await db.clearAllAndLoad(fixtures); - - setupTestUploadedPaths(); + await setupTestUploadedPaths(); }); afterAll(async () => db.disconnect()); diff --git a/app/api/sync/specs/uploadRoute.spec.ts b/app/api/sync/specs/uploadRoute.spec.ts index fb07db8ab2..38eec682d0 100644 --- a/app/api/sync/specs/uploadRoute.spec.ts +++ b/app/api/sync/specs/uploadRoute.spec.ts @@ -28,9 +28,9 @@ jest.mock( describe('sync', () => { describe('sync/upload', () => { - beforeAll(() => { + beforeAll(async () => { testingTenants.mockCurrentTenant({}); - setupTestUploadedPaths(); + await setupTestUploadedPaths('sync'); }); afterAll(() => { diff --git a/app/api/thesauri/specs/routes.spec.ts b/app/api/thesauri/specs/routes.spec.ts index 24d9192581..1a8999fc9b 100644 --- a/app/api/thesauri/specs/routes.spec.ts +++ b/app/api/thesauri/specs/routes.spec.ts @@ -29,7 +29,7 @@ describe('Thesauri routes', () => { spyOn(Date, 'now').and.returnValue(1000); spyOn(errorLog, 'error'); //just to avoid annoying console output await db.clearAllAndLoad(fixtures); - setupTestUploadedPaths(); + await setupTestUploadedPaths(); }); afterAll(async () => db.disconnect()); diff --git a/app/api/utils/async-fs.js b/app/api/utils/async-fs.js index 2e309effad..396fedac43 100644 --- a/app/api/utils/async-fs.js +++ b/app/api/utils/async-fs.js @@ -11,4 +11,5 @@ export default { rename: promisify(fs.rename), readFile: promisify(fs.readFile), readdir: promisify(fs.readdir), + mkdir: promisify(fs.mkdir), }; diff --git a/app/api/utils/specs/staticFilesMiddleware.spec.ts b/app/api/utils/specs/staticFilesMiddleware.spec.ts index ef9b5e9c9a..206fa24f2e 100644 --- a/app/api/utils/specs/staticFilesMiddleware.spec.ts +++ b/app/api/utils/specs/staticFilesMiddleware.spec.ts @@ -8,9 +8,9 @@ describe('static file middleware', () => { const app: Application = express(); app.get('/static-files/:fileName', staticFilesMiddleware([uploadsPath, attachmentsPath])); - beforeEach(() => { + beforeEach(async () => { testingTenants.mockCurrentTenant({ name: 'default' }); - setupTestUploadedPaths(); + await setupTestUploadedPaths(); }); it('should return file requested', async () => { diff --git a/app/api/utils/testing_db.ts b/app/api/utils/testing_db.ts index ac029e82b1..3a157660dc 100644 --- a/app/api/utils/testing_db.ts +++ b/app/api/utils/testing_db.ts @@ -4,8 +4,7 @@ import { Db, ObjectId } from 'mongodb'; import { FileType } from 'shared/types/fileType'; import { EntitySchema } from 'shared/types/entityType'; import { DB } from 'api/odm'; -import { tenants } from 'api/tenants/tenantContext'; -import { setupTestUploadedPaths, testingUploadPaths } from 'api/files/filesystem'; +import { setupTestUploadedPaths } from 'api/files/filesystem'; import { ThesaurusSchema } from 'shared/types/thesaurusType'; import { elasticTesting } from './elastic_testing'; import { testingTenants } from './testingTenants'; @@ -78,21 +77,12 @@ const testingDB: { this.dbName = await mongod.getDbName(); if (options.defaultTenant) { - tenants.add( - testingTenants.createTenant({ - name: this.dbName, - dbName: this.dbName, - indexName: 'index', - ...testingUploadPaths, - }) - ); - testingTenants.mockCurrentTenant({ name: this.dbName, dbName: this.dbName, indexName: 'index', }); - setupTestUploadedPaths(); + await setupTestUploadedPaths(); } } From 43a920194d66a139d8686fce9b6f48a58195454d Mon Sep 17 00:00:00 2001 From: Daneryl Date: Wed, 24 Mar 2021 20:59:26 +0100 Subject: [PATCH 39/48] revert customUploads as testing path --- app/api/files/filesystem.ts | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/app/api/files/filesystem.ts b/app/api/files/filesystem.ts index c320f772e9..9777d642af 100644 --- a/app/api/files/filesystem.ts +++ b/app/api/files/filesystem.ts @@ -44,13 +44,6 @@ async function deleteFiles(files: FilePath[]) { return Promise.all(files.map(async file => deleteFile(file))); } -const testingUploadPaths = { - uploadedDocuments: `${__dirname}/specs/uploads/`, - attachments: `${__dirname}/specs/uploads/`, - customUploads: `${__dirname}/specs/uploads/`, - temporalFiles: `${__dirname}/specs/uploads/`, -}; - const createDirIfNotExists = async (dirPath: string) => { try { await asyncFS.mkdir(dirPath); @@ -64,11 +57,12 @@ const createDirIfNotExists = async (dirPath: string) => { const generateUploadsPath = async (subPath: string) => { if (subPath) { await createDirIfNotExists(`${__dirname}/specs/uploads/${subPath}`); + await createDirIfNotExists(`${__dirname}/specs/customUploads/${subPath}`); } return { uploadedDocuments: `${__dirname}/specs/uploads/${subPath}`, attachments: `${__dirname}/specs/uploads/${subPath}`, - customUploads: `${__dirname}/specs/uploads/${subPath}`, + customUploads: `${__dirname}/specs/customUploads/${subPath}`, temporalFiles: `${__dirname}/specs/uploads/${subPath}`, }; }; @@ -144,7 +138,6 @@ const getFileContent = async (fileName: FilePath): Promise => export { setupTestUploadedPaths, - testingUploadPaths, deleteUploadedFiles, createDirIfNotExists, deleteFiles, From 4cc5e212c0ddbcb44058d80c843e2bb38290141a Mon Sep 17 00:00:00 2001 From: Daneryl Date: Wed, 24 Mar 2021 21:00:10 +0100 Subject: [PATCH 40/48] properly await async function --- app/api/files/specs/jsRoutes.spec.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/app/api/files/specs/jsRoutes.spec.js b/app/api/files/specs/jsRoutes.spec.js index 58f3a6ae05..30a20cef9b 100644 --- a/app/api/files/specs/jsRoutes.spec.js +++ b/app/api/files/specs/jsRoutes.spec.js @@ -26,10 +26,9 @@ describe('upload routes', () => { let routes; let req; let file; + const directory = `${__dirname}/uploads/upload_routes`; const deleteAllFiles = async cb => { - const directory = `${__dirname}/uploads/upload_routes`; - await createDirIfNotExists(directory); const dontDeleteFiles = [ 'import.zip', 'eng.pdf', @@ -54,6 +53,7 @@ describe('upload routes', () => { }; beforeEach(async done => { + await createDirIfNotExists(directory); await deleteAllFiles(() => { spyOn(search, 'delete').and.returnValue(Promise.resolve()); spyOn(search, 'indexEntities').and.returnValue(Promise.resolve()); @@ -84,8 +84,8 @@ describe('upload routes', () => { }); describe('api/public', () => { - beforeEach(done => { - deleteAllFiles(() => { + beforeEach(async done => { + await deleteAllFiles(() => { spyOn(Date, 'now').and.returnValue(1000); spyOn(mailer, 'send'); const buffer = fs.readFileSync(`${__dirname}/12345.test.pdf`); @@ -182,8 +182,8 @@ describe('upload routes', () => { }); }); - afterAll(done => { - deleteAllFiles(() => { + afterAll(async done => { + await deleteAllFiles(() => { db.disconnect().then(done); }); }); From d6a3528800e73081b6d4b6c0e103c3dc25cae6ca Mon Sep 17 00:00:00 2001 From: Daneryl Date: Thu, 25 Mar 2021 11:50:21 +0100 Subject: [PATCH 41/48] make watch work with babel-module-resolver --- app/api/csv/typeParsers.ts | 2 +- app/jest.client.config.js | 7 ++++++- app/jest.server.config.js | 8 ++++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/app/api/csv/typeParsers.ts b/app/api/csv/typeParsers.ts index 5ba89689f0..c8e0716f43 100644 --- a/app/api/csv/typeParsers.ts +++ b/app/api/csv/typeParsers.ts @@ -1,6 +1,6 @@ import url from 'url'; -import { RawEntity } from 'api/csv/entityRow.js'; +import { RawEntity } from 'api/csv/entityRow'; import { PropertySchema, MetadataObjectSchema } from 'shared/types/commonTypes'; import { ensure } from 'shared/tsUtils'; diff --git a/app/jest.client.config.js b/app/jest.client.config.js index 70ca123198..ed0923b8f0 100644 --- a/app/jest.client.config.js +++ b/app/jest.client.config.js @@ -1,4 +1,5 @@ -/** @format */ +// eslint-disable-next-line import/no-extraneous-dependencies +const { defaults } = require('jest-config'); module.exports = { name: 'client', @@ -7,8 +8,12 @@ module.exports = { testPathIgnorePatterns: [], testEnvironment: 'node', setupFilesAfterEnv: ['/setUpJestClient.js'], + moduleFileExtensions: [...defaults.moduleFileExtensions, 'd.ts'], moduleNameMapper: { '\\.(css|scss)$': 'identity-obj-proxy', + '^shared/(.*)': '/shared/$1', + '^app/(.*)': '/react/$1', + '^app/UI/(.*)': '/react/UI/$1', }, snapshotSerializers: ['enzyme-to-json/serializer'], }; diff --git a/app/jest.server.config.js b/app/jest.server.config.js index 353cffb072..59f8b55f2b 100644 --- a/app/jest.server.config.js +++ b/app/jest.server.config.js @@ -1,7 +1,15 @@ +// eslint-disable-next-line import/no-extraneous-dependencies +const { defaults } = require('jest-config'); + module.exports = { name: 'server', displayName: 'Server', testMatch: ['**/api/**/specs/*spec.(j|t)s?(x)', '**/shared/**/specs/*spec.(j|t)s?(x)'], testEnvironment: 'node', setupFilesAfterEnv: ['/setUpJestServer.js'], + moduleFileExtensions: [...defaults.moduleFileExtensions, 'd.ts'], + moduleNameMapper: { + '^api/(.*)': '/api/$1', + '^shared/(.*)': '/shared/$1', + }, }; From 0cdb26f64e738c5745fc307340ca1557c34f19e5 Mon Sep 17 00:00:00 2001 From: RafaPolit Date: Thu, 25 Mar 2021 16:49:09 -0500 Subject: [PATCH 42/48] Added custom title to translations in sync --- app/api/sync/processNamespaces.ts | 15 ++++++++++----- app/api/sync/specs/fixtures.js | 2 ++ app/api/sync/specs/syncWorker.spec.js | 3 ++- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/app/api/sync/processNamespaces.ts b/app/api/sync/processNamespaces.ts index 5f3d9e62c0..d5cde138e4 100644 --- a/app/api/sync/processNamespaces.ts +++ b/app/api/sync/processNamespaces.ts @@ -317,11 +317,16 @@ class ProcessNamespaces { templatesData.find(t => t._id.toString() === context.id.toString()) ); const templateConfigProperties = this.templatesConfig[context.id.toString()].properties; - const approvedKeys = [contextTemplate.name].concat( - (contextTemplate.properties || []) - .filter(p => templateConfigProperties.includes(p._id?.toString() || '')) - .map(p => p.label) - ); + const templateTitle = contextTemplate.commonProperties?.find(p => p.name === 'title') + ?.label; + + const approvedKeys = [contextTemplate.name, templateTitle] + .concat( + (contextTemplate.properties || []) + .filter(p => templateConfigProperties.includes(p._id?.toString() || '')) + .map(p => p.label) + ) + .filter(k => Boolean(k)); context.values = (context.values || []).filter((v: any) => approvedKeys.includes(v.key)); return context; diff --git a/app/api/sync/specs/fixtures.js b/app/api/sync/specs/fixtures.js index b5b31a547a..b243c2b47c 100644 --- a/app/api/sync/specs/fixtures.js +++ b/app/api/sync/specs/fixtures.js @@ -569,6 +569,7 @@ export default { { _id: template1, name: 'template1', + commonProperties: [{ label: 'Template Title', name: 'title' }], properties: [ { _id: template1Property1, @@ -724,6 +725,7 @@ export default { { key: 't1Relationship2L', value: 't1Relationship2T' }, { key: 't1Thesauri2SelectL', value: 't1Thesauri2SelectT' }, { key: 't1Thesauri3MultiSelectL', value: 't1Thesauri3MultiSelectT' }, + { key: 'Template Title', value: 'Template Title translated' }, ], }, { diff --git a/app/api/sync/specs/syncWorker.spec.js b/app/api/sync/specs/syncWorker.spec.js index 7f31ceec2b..50e3834b19 100644 --- a/app/api/sync/specs/syncWorker.spec.js +++ b/app/api/sync/specs/syncWorker.spec.js @@ -225,7 +225,7 @@ describe('syncWorker', () => { }); describe('thesauris (dictionaries collection)', () => { - it('should sync whitelisted thesauris through template configs (deleting even non whitelisted ones)', async () => { + it('should sync whitelisted thesauris through template configs (deleting non-whitelisted ones)', async () => { await syncWorkerWithConfig({ templates: { [template1.toString()]: [ @@ -338,6 +338,7 @@ describe('syncWorker', () => { { key: 'template1', value: 'template1T' }, { key: 't1Relationship1L', value: 't1Relationship1T' }, { key: 't1Thesauri3MultiSelectL', value: 't1Thesauri3MultiSelectT' }, + { key: 'Template Title', value: 'Template Title translated' }, ]); expect(contexts.find(c => c.id.toString() === template2.toString()).values).toEqual([ { key: 'template2', value: 'template2T' }, From 187fcd1ce44b3cabdb1b91acf43d592e47fd1379 Mon Sep 17 00:00:00 2001 From: RafaPolit Date: Fri, 26 Mar 2021 11:13:07 -0500 Subject: [PATCH 43/48] Created migration to resync translations --- .../37-resync-translations/index.js | 17 +++++++ .../specs/37-resync-translations.spec.js | 47 +++++++++++++++++++ .../37-resync-translations/specs/fixtures.js | 37 +++++++++++++++ 3 files changed, 101 insertions(+) create mode 100644 app/api/migrations/migrations/37-resync-translations/index.js create mode 100644 app/api/migrations/migrations/37-resync-translations/specs/37-resync-translations.spec.js create mode 100644 app/api/migrations/migrations/37-resync-translations/specs/fixtures.js diff --git a/app/api/migrations/migrations/37-resync-translations/index.js b/app/api/migrations/migrations/37-resync-translations/index.js new file mode 100644 index 0000000000..619f52d0b9 --- /dev/null +++ b/app/api/migrations/migrations/37-resync-translations/index.js @@ -0,0 +1,17 @@ +export default { + delta: 37, + + name: 'resync-tranlsations', + + description: 'Moves timestamp of current translation update logs forward to force resync', + + async up(db) { + process.stdout.write('Updating updatelogs of translations...\r\n'); + + await db + .collection('updatelogs') + .updateMany({ namespace: 'translations' }, { $set: { timestamp: Date.now() } }); + + process.stdout.write('Updatelogs updated\r\n'); + }, +}; diff --git a/app/api/migrations/migrations/37-resync-translations/specs/37-resync-translations.spec.js b/app/api/migrations/migrations/37-resync-translations/specs/37-resync-translations.spec.js new file mode 100644 index 0000000000..7d647ede6a --- /dev/null +++ b/app/api/migrations/migrations/37-resync-translations/specs/37-resync-translations.spec.js @@ -0,0 +1,47 @@ +import testingDB from 'api/utils/testing_db'; +import migration from '../index.js'; +import fixtures, { translation1, translation2, translation3, entity1 } from './fixtures'; + +describe('migration resync translations', () => { + let updatelogs; + + beforeEach(async () => { + await testingDB.clearAllAndLoad(fixtures); + spyOn(process.stdout, 'write'); + spyOn(Date, 'now').and.returnValue(1000); + await migration.up(testingDB.mongodb); + }); + + afterAll(async () => { + await testingDB.disconnect(); + }); + + const getUpdatelog = mongoId => updatelogs.find(l => l.mongoId.toString() === mongoId.toString()); + + const expectLog = (logEntry, [namespace, deleted, timestamp]) => { + expect(logEntry).toEqual(expect.objectContaining({ namespace, deleted, timestamp })); + }; + + it('should have a delta number', () => { + expect(migration.delta).toBe(37); + }); + + it('should update the translation updatelogs to current timestamp and not affect others', async () => { + updatelogs = await testingDB.mongodb + .collection('updatelogs') + .find({}) + .toArray(); + + expect(updatelogs.length).toBe(4); + + const logTranslation1 = getUpdatelog(translation1); + const logTranslation2 = getUpdatelog(translation2); + const logTranslation3 = getUpdatelog(translation3); + const logEntity1 = getUpdatelog(entity1); + + expectLog(logTranslation1, ['translations', false, 1000]); + expectLog(logTranslation2, ['translations', false, 1000]); + expectLog(logTranslation3, ['translations', true, 1000]); + expectLog(logEntity1, ['entities', true, 8]); + }); +}); diff --git a/app/api/migrations/migrations/37-resync-translations/specs/fixtures.js b/app/api/migrations/migrations/37-resync-translations/specs/fixtures.js new file mode 100644 index 0000000000..b57aaa5304 --- /dev/null +++ b/app/api/migrations/migrations/37-resync-translations/specs/fixtures.js @@ -0,0 +1,37 @@ +import testingDB from 'api/utils/testing_db'; + +const translation1 = testingDB.id(); +const translation2 = testingDB.id(); +const translation3 = testingDB.id(); +const entity1 = testingDB.id(); + +export default { + updatelogs: [ + { + mongoId: translation1, + namespace: 'translations', + deleted: false, + timestamp: 6, + }, + { + mongoId: entity1, + namespace: 'entities', + deleted: true, + timestamp: 8, + }, + { + mongoId: translation2, + namespace: 'translations', + deleted: false, + timestamp: 10, + }, + { + mongoId: translation3, + namespace: 'translations', + deleted: true, + timestamp: 12, + }, + ], +}; + +export { translation1, translation2, translation3, entity1 }; From 238249b77224cfe791ee9f95cbcb3f860c2d6c7e Mon Sep 17 00:00:00 2001 From: Daneryl Date: Tue, 30 Mar 2021 13:03:08 +0200 Subject: [PATCH 44/48] getDocument returns relations. --- .../Documents/components/DocumentSidePanel.js | 14 ++++++-------- app/react/Documents/helpers.js | 4 ++++ app/react/Layout/Item.js | 3 ++- app/react/Library/components/Doc.js | 3 ++- app/react/Viewer/PDFView.js | 10 +--------- app/react/Viewer/actions/routeActions.js | 5 ++++- .../Viewer/actions/specs/documentActions.spec.js | 2 +- app/react/Viewer/components/Connection.js | 6 +++++- app/react/Viewer/components/Document.js | 7 +++++-- app/react/Viewer/specs/PDFView.spec.js | 4 +--- 10 files changed, 31 insertions(+), 27 deletions(-) diff --git a/app/react/Documents/components/DocumentSidePanel.js b/app/react/Documents/components/DocumentSidePanel.js index 9142b14b69..2d40249c13 100644 --- a/app/react/Documents/components/DocumentSidePanel.js +++ b/app/react/Documents/components/DocumentSidePanel.js @@ -25,6 +25,7 @@ import { entityDefaultDocument } from 'shared/entityDefaultDocument'; import SearchText from './SearchText'; import ShowToc from './ShowToc'; import SnippetsTab from './SnippetsTab'; +import helpers from '../helpers'; export class DocumentSidePanel extends Component { constructor(props) { @@ -271,7 +272,8 @@ export class DocumentSidePanel extends Component { const TocForm = this.props.tocFormComponent; - const { attachments, documents, language, defaultDoc } = doc.toJS(); + const jsDoc = helpers.performantDocToJSWithoutRelations(doc); + const { attachments, documents, language, defaultDoc } = jsDoc; const isEntity = !documents || !documents.length; const defaultDocumentToC = @@ -404,16 +406,12 @@ export class DocumentSidePanel extends Component {
- + - +
diff --git a/app/react/Documents/helpers.js b/app/react/Documents/helpers.js index 434b6481cc..d9de7c0b2e 100644 --- a/app/react/Documents/helpers.js +++ b/app/react/Documents/helpers.js @@ -3,6 +3,10 @@ import moment from 'moment'; export default { + performantDocToJSWithoutRelations(doc) { + return doc.delete('relations').toJS(); + }, + prepareMetadata(doc, templates, thesauris) { const template = templates.find(t => t._id === doc.template); diff --git a/app/react/Layout/Item.js b/app/react/Layout/Item.js index 0bf9999dec..3c6fc35931 100644 --- a/app/react/Layout/Item.js +++ b/app/react/Layout/Item.js @@ -7,6 +7,7 @@ import prioritySortingCriteria from 'app/utils/prioritySortingCriteria'; import { FeatureToggle } from 'app/components/Elements/FeatureToggle'; import { FavoriteBanner } from 'app/Favorites'; +import helpers from 'app/Documents/helpers'; import { RowList, ItemFooter } from './Lists'; import DocumentLanguage from './DocumentLanguage'; @@ -36,7 +37,7 @@ export class Item extends Component { buttons, } = this.props; - const doc = this.props.doc.toJS(); + const doc = helpers.performantDocToJSWithoutRelations(this.props.doc); const Snippet = additionalText ? (
{additionalText}
diff --git a/app/react/Library/components/Doc.js b/app/react/Library/components/Doc.js index 1d4935db83..7a33eb1f11 100644 --- a/app/react/Library/components/Doc.js +++ b/app/react/Library/components/Doc.js @@ -10,6 +10,7 @@ import { Icon } from 'UI'; import { Item } from 'app/Layout'; import { is, Map } from 'immutable'; +import helpers from 'app/Documents/helpers'; export class Doc extends Component { shouldComponentUpdate(nextProps) { @@ -67,7 +68,7 @@ export class Doc extends Component { render() { const { className, additionalText, targetReference } = this.props; - const doc = this.props.doc.toJS(); + const doc = helpers.performantDocToJSWithoutRelations(this.props.doc); const { sharedId, file, processed } = doc; let itemConnections = null; diff --git a/app/react/Viewer/PDFView.js b/app/react/Viewer/PDFView.js index 10da6f701f..2cafb52fe6 100644 --- a/app/react/Viewer/PDFView.js +++ b/app/react/Viewer/PDFView.js @@ -64,15 +64,7 @@ class PDFView extends Component { const { ref } = this.props.location.query; if (ref) { const reference = doc.get('relations').find(r => r.get('_id') === ref); - this.context.store.dispatch( - activateReference( - reference.toJS(), - doc - .get('defaultDoc') - .get('pdfInfo') - .toJS() - ) - ); + this.context.store.dispatch(activateReference(reference.toJS())); } } diff --git a/app/react/Viewer/actions/routeActions.js b/app/react/Viewer/actions/routeActions.js index a3ca9a9fea..ae3cc7c897 100644 --- a/app/react/Viewer/actions/routeActions.js +++ b/app/react/Viewer/actions/routeActions.js @@ -47,7 +47,10 @@ export async function requestViewerState(requestParams, globalResources) { return [ setViewerState({ documentViewer: { - doc, + doc: { + ...doc, + relations: references, + }, references, relationTypes, rawText, diff --git a/app/react/Viewer/actions/specs/documentActions.spec.js b/app/react/Viewer/actions/specs/documentActions.spec.js index f53803be63..2d4e5dee50 100644 --- a/app/react/Viewer/actions/specs/documentActions.spec.js +++ b/app/react/Viewer/actions/specs/documentActions.spec.js @@ -233,7 +233,7 @@ describe('documentActions', () => { rows: [{ documents: [{ pdfInfo: 'processed pdf', _id: 'pdfReady' }] }], }), }) - .get(`${APIURL}entities?sharedId=docWithPDFNotRdy&omitRelationships=true`, { + .get(`${APIURL}entities?sharedId=docWithPDFNotRdy`, { body: JSON.stringify({ rows: [ { diff --git a/app/react/Viewer/components/Connection.js b/app/react/Viewer/components/Connection.js index d14ee2fe3f..d7095d5468 100644 --- a/app/react/Viewer/components/Connection.js +++ b/app/react/Viewer/components/Connection.js @@ -16,11 +16,15 @@ import { } from 'app/Viewer/actions/uiActions'; import { Item } from 'app/Layout'; import { createSelector } from 'reselect'; +import helpers from 'app/Documents/helpers'; const selectDoc = createSelector( s => s.documentViewer.targetDoc, s => s.documentViewer.doc, - (targetDoc, doc) => (targetDoc.get('_id') ? targetDoc.toJS() : doc.toJS()) + (targetDoc, doc) => + targetDoc.get('_id') + ? helpers.performantDocToJSWithoutRelations(targetDoc) + : helpers.performantDocToJSWithoutRelations(doc) ); export class Connection extends Component { diff --git a/app/react/Viewer/components/Document.js b/app/react/Viewer/components/Document.js index 275939cc42..19d1c2bc00 100644 --- a/app/react/Viewer/components/Document.js +++ b/app/react/Viewer/components/Document.js @@ -104,13 +104,16 @@ export class Document extends Component { } render() { - const doc = this.props.doc.toJS(); const { file } = this.props; const Header = this.props.header; return (
-
+
{ it('should activate text reference if query parameters have reference id', () => { spyOn(uiActions, 'activateReference'); props.location = { query: { raw: 'false', ref: 'refId' }, pathname: 'pathname' }; - const pdfInfo = { 1: { chars: 100 } }; const reference = { _id: 'refId', range: { start: 200, end: 300 }, text: 'test' }; const doc = fromJS({ - defaultDoc: { pdfInfo }, relations: [{ _id: 'otherRef' }, reference], }); render(); instance.onDocumentReady(doc); - expect(uiActions.activateReference).toHaveBeenCalledWith(reference, pdfInfo); + expect(uiActions.activateReference).toHaveBeenCalledWith(reference); }); it('should emit documentLoaded event', () => { From 518ff30fe0251414663d089b9ed5bee03755968d Mon Sep 17 00:00:00 2001 From: Santiago Date: Mon, 5 Apr 2021 13:35:28 -0300 Subject: [PATCH 45/48] Added hash to all node_modules js files --- webpack/config.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webpack/config.js b/webpack/config.js index 631f5b4e0d..28123fe7ef 100644 --- a/webpack/config.js +++ b/webpack/config.js @@ -32,7 +32,7 @@ module.exports = production => { path: outputPath, publicPath: '/', filename: `[name]${jsChunkHashName}.js`, - chunkFilename: '[name].bundle.js', + chunkFilename: `[name]${jsChunkHashName}.bundle.js`, }, resolve: { extensions: ['*', '.webpack.js', '.web.js', '.js', '.tsx', '.ts'], From fe4f13d5f57a29ee91616bff0c0c884800207634 Mon Sep 17 00:00:00 2001 From: Daneryl Date: Tue, 6 Apr 2021 08:42:31 +0200 Subject: [PATCH 46/48] 1.27.0-rc1 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 4ebaaf03a4..31bab9e523 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "uwazi", - "version": "1.26.0-rc1", + "version": "1.27.0-rc1", "description": "Uwazi is a free, open-source solution for organising, analysing and publishing your documents.", "keywords": [ "react" From bf4373862dbc39e73a0408987920a8e8a9452fb9 Mon Sep 17 00:00:00 2001 From: Daneryl Date: Tue, 6 Apr 2021 10:37:59 +0200 Subject: [PATCH 47/48] 1.27.0-rc2 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 31bab9e523..a76d00d016 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "uwazi", - "version": "1.27.0-rc1", + "version": "1.27.0-rc2", "description": "Uwazi is a free, open-source solution for organising, analysing and publishing your documents.", "keywords": [ "react" From 834285732c25a22b4c66d5c04bf76a350741535f Mon Sep 17 00:00:00 2001 From: Daneryl Date: Tue, 6 Apr 2021 11:00:48 +0200 Subject: [PATCH 48/48] 1.27.0-rc3 (only for rebuild) --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index a76d00d016..bb7e793389 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "uwazi", - "version": "1.27.0-rc2", + "version": "1.27.0-rc3", "description": "Uwazi is a free, open-source solution for organising, analysing and publishing your documents.", "keywords": [ "react"