Merge remote-tracking branch 'origin/release'

app/api/auth/specs/captchaMiddleware.spec.js

@@ -28,6 +28,8 @@ describe('captchaMiddleware', () => {
       .catch(catchErrors(done));
   });
 
+  afterAll(async () => db.disconnect());
+
   it('should return an error when there is no captcha in the request', async () => {
     const middleWare = captchaMiddleware();
     await middleWare(req, res, next);

app/api/csv/specs/csvLoaderLanguages.spec.ts

@@ -25,7 +25,7 @@ describe('csvLoader languages', () => {
 
   beforeAll(async () => {
     await db.clearAllAndLoad(fixtures);
-    filesystem.setupTestUploadedPaths();
+    await filesystem.setupTestUploadedPaths('csvLoader');
     spyOn(search, 'indexEntities').and.returnValue(Promise.resolve());
 
     const { languages = [] } = await settings.get();
@@ -40,7 +40,7 @@ describe('csvLoader languages', () => {
       'testLanguages.zip'
     );
     const csv = path.join(__dirname, 'zipData/testLanguages.zip');
-    spyOn(filesystem, 'generateFileName').and.callFake(file => `generated${file.originalname}`);
+    spyOn(filesystem, 'generateFileName').and.callFake(file => `generatedLang${file.originalname}`);
     await loader.load(csv, template1Id, { language: 'en', user: {} });
 
     imported = await entities.get();
@@ -50,8 +50,8 @@ describe('csvLoader languages', () => {
     const generatedImages = (await files.get({})).map(u => u._id.toString());
 
     await filesystem.deleteFiles([
-      filesystem.uploadsPath('generated1.pdf'),
-      filesystem.uploadsPath('generated2.pdf'),
+      filesystem.uploadsPath('generatedLang1.pdf'),
+      filesystem.uploadsPath('generatedLang2.pdf'),
       filesystem.uploadsPath(`${generatedImages[0]}.jpg`),
       filesystem.uploadsPath(`${generatedImages[1]}.jpg`),
       filesystem.uploadsPath(`${generatedImages[2]}.jpg`),
@@ -84,7 +84,10 @@ describe('csvLoader languages', () => {
 
   it('should import translated files', async () => {
     const importedFiles = await files.get({ type: 'document' });
-    expect(importedFiles.map(f => f.filename)).toEqual(['generated2.pdf', 'generated1.pdf']);
+    expect(importedFiles.map(f => f.filename)).toEqual([
+      'generatedLang2.pdf',
+      'generatedLang1.pdf',
+    ]);
   });
 
   it('should import attachment files', async () => {
@@ -98,19 +101,19 @@ describe('csvLoader languages', () => {
 
     expect(enAttachments).toEqual([
       expect.objectContaining({
-        filename: 'generated1.pdf',
+        filename: 'generatedLang1.pdf',
       }),
       expect.objectContaining({
-        filename: 'generated2.pdf',
+        filename: 'generatedLang2.pdf',
       }),
     ]);
 
     expect(esAttachments).toEqual([
       expect.objectContaining({
-        filename: 'generated1.pdf',
+        filename: 'generatedLang1.pdf',
       }),
       expect.objectContaining({
-        filename: 'generated2.pdf',
+        filename: 'generatedLang2.pdf',
       }),
     ]);
   });

app/api/csv/specs/csvLoaderZip.spec.ts

@@ -26,7 +26,7 @@ describe('csvLoader zip file', () => {
     const zip = path.join(__dirname, '/zipData/test.zip');
     const loader = new CSVLoader();
     await db.clearAllAndLoad(fixtures);
-    filesystem.setupTestUploadedPaths();
+    await filesystem.setupTestUploadedPaths('csvLoaderZip');
     await createTestingZip(
       [
         path.join(__dirname, '/zipData/test.csv'),

app/api/csv/typeParsers.ts

@@ -1,6 +1,6 @@
 import url from 'url';
 
-import { RawEntity } from 'api/csv/entityRow.js';
+import { RawEntity } from 'api/csv/entityRow';
 import { PropertySchema, MetadataObjectSchema } from 'shared/types/commonTypes';
 import { ensure } from 'shared/tsUtils';
 

app/api/csv/typeParsers/specs/relationship.spec.js

@@ -47,8 +47,6 @@ describe('relationship', () => {
       templateProp
     );
 
-    afterAll(async () => db.disconnect());
-
     value2 = await typeParsers.relationship(
       { relationship_prop: 'value1|value2', language: 'en' },
       templateProp
@@ -73,6 +71,8 @@ describe('relationship', () => {
     entitiesRelated = await entities.get({ template: templateToRelateId, language: 'en' });
   });
 
+  afterAll(async () => db.disconnect());
+
   it('should create entities and return the ids', async () => {
     expect(entitiesRelated[0].title).toBe('value1');
     expect(entitiesRelated[1].title).toBe('value3');

app/api/files/filesystem.ts

@@ -44,15 +44,31 @@ async function deleteFiles(files: FilePath[]) {
   return Promise.all(files.map(async file => deleteFile(file)));
 }
 
-const testingUploadPaths = {
-  uploadedDocuments: `${__dirname}/specs/uploads/`,
-  attachments: `${__dirname}/specs/uploads/`,
-  customUploads: `${__dirname}/specs/customUploads/`,
-  temporalFiles: `${__dirname}/specs/uploads/`,
+const createDirIfNotExists = async (dirPath: string) => {
+  try {
+    await asyncFS.mkdir(dirPath);
+  } catch (e) {
+    if (!e.message.match(/file already exists/)) {
+      throw e;
+    }
+  }
 };
 
-const setupTestUploadedPaths = () => {
-  testingTenants.changeCurrentTenant(testingUploadPaths);
+const generateUploadsPath = async (subPath: string) => {
+  if (subPath) {
+    await createDirIfNotExists(`${__dirname}/specs/uploads/${subPath}`);
+    await createDirIfNotExists(`${__dirname}/specs/customUploads/${subPath}`);
+  }
+  return {
+    uploadedDocuments: `${__dirname}/specs/uploads/${subPath}`,
+    attachments: `${__dirname}/specs/uploads/${subPath}`,
+    customUploads: `${__dirname}/specs/customUploads/${subPath}`,
+    temporalFiles: `${__dirname}/specs/uploads/${subPath}`,
+  };
+};
+
+const setupTestUploadedPaths = async (subFolder: string = '') => {
+  testingTenants.changeCurrentTenant(await generateUploadsPath(subFolder));
 };
 
 const deleteUploadedFiles = async (files: FileType[]) =>
@@ -122,8 +138,8 @@ const getFileContent = async (fileName: FilePath): Promise<string> =>
 
 export {
   setupTestUploadedPaths,
-  testingUploadPaths,
   deleteUploadedFiles,
+  createDirIfNotExists,
   deleteFiles,
   deleteFile,
   generateFileName,

app/api/files/specs/jsRoutes.spec.js

@@ -14,6 +14,7 @@ import { fixtures, templateId } from './fixtures';
 import instrumentRoutes from '../../utils/instrumentRoutes';
 import uploadRoutes from '../jsRoutes.js';
 import errorLog from '../../log/errorLog';
+import { createDirIfNotExists } from '../filesystem';
 
 const mockExport = jest.fn();
 jest.mock('api/csv/csvExporter', () =>
@@ -25,9 +26,9 @@ describe('upload routes', () => {
   let routes;
   let req;
   let file;
+  const directory = `${__dirname}/uploads/upload_routes`;
 
-  const deleteAllFiles = cb => {
-    const directory = `${__dirname}/uploads/`;
+  const deleteAllFiles = async cb => {
     const dontDeleteFiles = [
       'import.zip',
       'eng.pdf',
@@ -51,8 +52,9 @@ describe('upload routes', () => {
     });
   };
 
-  beforeEach(done => {
-    deleteAllFiles(() => {
+  beforeEach(async done => {
+    await createDirIfNotExists(directory);
+    await deleteAllFiles(() => {
       spyOn(search, 'delete').and.returnValue(Promise.resolve());
       spyOn(search, 'indexEntities').and.returnValue(Promise.resolve());
       routes = instrumentRoutes(uploadRoutes);
@@ -82,8 +84,8 @@ describe('upload routes', () => {
   });
 
   describe('api/public', () => {
-    beforeEach(done => {
-      deleteAllFiles(() => {
+    beforeEach(async done => {
+      await deleteAllFiles(() => {
         spyOn(Date, 'now').and.returnValue(1000);
         spyOn(mailer, 'send');
         const buffer = fs.readFileSync(`${__dirname}/12345.test.pdf`);
@@ -180,8 +182,8 @@ describe('upload routes', () => {
     });
   });
 
-  afterAll(done => {
-    deleteAllFiles(() => {
+  afterAll(async done => {
+    await deleteAllFiles(() => {
       db.disconnect().then(done);
     });
   });

app/api/files/specs/publicRoutes.spec.ts

@@ -43,7 +43,7 @@ describe('public routes', () => {
     spyOn(Date, 'now').and.returnValue(1000);
     spyOn(errorLog, 'error');
     await db.clearAllAndLoad(fixtures);
-    setupTestUploadedPaths();
+    await setupTestUploadedPaths();
   });
 
   afterAll(async () => db.disconnect());

app/api/files/specs/uploadRoutes.spec.ts

@@ -35,8 +35,7 @@ describe('upload routes', () => {
     spyOn(Date, 'now').and.returnValue(1000);
     spyOn(errorLog, 'error'); //just to avoid annoying console output
     await db.clearAllAndLoad(fixtures);
-
-    setupTestUploadedPaths();
+    await setupTestUploadedPaths();
   });
 
   afterAll(async () => db.disconnect());

app/api/log/specs/errorLog.spec.js

@@ -50,6 +50,7 @@ describe('errorLog', () => {
     process.env.LOGS_DIR = './some_dir';
 
     const anErrorLog = createErrorLog();
+    spyOn(anErrorLog.transports[1], 'log');
 
     expect(anErrorLog.transports[0].dirname).toBe('./some_dir');
     expect(anErrorLog.transports[0].filename).toBe('error.log');

app/api/migrations/migrations/33-character-count-to-absolute-position/specs/33-character-count-to-absolute-position.spec.js

@@ -20,6 +20,7 @@ import migration from '../index.js';
 describe('conversion of character count to absolute position', () => {
   beforeEach(done => {
     spyOn(process.stdout, 'write');
+    spyOn(process.stderr, 'write');
     spyOn(errorLog, 'error');
     config.defaultTenant.uploadedDocuments = __dirname;
     testingDB

app/api/migrations/migrations/34-move-attachments/index.js

@@ -16,13 +16,15 @@ export default {
 
       await Promise.all(
         entity.attachments.map(async ({ filename, originalname }) =>
-          db.collection('files').update(
+          db.collection('files').updateMany(
             { filename },
             {
-              entity: entity.sharedId,
-              filename,
-              originalname,
-              type: 'attachment',
+              $set: {
+                entity: entity.sharedId,
+                filename,
+                originalname,
+                type: 'attachment',
+              },
             },
             { upsert: true }
           )
@@ -32,7 +34,7 @@ export default {
       process.stdout.write(`-> processed: ${index} \r`);
       index += 1;
     }
-    db.collection('entities').update({}, { $unset: { attachments: 1 } }, { multi: true });
+    await db.collection('entities').updateMany({}, { $unset: { attachments: 1 } }, { multi: true });
     process.stdout.write('\r\n');
   },
 };

app/api/migrations/migrations/34-move-attachments/specs/34-move-attachments.spec.js

@@ -22,14 +22,10 @@ describe('migration move-attachments', () => {
     const attachments = await testingDB.mongodb
       .collection('files')
       .find({ type: 'attachment' })
+      .sort({ originalname: 1 })
       .toArray();
 
     expect(attachments).toEqual([
-      expect.objectContaining({
-        originalname: 'The chain',
-        filename: 'thechain.mp3',
-        entity: 'fleet_wood',
-      }),
       expect.objectContaining({
         originalname: 'Dont let me down',
         filename: 'dontletmedown.mp3',
@@ -40,6 +36,11 @@ describe('migration move-attachments', () => {
         filename: 'strangemagic.mp3',
         entity: 'electric_light_orchestra',
       }),
+      expect.objectContaining({
+        originalname: 'The chain',
+        filename: 'thechain.mp3',
+        entity: 'fleet_wood',
+      }),
     ]);
   });
 

app/api/migrations/migrations/36-populate-mimetype-on-attachments/index.js

@@ -0,0 +1,38 @@
+import request from 'shared/JSONRequest';
+import { attachmentsPath } from 'api/files/filesystem';
+import mime from 'mime-types';
+
+export default {
+  delta: 36,
+
+  name: 'populate-mimetype-to-attachment',
+
+  description: 'Populates mimetype of an attachment from a url',
+
+  async up(db) {
+    process.stdout.write(`${this.name}...\r\n`);
+    const cursor = await db.collection('files').find({});
+
+    // eslint-disable-next-line no-await-in-loop
+    while (await cursor.hasNext()) {
+      // eslint-disable-next-line no-await-in-loop
+      const file = await cursor.next();
+      if (file.url && !file.mimetype) {
+        // eslint-disable-next-line no-await-in-loop
+        const response = await request.head(file.url);
+        const mimetype = response.headers.get('content-type') || undefined;
+        // eslint-disable-next-line no-await-in-loop
+        await this.updateFile(db, file, mimetype);
+      } else if (file.filename && file.type === 'attachment' && !file.mimetype) {
+        const mimetype = mime.lookup(attachmentsPath(file.filename)) || undefined;
+        // eslint-disable-next-line no-await-in-loop
+        await this.updateFile(db, file, mimetype);
+      }
+    }
+  },
+  async updateFile(db, file, mimetype) {
+    if (mimetype) {
+      await db.collection('files').updateOne({ _id: file._id }, { $set: { mimetype } });
+    }
+  },
+};