Merge branch 'development'

.github/PULL_REQUEST_TEMPLATE.md

@@ -2,6 +2,7 @@ Fixes issue #
 
 PR check list:
 
+- [ ] Rebase current DEV branch
 - [ ] Client test
 - [ ] Server test
 - [ ] End-to-end test

.gitignore

@@ -11,6 +11,9 @@ uwazi-fixtures
 cejil-fixtures
 conversions/*.json
 app/api/upload/specs/uploads/*
+!app/api/upload/specs/uploads/f2082bf51b6ef839690485d7153e847a.pdf
+!app/api/upload/specs/uploads/eng.pdf
+!app/api/upload/specs/uploads/spn.pdf
 app/api/attachments/specs/uploads/toDelete.txt
 uploaded_documents/*
 !uploads/index.html

app/api/attachments/routes.js

@@ -19,6 +19,36 @@ const storage = multer.diskStorage({
   }
 });
 
+const processSingleLanguage = (entity, req) => {
+  const addedFile = req.files[0];
+  addedFile._id = mongoose.Types.ObjectId();
+
+  entity.attachments = entity.attachments || [];
+  entity.attachments.push(addedFile);
+  return Promise.all([addedFile, entities.saveMultiple([entity])]);
+};
+
+const processAllLanguages = (entity, req) => {
+  return processSingleLanguage(entity, req)
+  .then(([addedFile]) => {
+    return Promise.all([addedFile, entities.get({sharedId: entity.sharedId, _id: {$ne: entity._id}})]);
+  })
+  .then(([addedFile, siblings]) => {
+    const genericAddedFile = Object.assign({}, addedFile);
+    delete genericAddedFile._id;
+
+    const additionalLanguageUpdates = [];
+
+    siblings.forEach(sibling => {
+      sibling.attachments = sibling.attachments || [];
+      sibling.attachments.push(genericAddedFile);
+      additionalLanguageUpdates.push(entities.saveMultiple([sibling]));
+    });
+
+    return Promise.all([addedFile, additionalLanguageUpdates]);
+  });
+};
+
 export default (app) => {
   let upload = multer({storage});
 
@@ -33,19 +63,12 @@ export default (app) => {
   });
 
   app.post('/api/attachments/upload', needsAuthorization(['admin', 'editor']), upload.any(), (req, res) => {
-    let addedFile;
-
     return entities.getById(req.body.entityId)
     .then(entity => {
-      entity.attachments = entity.attachments || [];
-
-      addedFile = req.files[0];
-      addedFile._id = mongoose.Types.ObjectId();
-
-      entity.attachments.push(addedFile);
-      return entities.saveMultiple([entity]);
+      return req.body.allLanguages === 'true' ? processAllLanguages(entity, req) :
+                                                processSingleLanguage(entity, req);
     })
-    .then(() => {
+    .then(([addedFile]) => {
       res.json(addedFile);
     })
     .catch(error => res.json({error}));
@@ -82,10 +105,17 @@ export default (app) => {
     return entities.getById(req.query.entityId)
     .then(entity => {
       entity.attachments = (entity.attachments || []).filter(a => a.filename !== req.query.filename);
-      return entities.saveMultiple([entity]);
+      return Promise.all([entities.saveMultiple([entity]), entities.get({sharedId: entity.sharedId, _id: {$ne: entity._id}}, {attachments: 1})]);
     })
-    .then(response => {
-      return new Promise((resolve, reject) => {
+    .then(([response, siblings]) => {
+      const shouldUnlink = siblings.reduce((memo, sibling) => {
+        if (sibling.attachments && sibling.attachments.find(a => a.filename === req.query.filename)) {
+          return false;
+        }
+        return memo;
+      }, true);
+
+      return !shouldUnlink ? res.json(response[0]) : new Promise((resolve, reject) => {
         fs.unlink(attachmentsPath + req.query.filename, (err) => {
           if (err) {
             reject(err);

app/api/attachments/specs/fixtures.js

@@ -1,18 +1,27 @@
+/* eslint-disable max-len */
 import db from 'api/utils/testing_db';
 
 const entityId = db.id();
+const entityIdEn = db.id();
+const entityIdPt = db.id();
+const sharedId = 'sharedId';
 const toDeleteId = db.id();
 const attachmentToEdit = db.id();
 
 export default {
   entities: [
-    {_id: toDeleteId, attachments: [{filename: 'other.doc'}, {filename: 'toDelete.txt', originalname: 'common name 1.not'}]},
-    {_id: entityId, file: {originalname: 'source doc', filename: 'filename'}, attachments: [{_id: db.id(), originalname: 'o1', filename: 'other.doc'}, {_id: attachmentToEdit, filename: 'match.doc', originalname: 'common name 2.not'}]}
+    {sharedId: toDeleteId.toString(), _id: toDeleteId, attachments: [{filename: 'other.doc'}, {filename: 'toDelete.txt', originalname: 'common name 1.not'}]},
+    {sharedId, _id: entityId, file: {originalname: 'source doc', filename: 'filename'}, attachments: [{_id: db.id(), originalname: 'o1', filename: 'other.doc'}, {_id: attachmentToEdit, filename: 'match.doc', originalname: 'common name 2.not'}]},
+    {sharedId, _id: entityIdEn, file: {originalname: 'source doc', filename: 'filenameEn'}, attachments: [{_id: db.id(), originalname: 'o1', filename: 'otherEn.doc'}]},
+    {sharedId, _id: entityIdPt, file: {originalname: 'source doc', filename: 'filenamePt'}}
   ]
 };
 
 export {
   entityId,
+  entityIdEn,
+  entityIdPt,
+  sharedId,
   toDeleteId,
   attachmentToEdit
 };

app/api/attachments/specs/routes.spec.js

@@ -6,7 +6,7 @@ import paths, {attachmentsPath} from '../../config/paths';
 
 import entities from '../../entities';
 import attachmentsRoutes from '../routes';
-import fixtures, {entityId, toDeleteId, attachmentToEdit} from './fixtures';
+import fixtures, {entityId, entityIdEn, entityIdPt, toDeleteId, attachmentToEdit} from './fixtures';
 import db from 'api/utils/testing_db';
 
 describe('Attachments Routes', () => {
@@ -73,6 +73,43 @@ describe('Attachments Routes', () => {
         expect(dbEntity.attachments[2].originalname).toEqual(file.originalname);
         expect(addedFile.filename).toBe('mockfile.doc');
         expect(addedFile._id).toBeDefined();
+        expect(addedFile._id.toString()).toBe(dbEntity.attachments[2]._id.toString());
+        done();
+      })
+      .catch(catchErrors(done));
+    });
+
+    it('should add the uploaded file to all shared entities and return the file, incluiding its new ID', (done) => {
+      req.body.allLanguages = 'true';
+
+      routes.post('/api/attachments/upload', req)
+      .then(addedFile => {
+        return Promise.all([addedFile, entities.get({sharedId: 'sharedId'})]);
+      })
+      .then(([addedFile, dbEntities]) => {
+        const dbEntity = dbEntities.find(e => e._id.toString() === entityId.toString());
+        const dbEntityEn = dbEntities.find(e => e._id.toString() === entityIdEn.toString());
+        const dbEntityPt = dbEntities.find(e => e._id.toString() === entityIdPt.toString());
+
+        expect(dbEntity.attachments.length).toBe(3);
+        expect(dbEntity.attachments[2].filename).toBe(file.filename);
+        expect(dbEntity.attachments[2].originalname).toBe(file.originalname);
+        expect(dbEntity.attachments[2]._id.toString()).toBe(addedFile._id.toString());
+        expect(addedFile.filename).toBe('mockfile.doc');
+
+        expect(dbEntityEn.attachments.length).toBe(2);
+        expect(dbEntityEn.attachments[0].filename).toBe('otherEn.doc');
+        expect(dbEntityEn.file.filename).toBe('filenameEn');
+        expect(dbEntityEn.attachments[1].filename).toBe(file.filename);
+        expect(dbEntityEn.attachments[1].originalname).toBe(file.originalname);
+        expect(dbEntityEn.attachments[1]._id.toString()).not.toBe(addedFile._id.toString());
+
+        expect(dbEntityPt.attachments.length).toBe(1);
+        expect(dbEntityPt.file.filename).toBe('filenamePt');
+        expect(dbEntityPt.attachments[0].filename).toBe(file.filename);
+        expect(dbEntityPt.attachments[0].originalname).toBe(file.originalname);
+        expect(dbEntityPt.attachments[0]._id.toString()).not.toBe(addedFile._id.toString());
+
         done();
       })
       .catch(catchErrors(done));
@@ -152,6 +189,8 @@ describe('Attachments Routes', () => {
         return Promise.all([response, entities.getById(req.query.entityId)]);
       })
       .then(([response, dbEntity]) => {
+        expect(response._id.toString()).toBe(toDeleteId.toString());
+        expect(response.attachments.length).toBe(1);
         expect(dbEntity.attachments.length).toBe(1);
         expect(dbEntity.attachments[0].filename).toBe('other.doc');
         expect(fs.existsSync(paths.attachmentsPath + 'toDelete.txt')).toBe(false);
@@ -160,6 +199,25 @@ describe('Attachments Routes', () => {
       .catch(done.fail);
     });
 
+    it('should not delte the local file if other siblings are using it', (done) => {
+      expect(fs.existsSync(paths.attachmentsPath + 'toDelete.txt')).toBe(true);
+      const sibling = {sharedId: toDeleteId.toString(), attachments: [{filename: 'toDelete.txt', originalname: 'common name 1.not'}]};
+      entities.saveMultiple([sibling])
+      .then(() => {
+        return routes.delete('/api/attachments/delete', req);
+      })
+      .then(response => {
+        return Promise.all([response, entities.getById(req.query.entityId)]);
+      })
+      .then(([response, dbEntity]) => {
+        expect(response._id.toString()).toBe(toDeleteId.toString());
+        expect(dbEntity.attachments.length).toBe(1);
+        expect(fs.existsSync(paths.attachmentsPath + 'toDelete.txt')).toBe(true);
+        done();
+      })
+      .catch(done.fail);
+    });
+
     it('should not fail if, for some reason, file doesnt exist', (done) => {
       expect(fs.existsSync(paths.attachmentsPath + 'toDelete.txt')).toBe(true);
       fs.unlinkSync(paths.attachmentsPath + 'toDelete.txt');

app/api/documents/specs/documents.spec.js

@@ -30,7 +30,7 @@ describe('documents', () => {
         documents.get({sharedId: 'shared'})
         .then((docs) => {
           expect(docs[1].title).toBe('Penguin almost done');
-          expect(docs[1].fullText).not.toBeDefined();
+          expect(docs[1].file.fullText).not.toBeDefined();
           expect(docs[0].title).toBe('Batman finishes');
           done();
         })

app/api/documents/specs/fixtures.js

@@ -8,7 +8,7 @@ const referenceId = db.id();
 export default {
   entities: [
     {_id: batmanFinishesId, sharedId: 'shared', template: templateId, language: 'en', title: 'Batman finishes', published: true, user: {username: 'username'}, file: {filename: '8202c463d6158af8065022d9b5014cc1.pdf', originalname: 'Batman original.jpg'}},
-    {_id: db.id(), sharedId: 'shared', language: 'es', title: 'Penguin almost done', creationDate: 1, published: true, user: {username: 'username'}, file: {filename: '8202c463d6158af8065022d9b5014ccb.pdf'}},
+    {_id: db.id(), sharedId: 'shared', language: 'es', title: 'Penguin almost done', creationDate: 1, published: true, user: {username: 'username'}, file: {filename: '8202c463d6158af8065022d9b5014ccb.pdf', fullText: 'fullText'}},
     {
       _id: db.id(), sharedId: 'shared', language: 'pt', title: 'Penguin almost done', creationDate: 1, published: true, metadata: {text: 'test'},
       user: {username: 'username'}

app/api/entities/entitiesModel.js

@@ -34,6 +34,8 @@ import instanceModel from 'api/odm';
   *             type: string
   *           mimetype:
   *             type: string
+  *           language:
+  *             type: string
   *           size:
   *             type: integer
   *       icon:
@@ -99,7 +101,9 @@ const entitySchema = new mongoose.Schema({
     originalname: String,
     filename: String,
     mimetype: String,
-    size: Number
+    size: Number,
+    fullText: {type: mongoose.Schema.Types.Mixed, select: false},
+    language: String
   },
   icon: new mongoose.Schema({
     _id: String,
@@ -121,7 +125,6 @@ const entitySchema = new mongoose.Schema({
     size: Number
   }],
   creationDate: Number,
-  fullText: {type: mongoose.Schema.Types.Mixed, select: false},
   processed: Boolean,
   uploaded: Boolean,
   published: Boolean,

app/api/entities/specs/entities.spec.js

@@ -55,7 +55,7 @@ describe('entities', () => {
     });
 
     it('should return the newly created document for the passed language', (done) => {
-      let doc = {title: 'the dark knight', fullText: 'the full text!'};
+      let doc = {title: 'the dark knight', file: {fullText: 'the full text!'}};
       let user = {_id: db.id()};
 
       entities.save(doc, {user, language: 'en'})
@@ -64,7 +64,7 @@ describe('entities', () => {
         expect(createdDocument.title).toBe(doc.title);
         expect(createdDocument.user.equals(user._id)).toBe(true);
         expect(createdDocument.language).toEqual('en');
-        expect(createdDocument.fullText).not.toBeDefined();
+        expect(createdDocument.file.fullText).not.toBeDefined();
         done();
       })
       .catch(catchErrors(done));
@@ -466,13 +466,8 @@ describe('entities', () => {
   });
 
   describe('delete', () => {
-    beforeEach(() => {
-      fs.writeFileSync('./uploaded_documents/8202c463d6158af8065022d9b5014ccb.pdf');
-      fs.writeFileSync('./uploaded_documents/8202c463d6158af8065022d9b5014cc1.pdf');
-    });
-
     describe('when the original file does not exist', () => {
-      it('should delete the entiti and not throw an error', (done) => {
+      it('should delete the entity and not throw an error', (done) => {
         entities.delete('shared1')
         .then(() => entities.get({sharedId: 'shared1'}))
         .then((response) => {
@@ -516,6 +511,12 @@ describe('entities', () => {
     });
 
     it('should delete the original file', (done) => {
+      fs.writeFileSync('./uploaded_documents/8202c463d6158af8065022d9b5014ccb.pdf');
+      fs.writeFileSync('./uploaded_documents/8202c463d6158af8065022d9b5014cc1.pdf');
+
+      expect(fs.existsSync('./uploaded_documents/8202c463d6158af8065022d9b5014ccb.pdf')).toBe(true);
+      expect(fs.existsSync('./uploaded_documents/8202c463d6158af8065022d9b5014cc1.pdf')).toBe(true);
+
       entities.delete('shared')
       .then(() => {
         expect(fs.existsSync('./uploaded_documents/8202c463d6158af8065022d9b5014ccb.pdf')).toBe(false);

app/api/i18n/systemKeys.js

@@ -5,11 +5,12 @@ export default [
 {key: 'Add document type', label: 'Add document type'},
 {key: 'Add entity type', label: 'Add entity type'},
 {key: 'Add entity type', label: 'Add entity type'},
+{key: 'Add file', label: 'Add file'},
 {key: 'Add link', label: 'Add link'},
 {key: 'Add page', label: 'Add page'},
+{key: 'Add to all languages', label: 'Add to all languages'},
 {key: 'Add user', label: 'Add user'},
 {key: 'Add', label: 'Add'},
-{key: 'Add', label: 'Add'},
 {key: 'Add/delete users and assign roles', label: 'Add/delete users and assign roles'},
 {key: 'Add/edit translations', label: 'Add/edit translations'},
 {key: 'Admin', label: 'Admin'},
@@ -42,6 +43,7 @@ export default [
 {key: 'Custom page', label: 'Custom page'},
 {key: 'Date added', label: 'Date added'},
 {key: 'Date format', label: 'Date format'},
+{key: 'Downloads', label: 'Downloads'},
 {key: 'Duplicated name', label: 'Duplicated name'},
 {key: 'Separator', label: 'Separator'},
 {key: 'Order', label: 'Order'},
@@ -63,8 +65,11 @@ export default [
 {key: 'Entity types', label: 'Entity types'},
 {key: 'Filename label', label: 'filename'},
 {key: 'Filters', label: 'Filters'},
+{key: 'Filters AND operator', label: 'AND'},
+{key: 'Filters OR operator', label: 'OR'},
 {key: 'Flag', label: 'Flag'},
 {key: 'Forgot Password?', label: 'Forgot Password?'},
+{key: 'Google Analytics ID', label: 'Google Analytics ID'},
 {key: 'Icon', label: 'Icon'},
 {key: 'Info', label: 'Info'},
 {key: 'Info', label: 'Info'},

app/api/migrations/assign_file_languages.js

@@ -0,0 +1,43 @@
+import P from 'bluebird';
+import entities from '../entities/entitiesModel';
+import mongoose from 'mongoose';
+
+import languages from '../../shared/languages';
+
+entities.get({type: 'document'}, {_id: 1})
+.then(documents => {
+  return P.resolve(documents).map(({_id}) => {
+    return entities.get({_id})
+    .then(([doc]) => {
+      if (!doc.file || !doc.fullText) {
+        return;
+      }
+
+      doc.file.fullText = doc.file.fullText || doc.fullText;
+      doc.file.language = languages.detect(doc.file.fullText, 'franc');
+      delete doc.fullText;
+
+      return entities.save(doc)
+      .then(() => {
+        console.log(doc.title + ' Migrated!');
+      });
+    });
+  }, {concurrency: 1});
+})
+.then(() => {
+  entities.db.collection.update({},
+    {$unset: {fullText: true}},
+    {multi: true, safe: true},
+    (err) => {
+      if (err) {
+        throw err;
+      }
+      console.log('FullText deleted from collection\'s root!');
+      mongoose.disconnect();
+    }
+  );
+})
+.catch((e) => {
+  console.log(e);
+  mongoose.disconnect();
+});