toc_generation service

app/api/entities/entities.js

@@ -163,6 +163,10 @@ async function updateEntity(entity, _template) {
       if (typeof entity.template !== 'undefined') {
         d.template = entity.template;
       }
+
+      if (typeof entity.generatedToc !== 'undefined') {
+        d.generatedToc = entity.generatedToc;
+      }
       return model.save(d);
     })
   );

app/api/entities/entitiesModel.ts

@@ -15,6 +15,7 @@ const mongoSchema = new mongoose.Schema(
     title: { type: String, required: true },
     template: { type: mongoose.Schema.Types.ObjectId, ref: 'templates', index: true },
     published: Boolean,
+    generatedToc: Boolean,
     icon: new mongoose.Schema({
       _id: String,
       label: String,

app/api/entities/specs/entities.spec.js

@@ -340,14 +340,15 @@ describe('entities', () => {
       });
     });
 
-    describe('when published/template property changes', () => {
+    describe('when published/template/generatedToc property changes', () => {
       it('should replicate the change for all the languages', done => {
         const doc = {
           _id: batmanFinishesId,
           sharedId: 'shared',
           metadata: {},
           published: false,
           template: templateId,
+          generatedToc: true,
         };
 
         entities
@@ -364,15 +365,32 @@ describe('entities', () => {
             expect(docES.template).toBeDefined();
 
             expect(docES.published).toBe(false);
+            expect(docES.generatedToc).toBe(true);
             expect(docES.template.equals(templateId)).toBe(true);
             expect(docEN.published).toBe(false);
+            expect(docEN.generatedToc).toBe(true);
             expect(docEN.template.equals(templateId)).toBe(true);
             done();
           })
           .catch(catchErrors(done));
       });
     });
 
+    describe('when generatedToc is undefined', () => {
+      it('should not replicate the value to all languages', async () => {
+        const doc = { _id: batmanFinishesId, sharedId: 'shared', generatedToc: true };
+        await entities.save(doc, { language: 'en' });
+        await entities.save({ _id: batmanFinishesId, sharedId: 'shared' }, { language: 'en' });
+        const [docES, docEN] = await Promise.all([
+          entities.getById('shared', 'es'),
+          entities.getById('shared', 'en'),
+        ]);
+
+        expect(docES.generatedToc).toBe(true);
+        expect(docEN.generatedToc).toBe(true);
+      });
+    });
+
     it('should sync select/multiselect/dates/multidate/multidaterange/numeric', done => {
       const doc = {
         _id: syncPropertiesEntityId,

app/api/files/files.ts

@@ -1,6 +1,7 @@
 import { deleteUploadedFiles } from 'api/files/filesystem';
 import connections from 'api/relationships';
 import { search } from 'api/search';
+import entities from 'api/entities';
 
 import model from './filesModel';
 import { validateFile } from '../../shared/types/fileSchema';
@@ -30,4 +31,27 @@ export const files = {
 
     return toDeleteFiles;
   },
+
+  async tocReviewed(_id: string, language: string) {
+    const savedFile = await files.save({ _id, generatedToc: false });
+    const sameEntityFiles = await files.get({ entity: savedFile.entity }, { generatedToc: 1 });
+    const [entity] = await entities.get({
+      sharedId: savedFile.entity,
+    });
+
+    await entities.save(
+      {
+        _id: entity._id,
+        sharedId: entity.sharedId,
+        template: entity.template,
+        generatedToc: sameEntityFiles.reduce<boolean>(
+          (generated, file) => generated || Boolean(file.generatedToc),
+          false
+        ),
+      },
+      { user: {}, language }
+    );
+
+    return savedFile;
+  },
 };

app/api/files/routes.ts

@@ -57,6 +57,28 @@ export default (app: Application) => {
       .catch(next);
   });
 
+  app.post(
+    '/api/files/tocReviewed',
+    needsAuthorization(['admin', 'editor']),
+    validation.validateRequest({
+      properties: {
+        body: {
+          required: ['fileId'],
+          properties: {
+            fileId: { type: 'string' },
+          },
+        },
+      },
+    }),
+    async (req, res, next) => {
+      try {
+        res.json(await files.tocReviewed(req.body.fileId, req.language));
+      } catch (e) {
+        next(e);
+      }
+    }
+  );
+
   app.get(
     '/api/files/:filename',
     validation.validateRequest({

app/api/files/specs/fixtures.ts

@@ -5,22 +5,29 @@ const entityEnId = db.id();
 const uploadId = db.id();
 const uploadId2 = db.id();
 const templateId = db.id();
+const importTemplate = db.id();
 const fileName1 = 'f2082bf51b6ef839690485d7153e847a.pdf';
 
 const fixtures: DBFixture = {
   files: [
     {
       _id: uploadId,
-      entity: 'entity',
+      entity: 'sharedId1',
+      generatedToc: true,
       originalname: 'upload1',
       filename: fileName1,
       type: 'custom',
     },
     {
       _id: uploadId2,
-      entity: 'entity',
+      generatedToc: true,
+      entity: 'sharedId1',
       filename: 'fileNotInDisk',
     },
+    {
+      entity: 'sharedId1',
+      filename: 'fileWithoutTocFlag',
+    },
     { _id: db.id(), filename: 'fileNotOnDisk' },
     { _id: db.id(), originalname: 'upload2', type: 'custom' },
     { _id: db.id(), originalname: 'upload3', type: 'document' },
@@ -36,11 +43,21 @@ const fixtures: DBFixture = {
       sharedId: 'sharedId1',
       language: 'es',
       title: 'Gadgets 01 ES',
-      toc: [{ _id: db.id(), label: 'existingToc' }],
+      generatedToc: true,
+      template: templateId,
     },
-    { _id: entityEnId, sharedId: 'sharedId1', language: 'en', title: 'Gadgets 01 EN' },
+    {
+      _id: entityEnId,
+      template: templateId,
+      sharedId: 'sharedId1',
+      language: 'en',
+      title: 'Gadgets 01 EN',
+    },
+  ],
+  templates: [
+    { _id: templateId, default: true, name: 'mydoc', properties: [] },
+    { _id: importTemplate, default: true, name: 'import', properties: [] },
   ],
-  templates: [{ _id: templateId, default: true, name: 'mydoc', properties: [] }],
   settings: [
     {
       _id: db.id(),
@@ -51,4 +68,13 @@ const fixtures: DBFixture = {
   ],
 };
 
-export { fixtures, entityId, entityEnId, fileName1, uploadId, uploadId2, templateId };
+export {
+  fixtures,
+  entityId,
+  entityEnId,
+  fileName1,
+  uploadId,
+  uploadId2,
+  templateId,
+  importTemplate,
+};

app/api/files/specs/routes.spec.ts

@@ -12,6 +12,7 @@ import { FileType } from 'shared/types/fileType';
 import { fixtures, uploadId, uploadId2 } from './fixtures';
 import { files } from '../files';
 import uploadRoutes from '../routes';
+import entities from 'api/entities';
 
 jest.mock(
   '../../auth/authMiddleware.ts',
@@ -47,7 +48,7 @@ describe('files routes', () => {
     });
 
     it('should reindex all entities that are related to the saved file', async () => {
-      expect(search.indexEntities).toHaveBeenCalledWith({ sharedId: 'entity' }, '+fullText');
+      expect(search.indexEntities).toHaveBeenCalledWith({ sharedId: 'sharedId1' }, '+fullText');
     });
   });
 
@@ -85,7 +86,7 @@ describe('files routes', () => {
         .query({ _id: uploadId2.toString() });
 
       expect(search.indexEntities).toHaveBeenCalledWith(
-        { sharedId: { $in: ['entity'] } },
+        { sharedId: { $in: ['sharedId1'] } },
         '+fullText'
       );
     });
@@ -107,5 +108,36 @@ describe('files routes', () => {
 
       expect(response.body.errors[0].message).toBe('should be string');
     });
+
+    describe('api/files/tocReviewed', () => {
+      it('should set tocGenerated to false on the file', async () => {
+        const response: SuperTestResponse = await request(app)
+          .post('/api/files/tocReviewed')
+          .set('content-language', 'es')
+          .send({ fileId: uploadId.toString() });
+
+        const [file] = await files.get({ _id: uploadId });
+        expect(file.generatedToc).toBe(false);
+        expect(response.body.entity).toBe('sharedId1');
+      });
+
+      it('should set tocGenerated to false on the entity when all associated files are false', async () => {
+        await request(app)
+          .post('/api/files/tocReviewed')
+          .send({ fileId: uploadId.toString() })
+          .expect(200);
+
+        let [entity] = await entities.get({ sharedId: 'sharedId1' });
+        expect(entity.generatedToc).toBe(true);
+
+        await request(app)
+          .post('/api/files/tocReviewed')
+          .send({ fileId: uploadId2.toString() })
+          .expect(200);
+
+        [entity] = await entities.get({ sharedId: 'sharedId1' });
+        expect(entity.generatedToc).toBe(false);
+      });
+    });
   });
 });

app/api/files/specs/uploadRoutes.spec.ts

@@ -16,7 +16,7 @@ import { setUpApp, socketEmit, iosocket } from 'api/utils/testingRoutes';
 import { FileType } from 'shared/types/fileType';
 import entities from 'api/entities';
 
-import { fixtures, templateId } from './fixtures';
+import { fixtures, templateId, importTemplate } from './fixtures';
 import { files } from '../files';
 import uploadRoutes from '../routes';
 
@@ -72,7 +72,10 @@ describe('upload routes', () => {
       expect(iosocket.emit).toHaveBeenCalledWith('conversionStart', 'sharedId1');
       expect(iosocket.emit).toHaveBeenCalledWith('documentProcessed', 'sharedId1');
 
-      const [upload] = await files.get({ entity: 'sharedId1' }, '+fullText');
+      const [upload] = await files.get(
+        { originalname: 'f2082bf51b6ef839690485d7153e847a.pdf' },
+        '+fullText'
+      );
 
       expect(upload).toEqual(
         expect.objectContaining({
@@ -105,14 +108,14 @@ describe('upload routes', () => {
       it('should detect English documents and store the result', async () => {
         await uploadDocument('uploads/eng.pdf');
 
-        const [upload] = await files.get({ entity: 'sharedId1' });
+        const [upload] = await files.get({ originalname: 'eng.pdf' });
         expect(upload.language).toBe('eng');
       }, 10000);
 
       it('should detect Spanish documents and store the result', async () => {
         await uploadDocument('uploads/spn.pdf');
 
-        const [upload] = await files.get({ entity: 'sharedId1' });
+        const [upload] = await files.get({ originalname: 'spn.pdf' });
         expect(upload.language).toBe('spa');
       });
     });
@@ -126,7 +129,7 @@ describe('upload routes', () => {
             .attach('file', path.join(__dirname, 'uploads/invalid_document.txt'))
         );
 
-        const [upload] = await files.get({ entity: 'sharedId1' }, '+fullText');
+        const [upload] = await files.get({ originalname: 'invalid_document.txt' }, '+fullText');
         expect(upload.status).toBe('failed');
       });
 
@@ -176,15 +179,15 @@ describe('upload routes', () => {
       await socketEmit('IMPORT_CSV_END', async () =>
         request(app)
           .post('/api/import')
-          .field('template', templateId.toString())
+          .field('template', importTemplate.toString())
           .attach('file', `${__dirname}/uploads/importcsv.csv`)
       );
 
       expect(iosocket.emit).toHaveBeenCalledWith('IMPORT_CSV_START');
       expect(iosocket.emit).toHaveBeenCalledWith('IMPORT_CSV_PROGRESS', 1);
       expect(iosocket.emit).toHaveBeenCalledWith('IMPORT_CSV_PROGRESS', 2);
 
-      const imported = await entities.get({ template: templateId });
+      const imported = await entities.get({ template: importTemplate });
       expect(imported).toEqual([
         expect.objectContaining({ title: 'imported entity one' }),
         expect.objectContaining({ title: 'imported entity two' }),

app/api/search/deprecatedRoutes.js

@@ -1,6 +1,6 @@
 import Joi from 'joi';
 import entities from 'api/entities';
-import { searchSchema } from 'api/search/searchSchema';
+import { searchParamsSchema } from 'shared/types/searchParams';
 import { search } from './search';
 import { validation, parseQuery } from '../utils';
 
@@ -25,7 +25,7 @@ export default app => {
   app.get(
     '/api/search',
     parseQuery,
-    validation.validateRequest(searchSchema),
+    validation.validateRequest(searchParamsSchema),
 
     (req, res, next) => {
       const action = req.query.geolocation ? 'searchGeolocations' : 'search';

app/api/search/documentQueryBuilder.js

@@ -2,7 +2,7 @@
 
 import { preloadOptionsSearch } from 'shared/config';
 import filterToMatch, { multiselectFilter } from './metadataMatchers';
-import { propertyToAggregation } from './metadataAggregations';
+import { propertyToAggregation, generatedTocAggregations } from './metadataAggregations';
 
 export default function() {
   const baseQuery = {
@@ -231,6 +231,17 @@ export default function() {
       }
     },
 
+    customFilters(filters = {}) {
+      Object.keys(filters).forEach(key => {
+        if (filters[key].values.length) {
+          addFilter({
+            terms: { [key]: filters[key].values },
+          });
+        }
+      });
+      return this;
+    },
+
     filterMetadata(filters = []) {
       filters.forEach(filter => {
         const match = filterToMatch(filter, filter.suggested ? 'suggestedMetadata' : 'metadata');
@@ -241,6 +252,10 @@ export default function() {
       return this;
     },
 
+    generatedTOCAggregations() {
+      baseQuery.aggregations.all.aggregations.generatedToc = generatedTocAggregations(baseQuery);
+    },
+
     aggregations(properties, dictionaries) {
       properties.forEach(property => {
         baseQuery.aggregations.all.aggregations[property.name] = propertyToAggregation(