diff --git a/app/api/entities/entities.js b/app/api/entities/entities.js index 753d02ddd4..a9bf824dbb 100644 --- a/app/api/entities/entities.js +++ b/app/api/entities/entities.js @@ -163,6 +163,10 @@ async function updateEntity(entity, _template) { if (typeof entity.template !== 'undefined') { d.template = entity.template; } + + if (typeof entity.generatedToc !== 'undefined') { + d.generatedToc = entity.generatedToc; + } return model.save(d); }) ); diff --git a/app/api/entities/entitiesModel.ts b/app/api/entities/entitiesModel.ts index 87143cf71c..9c62693655 100644 --- a/app/api/entities/entitiesModel.ts +++ b/app/api/entities/entitiesModel.ts @@ -15,6 +15,7 @@ const mongoSchema = new mongoose.Schema( title: { type: String, required: true }, template: { type: mongoose.Schema.Types.ObjectId, ref: 'templates', index: true }, published: Boolean, + generatedToc: Boolean, icon: new mongoose.Schema({ _id: String, label: String, diff --git a/app/api/entities/specs/entities.spec.js b/app/api/entities/specs/entities.spec.js index 3769354886..b604702ff9 100644 --- a/app/api/entities/specs/entities.spec.js +++ b/app/api/entities/specs/entities.spec.js @@ -340,7 +340,7 @@ describe('entities', () => { }); }); - describe('when published/template property changes', () => { + describe('when published/template/generatedToc property changes', () => { it('should replicate the change for all the languages', done => { const doc = { _id: batmanFinishesId, @@ -348,6 +348,7 @@ describe('entities', () => { metadata: {}, published: false, template: templateId, + generatedToc: true, }; entities @@ -364,8 +365,10 @@ describe('entities', () => { expect(docES.template).toBeDefined(); expect(docES.published).toBe(false); + expect(docES.generatedToc).toBe(true); expect(docES.template.equals(templateId)).toBe(true); expect(docEN.published).toBe(false); + expect(docEN.generatedToc).toBe(true); expect(docEN.template.equals(templateId)).toBe(true); done(); }) @@ -373,6 +376,21 @@ describe('entities', () => { }); }); + describe('when generatedToc is undefined', () => { + it('should not replicate the value to all languages', async () => { + const doc = { _id: batmanFinishesId, sharedId: 'shared', generatedToc: true }; + await entities.save(doc, { language: 'en' }); + await entities.save({ _id: batmanFinishesId, sharedId: 'shared' }, { language: 'en' }); + const [docES, docEN] = await Promise.all([ + entities.getById('shared', 'es'), + entities.getById('shared', 'en'), + ]); + + expect(docES.generatedToc).toBe(true); + expect(docEN.generatedToc).toBe(true); + }); + }); + it('should sync select/multiselect/dates/multidate/multidaterange/numeric', done => { const doc = { _id: syncPropertiesEntityId, diff --git a/app/api/files/files.ts b/app/api/files/files.ts index c6ecd6ee7c..2667669584 100644 --- a/app/api/files/files.ts +++ b/app/api/files/files.ts @@ -1,6 +1,7 @@ import { deleteUploadedFiles } from 'api/files/filesystem'; import connections from 'api/relationships'; import { search } from 'api/search'; +import entities from 'api/entities'; import model from './filesModel'; import { validateFile } from '../../shared/types/fileSchema'; @@ -30,4 +31,27 @@ export const files = { return toDeleteFiles; }, + + async tocReviewed(_id: string, language: string) { + const savedFile = await files.save({ _id, generatedToc: false }); + const sameEntityFiles = await files.get({ entity: savedFile.entity }, { generatedToc: 1 }); + const [entity] = await entities.get({ + sharedId: savedFile.entity, + }); + + await entities.save( + { + _id: entity._id, + sharedId: entity.sharedId, + template: entity.template, + generatedToc: sameEntityFiles.reduce( + (generated, file) => generated || Boolean(file.generatedToc), + false + ), + }, + { user: {}, language } + ); + + return savedFile; + }, }; diff --git a/app/api/files/routes.ts b/app/api/files/routes.ts index 5c4bb0fff2..577519b1cf 100644 --- a/app/api/files/routes.ts +++ b/app/api/files/routes.ts @@ -57,6 +57,28 @@ export default (app: Application) => { .catch(next); }); + app.post( + '/api/files/tocReviewed', + needsAuthorization(['admin', 'editor']), + validation.validateRequest({ + properties: { + body: { + required: ['fileId'], + properties: { + fileId: { type: 'string' }, + }, + }, + }, + }), + async (req, res, next) => { + try { + res.json(await files.tocReviewed(req.body.fileId, req.language)); + } catch (e) { + next(e); + } + } + ); + app.get( '/api/files/:filename', validation.validateRequest({ diff --git a/app/api/files/specs/fixtures.ts b/app/api/files/specs/fixtures.ts index 786593a43a..94b88d6f77 100644 --- a/app/api/files/specs/fixtures.ts +++ b/app/api/files/specs/fixtures.ts @@ -5,22 +5,29 @@ const entityEnId = db.id(); const uploadId = db.id(); const uploadId2 = db.id(); const templateId = db.id(); +const importTemplate = db.id(); const fileName1 = 'f2082bf51b6ef839690485d7153e847a.pdf'; const fixtures: DBFixture = { files: [ { _id: uploadId, - entity: 'entity', + entity: 'sharedId1', + generatedToc: true, originalname: 'upload1', filename: fileName1, type: 'custom', }, { _id: uploadId2, - entity: 'entity', + generatedToc: true, + entity: 'sharedId1', filename: 'fileNotInDisk', }, + { + entity: 'sharedId1', + filename: 'fileWithoutTocFlag', + }, { _id: db.id(), filename: 'fileNotOnDisk' }, { _id: db.id(), originalname: 'upload2', type: 'custom' }, { _id: db.id(), originalname: 'upload3', type: 'document' }, @@ -36,11 +43,21 @@ const fixtures: DBFixture = { sharedId: 'sharedId1', language: 'es', title: 'Gadgets 01 ES', - toc: [{ _id: db.id(), label: 'existingToc' }], + generatedToc: true, + template: templateId, }, - { _id: entityEnId, sharedId: 'sharedId1', language: 'en', title: 'Gadgets 01 EN' }, + { + _id: entityEnId, + template: templateId, + sharedId: 'sharedId1', + language: 'en', + title: 'Gadgets 01 EN', + }, + ], + templates: [ + { _id: templateId, default: true, name: 'mydoc', properties: [] }, + { _id: importTemplate, default: true, name: 'import', properties: [] }, ], - templates: [{ _id: templateId, default: true, name: 'mydoc', properties: [] }], settings: [ { _id: db.id(), @@ -51,4 +68,13 @@ const fixtures: DBFixture = { ], }; -export { fixtures, entityId, entityEnId, fileName1, uploadId, uploadId2, templateId }; +export { + fixtures, + entityId, + entityEnId, + fileName1, + uploadId, + uploadId2, + templateId, + importTemplate, +}; diff --git a/app/api/files/specs/routes.spec.ts b/app/api/files/specs/routes.spec.ts index 88b63e1262..28af682930 100644 --- a/app/api/files/specs/routes.spec.ts +++ b/app/api/files/specs/routes.spec.ts @@ -12,6 +12,7 @@ import { FileType } from 'shared/types/fileType'; import { fixtures, uploadId, uploadId2 } from './fixtures'; import { files } from '../files'; import uploadRoutes from '../routes'; +import entities from 'api/entities'; jest.mock( '../../auth/authMiddleware.ts', @@ -47,7 +48,7 @@ describe('files routes', () => { }); it('should reindex all entities that are related to the saved file', async () => { - expect(search.indexEntities).toHaveBeenCalledWith({ sharedId: 'entity' }, '+fullText'); + expect(search.indexEntities).toHaveBeenCalledWith({ sharedId: 'sharedId1' }, '+fullText'); }); }); @@ -85,7 +86,7 @@ describe('files routes', () => { .query({ _id: uploadId2.toString() }); expect(search.indexEntities).toHaveBeenCalledWith( - { sharedId: { $in: ['entity'] } }, + { sharedId: { $in: ['sharedId1'] } }, '+fullText' ); }); @@ -107,5 +108,36 @@ describe('files routes', () => { expect(response.body.errors[0].message).toBe('should be string'); }); + + describe('api/files/tocReviewed', () => { + it('should set tocGenerated to false on the file', async () => { + const response: SuperTestResponse = await request(app) + .post('/api/files/tocReviewed') + .set('content-language', 'es') + .send({ fileId: uploadId.toString() }); + + const [file] = await files.get({ _id: uploadId }); + expect(file.generatedToc).toBe(false); + expect(response.body.entity).toBe('sharedId1'); + }); + + it('should set tocGenerated to false on the entity when all associated files are false', async () => { + await request(app) + .post('/api/files/tocReviewed') + .send({ fileId: uploadId.toString() }) + .expect(200); + + let [entity] = await entities.get({ sharedId: 'sharedId1' }); + expect(entity.generatedToc).toBe(true); + + await request(app) + .post('/api/files/tocReviewed') + .send({ fileId: uploadId2.toString() }) + .expect(200); + + [entity] = await entities.get({ sharedId: 'sharedId1' }); + expect(entity.generatedToc).toBe(false); + }); + }); }); }); diff --git a/app/api/files/specs/uploadRoutes.spec.ts b/app/api/files/specs/uploadRoutes.spec.ts index b8a7fb44a5..3cb15c61dd 100644 --- a/app/api/files/specs/uploadRoutes.spec.ts +++ b/app/api/files/specs/uploadRoutes.spec.ts @@ -16,7 +16,7 @@ import { setUpApp, socketEmit, iosocket } from 'api/utils/testingRoutes'; import { FileType } from 'shared/types/fileType'; import entities from 'api/entities'; -import { fixtures, templateId } from './fixtures'; +import { fixtures, templateId, importTemplate } from './fixtures'; import { files } from '../files'; import uploadRoutes from '../routes'; @@ -72,7 +72,10 @@ describe('upload routes', () => { expect(iosocket.emit).toHaveBeenCalledWith('conversionStart', 'sharedId1'); expect(iosocket.emit).toHaveBeenCalledWith('documentProcessed', 'sharedId1'); - const [upload] = await files.get({ entity: 'sharedId1' }, '+fullText'); + const [upload] = await files.get( + { originalname: 'f2082bf51b6ef839690485d7153e847a.pdf' }, + '+fullText' + ); expect(upload).toEqual( expect.objectContaining({ @@ -105,14 +108,14 @@ describe('upload routes', () => { it('should detect English documents and store the result', async () => { await uploadDocument('uploads/eng.pdf'); - const [upload] = await files.get({ entity: 'sharedId1' }); + const [upload] = await files.get({ originalname: 'eng.pdf' }); expect(upload.language).toBe('eng'); }, 10000); it('should detect Spanish documents and store the result', async () => { await uploadDocument('uploads/spn.pdf'); - const [upload] = await files.get({ entity: 'sharedId1' }); + const [upload] = await files.get({ originalname: 'spn.pdf' }); expect(upload.language).toBe('spa'); }); }); @@ -126,7 +129,7 @@ describe('upload routes', () => { .attach('file', path.join(__dirname, 'uploads/invalid_document.txt')) ); - const [upload] = await files.get({ entity: 'sharedId1' }, '+fullText'); + const [upload] = await files.get({ originalname: 'invalid_document.txt' }, '+fullText'); expect(upload.status).toBe('failed'); }); @@ -176,7 +179,7 @@ describe('upload routes', () => { await socketEmit('IMPORT_CSV_END', async () => request(app) .post('/api/import') - .field('template', templateId.toString()) + .field('template', importTemplate.toString()) .attach('file', `${__dirname}/uploads/importcsv.csv`) ); @@ -184,7 +187,7 @@ describe('upload routes', () => { expect(iosocket.emit).toHaveBeenCalledWith('IMPORT_CSV_PROGRESS', 1); expect(iosocket.emit).toHaveBeenCalledWith('IMPORT_CSV_PROGRESS', 2); - const imported = await entities.get({ template: templateId }); + const imported = await entities.get({ template: importTemplate }); expect(imported).toEqual([ expect.objectContaining({ title: 'imported entity one' }), expect.objectContaining({ title: 'imported entity two' }), diff --git a/app/api/search/deprecatedRoutes.js b/app/api/search/deprecatedRoutes.js index 9526181f88..e5d369dbeb 100644 --- a/app/api/search/deprecatedRoutes.js +++ b/app/api/search/deprecatedRoutes.js @@ -1,6 +1,6 @@ import Joi from 'joi'; import entities from 'api/entities'; -import { searchSchema } from 'api/search/searchSchema'; +import { searchParamsSchema } from 'shared/types/searchParams'; import { search } from './search'; import { validation, parseQuery } from '../utils'; @@ -25,7 +25,7 @@ export default app => { app.get( '/api/search', parseQuery, - validation.validateRequest(searchSchema), + validation.validateRequest(searchParamsSchema), (req, res, next) => { const action = req.query.geolocation ? 'searchGeolocations' : 'search'; diff --git a/app/api/search/documentQueryBuilder.js b/app/api/search/documentQueryBuilder.js index 19d2b77b48..42eb896b7b 100644 --- a/app/api/search/documentQueryBuilder.js +++ b/app/api/search/documentQueryBuilder.js @@ -2,7 +2,7 @@ import { preloadOptionsSearch } from 'shared/config'; import filterToMatch, { multiselectFilter } from './metadataMatchers'; -import { propertyToAggregation } from './metadataAggregations'; +import { propertyToAggregation, generatedTocAggregations } from './metadataAggregations'; export default function() { const baseQuery = { @@ -231,6 +231,17 @@ export default function() { } }, + customFilters(filters = {}) { + Object.keys(filters).forEach(key => { + if (filters[key].values.length) { + addFilter({ + terms: { [key]: filters[key].values }, + }); + } + }); + return this; + }, + filterMetadata(filters = []) { filters.forEach(filter => { const match = filterToMatch(filter, filter.suggested ? 'suggestedMetadata' : 'metadata'); @@ -241,6 +252,10 @@ export default function() { return this; }, + generatedTOCAggregations() { + baseQuery.aggregations.all.aggregations.generatedToc = generatedTocAggregations(baseQuery); + }, + aggregations(properties, dictionaries) { properties.forEach(property => { baseQuery.aggregations.all.aggregations[property.name] = propertyToAggregation( diff --git a/app/api/search/elasticTypes.ts b/app/api/search/elasticTypes.ts index efee29aab9..ab0fcca438 100644 --- a/app/api/search/elasticTypes.ts +++ b/app/api/search/elasticTypes.ts @@ -1,5 +1,6 @@ import { RequestParams } from '@elastic/elasticsearch'; import { RequestBody } from '@elastic/elasticsearch/lib/Transport'; +import { Aggregations } from 'shared/types/Aggregations'; interface ShardsResponse { total: number; @@ -42,7 +43,7 @@ export interface SearchResponse { sort?: string[]; }>; }; - aggregations?: any; + aggregations?: Aggregations; } export type IndicesDelete = Omit; diff --git a/app/api/search/metadataAggregations.js b/app/api/search/metadataAggregations.js index a44f6acae5..3c57a2a996 100644 --- a/app/api/search/metadataAggregations.js +++ b/app/api/search/metadataAggregations.js @@ -137,6 +137,7 @@ const extractFilters = (baseQuery, path) => { (!match.terms || (match.terms && !match.terms[path])) && (!match.bool || !match.bool.should || + !match.bool.should[1] || !match.bool.should[1].terms || !match.bool.should[1].terms[path]) ); @@ -167,3 +168,10 @@ export const propertyToAggregation = (property, dictionaries, baseQuery, suggest return aggregation(path, should, filters); }; + +export const generatedTocAggregations = baseQuery => { + const path = 'generatedToc'; + const filters = extractFilters(baseQuery, path); + const { should } = baseQuery.query.bool; + return aggregation(path, should, filters); +}; diff --git a/app/api/search/search.js b/app/api/search/search.js index ad93ead8c4..f47b95bef1 100644 --- a/app/api/search/search.js +++ b/app/api/search/search.js @@ -241,7 +241,12 @@ const _denormalizeAggregations = async (aggregations, templates, dictionaries, l const properties = propertiesHelper.allUniqueProperties(templates); return Object.keys(aggregations).reduce(async (denormaLizedAgregationsPromise, key) => { const denormaLizedAgregations = await denormaLizedAgregationsPromise; - if (!aggregations[key].buckets || key === '_types' || aggregations[key].type === 'nested') { + if ( + !aggregations[key].buckets || + key === '_types' || + aggregations[key].type === 'nested' || + key === 'generatedToc' + ) { return Object.assign(denormaLizedAgregations, { [key]: aggregations[key] }); } @@ -377,6 +382,7 @@ const processResponse = async (response, templates, dictionaries, language, filt result._id = hit._id; return result; }); + const sanitizedAggregations = await _sanitizeAggregations( response.body.aggregations.all, templates, @@ -603,6 +609,7 @@ const buildQuery = async (query, language, user, resources) => { const filters = processFilters(query.filters, [...allUniqueProps, ...properties]); // this is where the query filters are built queryBuilder.filterMetadata(filters); + queryBuilder.customFilters(query.customFilters); // this is where the query aggregations are built queryBuilder.aggregations(aggregations, dictionaries); @@ -618,7 +625,10 @@ const search = { searchGeolocation(queryBuilder, templates); } - // queryBuilder.query() is the actual call + if (query.aggregateGeneratedToc) { + queryBuilder.generatedTOCAggregations(); + } + return elastic .search({ body: queryBuilder.query() }) .then(response => processResponse(response, templates, dictionaries, language, query.filters)) diff --git a/app/api/search/specs/fixtures_elastic.js b/app/api/search/specs/fixtures_elastic.js index 25d0432e2f..198e3ed3f1 100644 --- a/app/api/search/specs/fixtures_elastic.js +++ b/app/api/search/specs/fixtures_elastic.js @@ -72,6 +72,7 @@ export const fixtures = { language: 'en', title: 'Batman finishes en', published: true, + generatedToc: true, user: userId, metadata: { relationship: [ @@ -92,6 +93,7 @@ export const fixtures = { title: 'Batman finishes es', published: true, user: userId, + generatedToc: true, metadata: { relationship: [ { value: batmanBegins, label: 'Batman begins es' }, @@ -115,6 +117,7 @@ export const fixtures = { language: 'es', title: 'Batman begins es', published: true, + generatedToc: false, user: userId, }, { @@ -151,6 +154,7 @@ export const fixtures = { language: 'es', title: 'template1 title es', published: true, + generatedToc: false, user: userId, }, { diff --git a/app/api/search/specs/search.spec.js b/app/api/search/specs/search.spec.js index 63a9bafdf6..8b94c670b8 100644 --- a/app/api/search/specs/search.spec.js +++ b/app/api/search/specs/search.spec.js @@ -163,6 +163,15 @@ describe('search', () => { .catch(catchErrors(done)); }); + it('should return generatedToc aggregations when requested for', async () => { + const response = await search.search({ aggregateGeneratedToc: true }, 'es'); + + const aggregations = response.aggregations.all.generatedToc.buckets; + + expect(aggregations.find(a => a.key === 'false').filtered.doc_count).toBe(2); + expect(aggregations.find(a => a.key === 'true').filtered.doc_count).toBe(1); + }); + it('should return aggregations when searching by 2 terms', done => { search .search({ searchTerm: 'english document' }, 'es') @@ -769,6 +778,21 @@ describe('search', () => { }); }); + describe('customFilters', () => { + it('should filter by the values passed', async () => { + const query = { + customFilters: { + generatedToc: { + values: ['true'], + }, + }, + }; + + const { rows } = await search.search(query, 'en'); + expect(rows).toEqual([expect.objectContaining({ title: 'Batman finishes en' })]); + }); + }); + describe('autocompleteAggregations()', () => { it('should return a list of options matching by label and options related to the matching one', async () => { const query = { diff --git a/app/api/search/specs/searchSchema.spec.ts b/app/api/search/specs/searchSchema.spec.ts index 33722cd602..ffaecc57bf 100644 --- a/app/api/search/specs/searchSchema.spec.ts +++ b/app/api/search/specs/searchSchema.spec.ts @@ -1,6 +1,6 @@ import { ValidationError } from 'ajv'; import { validation } from 'api/utils'; -import { searchSchema } from '../searchSchema'; +import { searchParamsSchema } from 'shared/types/searchParams'; describe('search schema', () => { const validQuery = { @@ -27,11 +27,12 @@ describe('search schema', () => { it('should not have validation errors for valid search', async () => { const validSearch = { validQuery }; - await validation.validateRequest(searchSchema)(validSearch, null, expectValidSchema); + await validation.validateRequest(searchParamsSchema)(validSearch, null, expectValidSchema); }); + it('should support a number as a search term', async () => { const validSearch = { query: { ...validQuery, searchTerm: 3 } }; - await validation.validateRequest(searchSchema)(validSearch, null, expectValidSchema); + await validation.validateRequest(searchParamsSchema)(validSearch, null, expectValidSchema); }); }); @@ -42,7 +43,11 @@ describe('search schema', () => { async function testInvalidProperty(invalidProperty: any) { const invalidSearch = { query: { ...validQuery, ...invalidProperty } }; - await validation.validateRequest(searchSchema)(invalidSearch, null, expectInvalidSchema); + await validation.validateRequest(searchParamsSchema)( + invalidSearch, + null, + expectInvalidSchema + ); } it('should be invalid if allAgregations is not a boolean value', async () => { diff --git a/app/api/toc_generation/specs/fixtures.ts b/app/api/toc_generation/specs/fixtures.ts new file mode 100644 index 0000000000..2370acac38 --- /dev/null +++ b/app/api/toc_generation/specs/fixtures.ts @@ -0,0 +1,64 @@ +import { testingDB, DBFixture } from 'api/utils/testing_db'; + +const templateId = testingDB.id(); + +const fixtures: DBFixture = { + templates: [ + { + _id: templateId, + properties: [], + }, + ], + entities: [ + { + sharedId: 'shared1', + title: 'pdf1entity', + template: templateId, + }, + { + sharedId: 'shared3', + title: 'pdf3entity', + template: templateId, + }, + ], + files: [ + { + _id: testingDB.id(), + entity: 'shared1', + filename: 'pdf1.pdf', + language: 'es', + originalname: 'originalPdf1.pdf', + type: 'document', + }, + { + _id: testingDB.id(), + type: 'document', + language: 'es', + }, + { + _id: testingDB.id(), + type: 'custom', + filename: 'background.jpg', + language: 'es', + }, + { + _id: testingDB.id(), + type: 'document', + filename: 'pdf2.pdf', + originalname: 'originalPdf2.pdf', + toc: [{}], + language: 'es', + }, + { + _id: testingDB.id(), + entity: 'shared3', + type: 'document', + filename: 'pdf3.pdf', + toc: [], + originalname: 'originalPdf4.pdf', + language: 'es', + }, + ], +}; + +export { fixtures }; diff --git a/app/api/toc_generation/specs/tocService.spec.ts b/app/api/toc_generation/specs/tocService.spec.ts new file mode 100644 index 0000000000..a2398dd6e4 --- /dev/null +++ b/app/api/toc_generation/specs/tocService.spec.ts @@ -0,0 +1,142 @@ +import { testingDB } from 'api/utils/testing_db'; +import request from 'shared/JSONRequest'; +import { files } from 'api/files'; +import { elasticTesting } from 'api/utils/elastic_testing'; +import errorLog from 'api/log/errorLog'; +import { fixtures } from './fixtures'; +import { tocService } from '../tocService'; + +describe('tocService', () => { + const service = tocService('url'); + + let requestMock: jest.SpyInstance; + + beforeEach(async () => { + spyOn(errorLog, 'error'); + requestMock = jest.spyOn(request, 'uploadFile'); + requestMock.mockImplementation(async (url, filename) => { + expect(url).toBe('url'); + if (filename === 'pdf1.pdf') { + return Promise.resolve({ text: JSON.stringify([{ label: 'section1 pdf1' }]) }); + } + if (filename === 'pdf3.pdf') { + return Promise.resolve({ text: JSON.stringify([{ label: 'section1 pdf3' }]) }); + } + throw new Error(`this file is not supposed to be sent for toc generation ${filename}`); + }); + + const elasticIndex = 'toc.service.index'; + await testingDB.clearAllAndLoad(fixtures, elasticIndex); + await elasticTesting.resetIndex(); + await elasticTesting.refresh(); + }); + + afterAll(async () => { + await testingDB.disconnect(); + }); + + it('should not fail when there is no more to process', async () => { + await service.processNext(); + await service.processNext(); + await expect(service.processNext()).resolves.not.toThrow(); + }); + + it('should send the next pdfFile and save toc generated', async () => { + await service.processNext(); + await service.processNext(); + + let [fileProcessed] = await files.get({ filename: 'pdf1.pdf' }); + expect(fileProcessed.toc).toEqual([{ label: 'section1 pdf1' }]); + expect(fileProcessed.generatedToc).toEqual(true); + + [fileProcessed] = await files.get({ filename: 'pdf3.pdf' }); + expect(fileProcessed.toc).toEqual([{ label: 'section1 pdf3' }]); + expect(fileProcessed.generatedToc).toEqual(true); + }); + + it('should reindex the affected entities', async () => { + await service.processNext(); + await service.processNext(); + await elasticTesting.refresh(); + + const entitiesIndexed = await elasticTesting.getIndexedEntities(); + + expect(entitiesIndexed).toEqual([ + expect.objectContaining({ + title: 'pdf1entity', + generatedToc: true, + }), + expect.objectContaining({ + title: 'pdf3entity', + generatedToc: true, + }), + ]); + }); + + describe('error handling', () => { + it('should save a fake TOC when generated one is empty', async () => { + requestMock.mockImplementation(async () => Promise.resolve({ text: JSON.stringify([]) })); + await service.processNext(); + + const [fileProcessed] = await files.get({ filename: 'pdf1.pdf' }); + expect(fileProcessed.toc).toEqual([ + { + selectionRectangles: [{ top: 0, left: 0, width: 0, height: 0, page: '1' }], + label: 'ERROR: Toc was generated empty', + indentation: 0, + }, + ]); + expect(fileProcessed.generatedToc).toEqual(true); + + await elasticTesting.refresh(); + const entitiesIndexed = await elasticTesting.getIndexedEntities(); + expect(entitiesIndexed).toEqual([ + expect.objectContaining({ title: 'pdf1entity', generatedToc: true }), + ]); + }); + + it('should save a fake toc when there is an error', async () => { + requestMock.mockImplementation(async () => { + throw new Error('request error'); + }); + await service.processNext(); + + const [fileProcessed] = await files.get({ filename: 'pdf1.pdf' }); + expect(fileProcessed.toc).toEqual([ + { + selectionRectangles: [{ top: 0, left: 0, width: 0, height: 0, page: '1' }], + label: 'ERROR: Toc generation throwed an error', + indentation: 0, + }, + { + selectionRectangles: [{ top: 0, left: 0, width: 0, height: 0, page: '1' }], + label: 'request error', + indentation: 0, + }, + ]); + expect(fileProcessed.generatedToc).toEqual(true); + + await elasticTesting.refresh(); + const entitiesIndexed = await elasticTesting.getIndexedEntities(); + expect(entitiesIndexed).toEqual([ + expect.objectContaining({ title: 'pdf1entity', generatedToc: true }), + ]); + }); + + it('should not save anything when the error is ECONNREFUSED', async () => { + requestMock.mockImplementation(async () => { + // eslint-disable-next-line no-throw-literal + throw { code: 'ECONNREFUSED' }; + }); + await service.processNext(); + + const [fileProcessed] = await files.get({ filename: 'pdf1.pdf' }); + expect(fileProcessed.toc).not.toBeDefined(); + expect(fileProcessed.generatedToc).not.toBeDefined(); + + await elasticTesting.refresh(); + const entitiesIndexed = await elasticTesting.getIndexedEntities(); + expect(entitiesIndexed).toEqual([]); + }); + }); +}); diff --git a/app/api/toc_generation/tocService.ts b/app/api/toc_generation/tocService.ts new file mode 100644 index 0000000000..8831aaf1df --- /dev/null +++ b/app/api/toc_generation/tocService.ts @@ -0,0 +1,68 @@ +import { files, uploadsPath } from 'api/files'; +import { prettifyError } from 'api/utils/handleError'; +import errorLog from 'api/log/errorLog'; +import request from 'shared/JSONRequest'; +import entities from 'api/entities'; +import { TocSchema } from 'shared/types/commonTypes'; +import { FileType } from 'shared/types/fileType'; + +const fakeTocEntry = (label: string): TocSchema => ({ + selectionRectangles: [{ top: 0, left: 0, width: 0, height: 0, page: '1' }], + indentation: 0, + label, +}); + +const saveToc = async (file: FileType, toc: TocSchema[]) => { + await files.save({ ...file, toc, generatedToc: true }); + const [entity] = await entities.get({ sharedId: file.entity }, {}); + await entities.save( + { + ...entity, + generatedToc: true, + }, + { user: {}, language: file.language }, + false + ); +}; + +const generateToc = async (url: string, file: FileType): Promise => { + const response = await request.uploadFile(url, file.filename, uploadsPath(file.filename)); + + let toc = JSON.parse(response.text); + if (!toc.length) { + toc = [fakeTocEntry('ERROR: Toc was generated empty')]; + } + return toc; +}; + +const handleError = async (e: { code?: string; message: string }, file: FileType) => { + if (e?.code !== 'ECONNREFUSED' && e?.code !== 'ECONNRESET') { + const toc = [fakeTocEntry('ERROR: Toc generation throwed an error'), fakeTocEntry(e.message)]; + await saveToc(file, toc); + } +}; + +const tocService = (serviceUrl: string) => ({ + async processNext() { + const [nextFile] = await files.get( + { + $or: [{ toc: { $size: 0 } }, { toc: { $exists: false } }], + type: 'document', + filename: { $exists: true }, + }, + '', + { sort: { _id: 1 }, limit: 1 } + ); + + if (nextFile) { + try { + await saveToc(nextFile, await generateToc(serviceUrl, nextFile)); + } catch (e) { + await handleError(e, nextFile); + errorLog.error(prettifyError(e).prettyMessage); + } + } + }, +}); + +export { tocService }; diff --git a/app/react/App/scss/modules/_search.scss b/app/react/App/scss/modules/_search.scss index 59b2275316..97520e83a3 100644 --- a/app/react/App/scss/modules/_search.scss +++ b/app/react/App/scss/modules/_search.scss @@ -233,6 +233,16 @@ display: none; } +.admin-filter { + span, label { + color: #2B56C1; + } + + input { + border: #2B56C1; + } +} + .search__filter { list-style-type: none; margin: 0; diff --git a/app/react/App/scss/modules/_toc.scss b/app/react/App/scss/modules/_toc.scss index 3f195ab5ed..9f06fca32e 100644 --- a/app/react/App/scss/modules/_toc.scss +++ b/app/react/App/scss/modules/_toc.scss @@ -1,5 +1,10 @@ .toc { - padding: 15px; + padding: 0px 15px 15px 15px; +} + +div.tocHeader { + padding-left: 15px; + border-bottom: 1px solid #f4f4f4; } .toc-view{ diff --git a/app/react/App/scss/modules/_uploads.scss b/app/react/App/scss/modules/_uploads.scss index f865b298ce..aa4ff732b4 100644 --- a/app/react/App/scss/modules/_uploads.scss +++ b/app/react/App/scss/modules/_uploads.scss @@ -87,9 +87,14 @@ list-style: none; } + .upload-button { + margin-left: 15px; + } + .btn { border-radius: 2px; - margin-left: 15px; + margin-right: 15px; + margin-top: 15px; } } @@ -97,23 +102,11 @@ .file-form { padding: 15px; border-bottom: 1px solid #f4f4f4; - .btn { - float: right; - } &-originalname { margin-bottom: 15px; } - &-language { - background: #fff6da; - border: 1px solid rgba(0, 0, 0, 0.15); - box-sizing: border-box; - border-radius: 4px; - display: inline-block; - padding: 3px; - } - &-failed { box-sizing: border-box; border-radius: 4px; diff --git a/app/react/Attachments/components/File.tsx b/app/react/Attachments/components/File.tsx index 21fd3c5cb1..5be517c351 100644 --- a/app/react/Attachments/components/File.tsx +++ b/app/react/Attachments/components/File.tsx @@ -10,6 +10,7 @@ import { APIURL } from 'app/config.js'; import { LocalForm, Control } from 'react-redux-form'; import { updateFile, deleteFile } from 'app/Attachments/actions/actions'; import { wrapDispatch } from 'app/Multireducer'; +import { TocGeneratedLabel } from 'app/ToggledFeatures/tocGeneration'; import { NeedAuthorization } from 'app/Auth'; import { EntitySchema } from 'shared/types/entityType'; import { ViewDocumentLink } from './ViewDocumentLink'; @@ -100,29 +101,36 @@ export class File extends Component { const { language, filename = '' } = this.props.file; return (
-
- {language ? transformLanguage(language) || '' : ''} -
{' '} - - -   - Download - - - - - - View - + Download + + + + + + View + +
); } diff --git a/app/react/Attachments/components/specs/File.spec.tsx b/app/react/Attachments/components/specs/File.spec.tsx index 997ef2a635..49370757e8 100644 --- a/app/react/Attachments/components/specs/File.spec.tsx +++ b/app/react/Attachments/components/specs/File.spec.tsx @@ -2,8 +2,8 @@ import React from 'react'; import { shallow, ShallowWrapper } from 'enzyme'; import { LocalForm } from 'react-redux-form'; import { FileType } from 'shared/types/fileType'; -import { File, FileProps } from '../File'; import { Translate } from 'app/I18N'; +import { File, FileProps } from '../File'; describe('file', () => { let component: ShallowWrapper; @@ -32,6 +32,7 @@ describe('file', () => { }); const render = () => { + //eslint-disable-next-line react/jsx-props-no-spreading component = shallow(, { context }); }; @@ -41,7 +42,7 @@ describe('file', () => { expect(title).toBe('Human_name_1.pdf'); const language = component - .find('.file-language') + .find('.badge') .find(Translate) .props().children; expect(language).toBe('english'); diff --git a/app/react/DocumentForm/components/FormGroup.js b/app/react/DocumentForm/components/FormGroup.js index 74ce5089f0..df56f9b9aa 100644 --- a/app/react/DocumentForm/components/FormGroup.js +++ b/app/react/DocumentForm/components/FormGroup.js @@ -2,7 +2,7 @@ import PropTypes from 'prop-types'; import React from 'react'; const FormGroup = props => { - let className = 'form-group'; + let className = `${props.className} form-group`; if ((!props.pristine || props.submitFailed) && props.valid === false) { className += ' has-error'; } @@ -12,7 +12,12 @@ const FormGroup = props => { const childrenType = PropTypes.oneOfType([PropTypes.object, PropTypes.array]); +FormGroup.defaultProps = { + className: '', +}; + FormGroup.propTypes = { + className: PropTypes.string, pristine: PropTypes.bool, valid: PropTypes.bool, submitFailed: PropTypes.bool, diff --git a/app/react/Documents/components/DocumentSidePanel.js b/app/react/Documents/components/DocumentSidePanel.js index d476d6d047..26fff00d23 100644 --- a/app/react/Documents/components/DocumentSidePanel.js +++ b/app/react/Documents/components/DocumentSidePanel.js @@ -4,6 +4,7 @@ import { connect } from 'react-redux'; import Immutable from 'immutable'; import PropTypes from 'prop-types'; import React, { Component } from 'react'; +import './scss/toc.scss'; import { MetadataFormButtons, ShowMetadata } from 'app/Metadata'; import { NeedAuthorization } from 'app/Auth'; @@ -16,6 +17,7 @@ import ShowIf from 'app/App/ShowIf'; import SidePanel from 'app/Layout/SidePanel'; import DocumentSemanticSearchResults from 'app/SemanticSearch/components/DocumentResults'; import { CopyFromEntity } from 'app/Metadata/components/CopyFromEntity'; +import { TocGeneratedLabel, ReviewTocButton } from 'app/ToggledFeatures/tocGeneration'; import { Icon } from 'UI'; import * as viewerModule from 'app/Viewer'; @@ -321,11 +323,14 @@ export class DocumentSidePanel extends Component {
+ + Mark as Reviewed +
@@ -340,6 +345,15 @@ export class DocumentSidePanel extends Component { /> +
+

+ Table of contents +

+   + + auto-created ⓘ + +
{tocElement.get('label')} + + {tocElement.getIn(['selectionRectangles', 0]).get('page')} + diff --git a/app/react/Documents/components/scss/showToc.scss b/app/react/Documents/components/scss/showToc.scss new file mode 100644 index 0000000000..07a14bca5f --- /dev/null +++ b/app/react/Documents/components/scss/showToc.scss @@ -0,0 +1,58 @@ +.toc { + padding-left: 5px !important; + padding-right: 5px !important; + + ul.toc-view { + li.toc-indent-0, + li.toc-indent-1, + li.toc.indent-2 { + border-radius: 2px; + height: 28px; + + a:hover { + text-decoration: none; + }; + } + + li.toc-indent-0 { + a { + font-weight: 500; + font-size: 16px; + line-height: 18.75px; + color: #444; + } + } + + li.toc-indent-1 { + a { + font-weight: 400; + font-size: 14px; + line-height: 16.41px; + color: #333; + } + } + + li.toc-indent-2 { + a { + font-weight: 400; + font-size: 14px; + line-height: 16.41px; + color: #555; + } + } + + li.toc-indent-0:hover, + li.toc-indent-1:hover, + li.toc.indent-2:hover { + background-color: #E2E7F4; + } + } +} + +.page-number { + position: relative; + float: right; + font-weight: normal; + padding-right: 4px; +} + diff --git a/app/react/Documents/components/scss/toc.scss b/app/react/Documents/components/scss/toc.scss new file mode 100644 index 0000000000..ef4e9bf9e9 --- /dev/null +++ b/app/react/Documents/components/scss/toc.scss @@ -0,0 +1,20 @@ +.tocHeader { + h1 { + display: inline-block; + height: 23px; + font-weight: 500; + font-size: 20px; + line-height: 23.44px; + } +} + +.file, .tocHeader { + .badge { + border: 1px solid rgba(0, 0, 0, 0.15); + border-radius: 100px; + background-color: #CBE0FF; + color: #444444; + font-weight: normal; + margin-right: 5px; + } +} diff --git a/app/react/Library/actions/libraryActions.js b/app/react/Library/actions/libraryActions.js index 23ac150c94..c87266d1bc 100644 --- a/app/react/Library/actions/libraryActions.js +++ b/app/react/Library/actions/libraryActions.js @@ -249,6 +249,8 @@ export function searchDocuments( dispatch(actions.set(`${storeKey}.selectedSorting`, currentSearch)); } + searchParams.customFilters = currentSearch.customFilters; + setSearchInUrl(searchParams); }; } diff --git a/app/react/Library/actions/specs/filterActions.spec.js b/app/react/Library/actions/specs/filterActions.spec.js index 6db29dd2cf..f8e1fb23b3 100644 --- a/app/react/Library/actions/specs/filterActions.spec.js +++ b/app/react/Library/actions/specs/filterActions.spec.js @@ -36,6 +36,9 @@ describe('filterActions', () => { }; store = { + settings: { + collection: Immutable.Map({}), + }, library: { filters: Immutable.fromJS(filtersState), search, diff --git a/app/react/Library/actions/specs/libraryActions.spec.js b/app/react/Library/actions/specs/libraryActions.spec.js index 71d771da5a..8e5e5c14ae 100644 --- a/app/react/Library/actions/specs/libraryActions.spec.js +++ b/app/react/Library/actions/specs/libraryActions.spec.js @@ -29,6 +29,7 @@ describe('libraryActions', () => { const aggregations = [{ prop: { buckets: [] } }]; const templates = [{ name: 'Decision' }, { name: 'Ruling' }]; const thesauris = [{ _id: 'abc1' }]; + let getState; describe('setDocuments', () => { it('should return a SET_DOCUMENTS action ', () => { @@ -40,7 +41,6 @@ describe('libraryActions', () => { describe('setTemplates', () => { const documentTypes = ['typea']; let dispatch; - let getState; const filters = { documentTypes, properties: ['library properties'], @@ -48,9 +48,12 @@ describe('libraryActions', () => { beforeEach(() => { dispatch = jasmine.createSpy('dispatch'); - getState = jasmine - .createSpy('getState') - .and.returnValue({ library: { filters: Immutable.fromJS(filters), search: {} } }); + getState = jasmine.createSpy('getState').and.returnValue({ + settings: { + collection: Immutable.Map({}), + }, + library: { filters: Immutable.fromJS(filters), search: {} }, + }); }); it('should dispatch a SET_LIBRARY_TEMPLATES action ', () => { @@ -159,7 +162,6 @@ describe('libraryActions', () => { describe('searchDocuments', () => { let store; - let getState; let state; const storeKey = 'library'; beforeEach(() => { @@ -186,7 +188,19 @@ describe('libraryActions', () => { ], documentTypes: ['decision'], }; - store = { library: { filters: Immutable.fromJS(state), search: { searchTerm: 'batman' } } }; + store = { + settings: { + collection: Immutable.Map({}), + }, + library: { + filters: Immutable.fromJS(state), + search: { + searchTerm: 'batman', + customFilters: { property: { values: ['value'] } }, + filters: {}, + }, + }, + }; spyOn(browserHistory, 'getCurrentLocation').and.returnValue({ pathname: '/library', query: { view: 'chart' }, @@ -261,6 +275,16 @@ describe('libraryActions', () => { ); }); + it('should use customFilters from the current search on the store', () => { + const limit = 60; + spyOn(browserHistory, 'push'); + actions.searchDocuments({}, storeKey, limit)(dispatch, getState); + + expect(browserHistory.push).toHaveBeenCalledWith( + "/library/?view=chart&q=(customFilters:(property:(values:!(value))),filters:(),from:0,limit:60,searchTerm:'batman',sort:_score,types:!(decision))" //eslint-disable-line + ); + }); + it('should set the storeKey selectedSorting if user has selected a custom sorting', () => { const expectedDispatch = { type: 'library.selectedSorting/SET', diff --git a/app/react/Library/components/FiltersForm.js b/app/react/Library/components/FiltersForm.js index 6f773961c4..9d08345a1f 100644 --- a/app/react/Library/components/FiltersForm.js +++ b/app/react/Library/components/FiltersForm.js @@ -11,6 +11,7 @@ import { t } from 'app/I18N'; import { wrapDispatch } from 'app/Multireducer'; import debounce from 'app/utils/debounce'; import libraryHelper from 'app/Library/helpers/libraryFilters'; +import { FilterTocGeneration } from 'app/ToggledFeatures/tocGeneration'; import Filters from './FiltersFromProperties'; @@ -97,6 +98,7 @@ export class FiltersForm extends Component { translationContext={translationContext} storeKey={this.props.storeKey} /> + ); diff --git a/app/react/Library/components/ViewMetadataPanel.js b/app/react/Library/components/ViewMetadataPanel.js index 6f9ec1138f..b0469bdb12 100644 --- a/app/react/Library/components/ViewMetadataPanel.js +++ b/app/react/Library/components/ViewMetadataPanel.js @@ -9,6 +9,7 @@ import { actions } from 'app/Metadata'; import { deleteDocument, searchSnippets } from 'app/Library/actions/libraryActions'; import { deleteEntity } from 'app/Entities/actions/actions'; import { wrapDispatch } from 'app/Multireducer'; +import { entityDefaultDocument } from 'shared/entityDefaultDocument'; import modals from 'app/Modals'; import { @@ -22,9 +23,18 @@ const getTemplates = state => state.templates; const mapStateToProps = (state, props) => { const library = state[props.storeKey]; + const doc = library.ui.get('selectedDocuments').first() || Immutable.fromJS({ documents: [] }); + const defaultLanguage = state.settings.collection.get('languages').find(l => l.get('defautl')); + const file = entityDefaultDocument( + doc.get('documents') ? doc.get('documents').toJS() : [{}], + doc.get('language'), + defaultLanguage + ); + return { open: library.ui.get('selectedDocuments').size === 1, - doc: library.ui.get('selectedDocuments').first() || Immutable.fromJS({}), + doc, + file, references: library.sidepanel.references, tab: library.sidepanel.tab, docBeingEdited: !!Object.keys(library.sidepanel.metadata).length, diff --git a/app/react/Library/helpers/libraryFilters.js b/app/react/Library/helpers/libraryFilters.js index 79695577b3..9f39daa6fb 100644 --- a/app/react/Library/helpers/libraryFilters.js +++ b/app/react/Library/helpers/libraryFilters.js @@ -56,6 +56,7 @@ function URLQueryToState(query, templates, _thesauris, _relationTypes, forcedPro search: { searchTerm, filters, + customFilters: query.customFilters, sort, order, userSelectedSorting, diff --git a/app/react/Library/helpers/requestState.js b/app/react/Library/helpers/requestState.js index dc66ccd084..144f01f5b9 100644 --- a/app/react/Library/helpers/requestState.js +++ b/app/react/Library/helpers/requestState.js @@ -5,6 +5,7 @@ import prioritySortingCriteria from 'app/utils/prioritySortingCriteria'; import rison from 'rison-node'; import { getThesaurusPropertyNames } from 'shared/commonTopicClassification'; import { setTableViewColumns } from 'app/Library/actions/libraryActions'; +import { tocGenerationUtils } from 'app/ToggledFeatures/tocGeneration'; import { wrapDispatch } from 'app/Multireducer'; import { getTableColumns } from './tableColumns'; import setReduxState from './setReduxState.js'; @@ -36,12 +37,17 @@ export function processQuery(params, globalResources, key) { } const { userSelectedSorting, ...sanitizedQuery } = query; - return sanitizedQuery; + return tocGenerationUtils.aggregations( + sanitizedQuery, + globalResources.settings.collection.toJS() + ); } export default function requestState(request, globalResources, calculateTableColumns = false) { const docsQuery = processQuery(request.data, globalResources, 'library'); - const documentsRequest = request.set(docsQuery); + const documentsRequest = request.set( + tocGenerationUtils.aggregations(docsQuery, globalResources.settings.collection.toJS()) + ); const markersRequest = request.set({ ...docsQuery, geolocation: true }); return Promise.all([api.search(documentsRequest), api.search(markersRequest)]).then( diff --git a/app/react/Library/helpers/specs/libraryFilters.spec.js b/app/react/Library/helpers/specs/libraryFilters.spec.js index fa91a8bc15..b911e63c13 100644 --- a/app/react/Library/helpers/specs/libraryFilters.spec.js +++ b/app/react/Library/helpers/specs/libraryFilters.spec.js @@ -83,10 +83,14 @@ describe('library helper', () => { sort: 'sort', types: ['3'], filters: { country: 'countryValue', rich: 'search' }, + customFilters: { + property: { values: ['value'] }, + }, }; const state = libraryHelper.URLQueryToState(query, templates); expect(state.properties.length).toBe(1); + expect(state.search.customFilters).toEqual(query.customFilters); expect(state.search.filters.country).toBe('countryValue'); expect(state.search.filters.rich).toBe('search'); expect(state.search.searchTerm).toBe('searchTerm'); diff --git a/app/react/Library/helpers/specs/resquestState.spec.js b/app/react/Library/helpers/specs/resquestState.spec.js index 7df7599f3c..6377d1f40c 100644 --- a/app/react/Library/helpers/specs/resquestState.spec.js +++ b/app/react/Library/helpers/specs/resquestState.spec.js @@ -30,6 +30,7 @@ describe('static requestState()', () => { }; const globalResources = { templates: Immutable.fromJS(templates), + settings: { collection: Immutable.fromJS({ features: {} }) }, thesauris: Immutable.fromJS(thesauris), relationTypes: Immutable.fromJS(relationTypes), }; diff --git a/app/react/Templates/utils/getFieldLabel.js b/app/react/Templates/utils/getFieldLabel.js index 8807314df6..b8828ca03a 100644 --- a/app/react/Templates/utils/getFieldLabel.js +++ b/app/react/Templates/utils/getFieldLabel.js @@ -13,7 +13,7 @@ function getMetadataFieldLabel(field, template) { export default function getFieldLabel(field, template) { const _template = template && template.toJS ? template.toJS() : template; - if (field === 'title') { + if (field === 'title' && _template) { return getTitleLabel(_template); } if (field.startsWith('metadata.') && _template) { diff --git a/app/react/Templates/utils/specs/getFieldLabel.spec.js b/app/react/Templates/utils/specs/getFieldLabel.spec.js index e248a85a6a..b79d3f8a6c 100644 --- a/app/react/Templates/utils/specs/getFieldLabel.spec.js +++ b/app/react/Templates/utils/specs/getFieldLabel.spec.js @@ -40,6 +40,13 @@ describe('getFieldLabel', () => { field = 'title'; expect(runGetLabel()).toEqual(t(template._id, 'Name')); }); + describe('when template is not defined', () => { + it('should return the input field', () => { + template = undefined; + field = 'title'; + expect(runGetLabel()).toEqual('title'); + }); + }); }); describe('when field is not in template', () => { diff --git a/app/react/Thesauri/components/specs/__snapshots__/ThesauriForm.spec.js.snap b/app/react/Thesauri/components/specs/__snapshots__/ThesauriForm.spec.js.snap index a9f6ca8c7b..27837fc89a 100644 --- a/app/react/Thesauri/components/specs/__snapshots__/ThesauriForm.spec.js.snap +++ b/app/react/Thesauri/components/specs/__snapshots__/ThesauriForm.spec.js.snap @@ -22,7 +22,9 @@ exports[`ThesauriForm render should render DragAndDropContainer with thesauri it
- + diff --git a/app/react/Thesauri/components/specs/__snapshots__/ThesauriFormGroup.spec.js.snap b/app/react/Thesauri/components/specs/__snapshots__/ThesauriFormGroup.spec.js.snap index 45634e0802..fe81b422fc 100644 --- a/app/react/Thesauri/components/specs/__snapshots__/ThesauriFormGroup.spec.js.snap +++ b/app/react/Thesauri/components/specs/__snapshots__/ThesauriFormGroup.spec.js.snap @@ -20,7 +20,9 @@ exports[`ThesauriFormGroup render should render group field and DragAndDropConta className="group" key="group-1" > - + diff --git a/app/react/ToggledFeatures/tocGeneration/FilterTocGeneration.tsx b/app/react/ToggledFeatures/tocGeneration/FilterTocGeneration.tsx new file mode 100644 index 0000000000..3753a45758 --- /dev/null +++ b/app/react/ToggledFeatures/tocGeneration/FilterTocGeneration.tsx @@ -0,0 +1,49 @@ +import React from 'react'; +import { Aggregations } from 'shared/types/Aggregations.d.ts'; +import { FeatureToggle } from 'app/components/Elements/FeatureToggle'; +import SelectFilter from 'app/Library/components/SelectFilter'; +import FormGroup from 'app/DocumentForm/components/FormGroup'; +import { t } from 'app/I18N'; +import { NeedAuthorization } from 'app/Auth'; + +export interface FilterTocGenerationProps { + onChange: () => void; + aggregations: Aggregations; +} + +const filteredAggregation = (aggregations: Aggregations, key: string) => { + const bucket = (aggregations?.all?.generatedToc?.buckets || []).find(a => a.key === key) || { + filtered: { doc_count: 0 }, + }; + return bucket.filtered.doc_count; +}; + +const options = (aggregations: Aggregations = { all: {} }) => [ + { + label: t('System', 'Automatically generated'), + value: true, + results: filteredAggregation(aggregations, 'true'), + }, + { + label: t('System', 'Reviewed'), + value: false, + results: filteredAggregation(aggregations, 'false'), + }, +]; + +export const FilterTocGeneration = ({ onChange, aggregations }: FilterTocGenerationProps) => ( + + + + + + + +); diff --git a/app/react/ToggledFeatures/tocGeneration/ReviewTocButton.tsx b/app/react/ToggledFeatures/tocGeneration/ReviewTocButton.tsx new file mode 100644 index 0000000000..20a0a42bbb --- /dev/null +++ b/app/react/ToggledFeatures/tocGeneration/ReviewTocButton.tsx @@ -0,0 +1,34 @@ +import React from 'react'; +import { Icon } from 'UI'; +import { FeatureToggle } from 'app/components/Elements/FeatureToggle'; +import { connect, ConnectedProps } from 'react-redux'; +import { bindActionCreators, Dispatch } from 'redux'; +import { ClientFile } from 'app/istore'; +import { tocGenerationActions } from './actions'; + +export interface ReviewTocButtonProps { + file: ClientFile; + children: JSX.Element | string; +} + +const mapDispatchToProps = (dispatch: Dispatch<{}>) => + bindActionCreators({ onClick: tocGenerationActions.reviewToc }, dispatch); + +const connector = connect(null, mapDispatchToProps); + +type MappedProps = ConnectedProps; +type ComponentProps = ReviewTocButtonProps & MappedProps; + +const ReviewTocButton = ({ file, onClick, children }: ComponentProps) => ( + + {file.generatedToc && ( + + )} + +); + +const container = connector(ReviewTocButton); +export { container as ReviewTocButton }; diff --git a/app/react/ToggledFeatures/tocGeneration/TocGeneratedLabel.tsx b/app/react/ToggledFeatures/tocGeneration/TocGeneratedLabel.tsx new file mode 100644 index 0000000000..32eaa258f2 --- /dev/null +++ b/app/react/ToggledFeatures/tocGeneration/TocGeneratedLabel.tsx @@ -0,0 +1,14 @@ +import React from 'react'; +import { FeatureToggle } from 'app/components/Elements/FeatureToggle'; +import { FileType } from 'shared/types/fileType'; + +export interface TocGeneratedLabelProps { + file: FileType; + children: React.ReactChild; +} + +export const TocGeneratedLabel = ({ file, children }: TocGeneratedLabelProps) => ( + + {file.generatedToc &&
{children}
} +
+); diff --git a/app/react/ToggledFeatures/tocGeneration/actions.ts b/app/react/ToggledFeatures/tocGeneration/actions.ts new file mode 100644 index 0000000000..8ab8f5b5fc --- /dev/null +++ b/app/react/ToggledFeatures/tocGeneration/actions.ts @@ -0,0 +1,36 @@ +import { actions } from 'app/BasicReducer/reducer'; +import { actions as formActions } from 'react-redux-form'; +import { RequestParams } from 'app/utils/RequestParams'; +import api from 'app/utils/api'; +import { notificationActions } from 'app/Notifications'; +import { IStore } from 'app/istore'; +import { Dispatch } from 'redux'; +import { ensure } from 'shared/tsUtils'; +import { FileType } from 'shared/types/fileType'; + +const tocGenerationActions = { + reviewToc(fileId: string) { + return async (dispatch: Dispatch, getState: () => IStore) => { + const currentDoc = getState().documentViewer.doc.toJS(); + dispatch(formActions.reset('documentViewer.sidepanel.metadata')); + + const updatedFile = (await api.post('files/tocReviewed', new RequestParams({ fileId }))).json; + const doc = { + ...currentDoc, + defaultDoc: updatedFile, + documents: ensure(currentDoc.documents).map(d => { + if (d._id === updatedFile._id) { + return updatedFile; + } + return d; + }), + }; + + dispatch(notificationActions.notify('Document updated', 'success')); + dispatch(formActions.reset('documentViewer.sidepanel.metadata')); + dispatch(actions.set('viewer/doc', doc)); + }; + }, +}; + +export { tocGenerationActions }; diff --git a/app/react/ToggledFeatures/tocGeneration/index.ts b/app/react/ToggledFeatures/tocGeneration/index.ts new file mode 100644 index 0000000000..7785be5636 --- /dev/null +++ b/app/react/ToggledFeatures/tocGeneration/index.ts @@ -0,0 +1,4 @@ +export { TocGeneratedLabel } from './TocGeneratedLabel'; +export { ReviewTocButton } from './ReviewTocButton'; +export { FilterTocGeneration } from './FilterTocGeneration'; +export { tocGenerationUtils } from './utils'; diff --git a/app/react/ToggledFeatures/tocGeneration/specs/FilterTocGeneration.spec.tsx b/app/react/ToggledFeatures/tocGeneration/specs/FilterTocGeneration.spec.tsx new file mode 100644 index 0000000000..bef7674112 --- /dev/null +++ b/app/react/ToggledFeatures/tocGeneration/specs/FilterTocGeneration.spec.tsx @@ -0,0 +1,68 @@ +import React from 'react'; +import { shallow, ShallowWrapper } from 'enzyme'; +import SelectFilter from 'app/Library/components/SelectFilter'; +import { Aggregations } from 'shared/types/Aggregations'; +import { FilterTocGeneration } from '../FilterTocGeneration'; + +describe('FilterTocGeneration', () => { + let component: ShallowWrapper; + const aggregations = { + all: { + generatedToc: { + buckets: [ + { key: 'false', filtered: { doc_count: 2 } }, + { key: 'true', filtered: { doc_count: 5 } }, + ], + }, + }, + }; + + const render = (aggs: Aggregations = aggregations) => { + component = shallow( {}} aggregations={aggs} />); + }; + + it('should render nothing if file generatedToc is false', () => { + render(); + const options = component.find(SelectFilter).prop('options'); + expect(options).toEqual([ + expect.objectContaining({ value: true, results: 5 }), + expect.objectContaining({ value: false, results: 2 }), + ]); + }); + + describe('when there is bucket missing', () => { + it('should not fail (render blank state)', () => { + render({ + all: { + generatedToc: { + buckets: [{ key: 'false', filtered: { doc_count: 2 } }], + }, + }, + }); + const options = component.find(SelectFilter).prop('options'); + expect(options).toEqual([ + expect.objectContaining({ value: true, results: 0 }), + expect.objectContaining({ value: false, results: 2 }), + ]); + }); + }); + + describe('when aggregations are not defined/complete', () => { + it('should not fail (render blank state)', () => { + //@ts-ignore + render({}); + let options = component.find(SelectFilter).prop('options'); + expect(options).toEqual([ + expect.objectContaining({ value: true, results: 0 }), + expect.objectContaining({ value: false, results: 0 }), + ]); + + render({ all: {} }); + options = component.find(SelectFilter).prop('options'); + expect(options).toEqual([ + expect.objectContaining({ value: true, results: 0 }), + expect.objectContaining({ value: false, results: 0 }), + ]); + }); + }); +}); diff --git a/app/react/ToggledFeatures/tocGeneration/specs/ReviewTocButton.spec.tsx b/app/react/ToggledFeatures/tocGeneration/specs/ReviewTocButton.spec.tsx new file mode 100644 index 0000000000..43b032cc8d --- /dev/null +++ b/app/react/ToggledFeatures/tocGeneration/specs/ReviewTocButton.spec.tsx @@ -0,0 +1,37 @@ +import React from 'react'; +import { shallow, ShallowWrapper } from 'enzyme'; +import configureStore, { MockStore, MockStoreCreator } from 'redux-mock-store'; +import { Provider } from 'react-redux'; +import { ClientFile } from 'app/istore'; +import { ReviewTocButton } from '../ReviewTocButton'; + +describe('ReviewTocButton', () => { + let component: ShallowWrapper; + + const mockStoreCreator: MockStoreCreator = configureStore([]); + const render = (file: Partial) => { + const store: MockStore = mockStoreCreator({}); + component = shallow( + + + test + + + ) + .dive() + .dive(); + }; + + it('should render nothing if file generatedToc is false', () => { + render({ generatedToc: false }); + expect(component.find('button').length).toEqual(0); + + render({}); + expect(component.find('button').length).toEqual(0); + }); + + it('should render when generatedToc is true', () => { + render({ generatedToc: true }); + expect(component.find('button').length).toEqual(1); + }); +}); diff --git a/app/react/ToggledFeatures/tocGeneration/specs/actions.spec.ts b/app/react/ToggledFeatures/tocGeneration/specs/actions.spec.ts new file mode 100644 index 0000000000..8f2763b7ac --- /dev/null +++ b/app/react/ToggledFeatures/tocGeneration/specs/actions.spec.ts @@ -0,0 +1,78 @@ +import api from 'app/utils/api'; +import backend from 'fetch-mock'; +import * as notificationsTypes from 'app/Notifications/actions/actionTypes'; +import { actions as relationshipActions } from 'app/Relationships'; +import { APIURL } from 'app/config'; +import configureMockStore from 'redux-mock-store'; +import thunk from 'redux-thunk'; +import Immutable from 'immutable'; +import { mockID } from 'shared/uniqueID.js'; +import { ClientEntitySchema } from 'app/istore'; +import { tocGenerationActions } from '../actions'; + +const middlewares = [thunk]; +const mockStore = configureMockStore(middlewares); + +const createDoc = (generatedToc: boolean, fileId: string): ClientEntitySchema => ({ + name: 'doc', + _id: 'id', + sharedId: 'sharedId', + defaultDoc: { _id: fileId, generatedToc }, + documents: [{ _id: fileId, generatedToc }], +}); + +describe('reviewToc', () => { + it('should store the document with the response of reviewToc', done => { + mockID(); + const fileId = 'fileId'; + + backend.post(`${APIURL}files/tocReviewed`, { + body: JSON.stringify({ _id: fileId, generatedToc: false }), + }); + + spyOn(relationshipActions, 'reloadRelationships').and.returnValue({ + type: 'reloadRelationships', + }); + + const doc = createDoc(true, fileId); + const updatedEntity = createDoc(false, fileId); + + const expectedActions = [ + { type: 'rrf/reset', model: 'documentViewer.sidepanel.metadata' }, + { + type: notificationsTypes.NOTIFY, + notification: { message: 'Document updated', type: 'success', id: 'unique_id' }, + }, + { type: 'rrf/reset', model: 'documentViewer.sidepanel.metadata' }, + { type: 'viewer/doc/SET', value: updatedEntity }, + ]; + + const store = mockStore({ + documentViewer: { + doc: Immutable.fromJS(doc), + references: Immutable.fromJS([]), + targetDocReferences: Immutable.fromJS([]), + targetDoc: Immutable.fromJS(doc), + uiState: Immutable.fromJS({}), + }, + }); + + spyOn(api, 'post').and.callThrough(); + store + //fot this to be properly typed, redux, redux-thunk need to be updated (and probably others), + //producing hundreds of type errors + //@ts-ignore + .dispatch(tocGenerationActions.reviewToc(fileId)) + .then(() => { + expect(api.post).toHaveBeenCalledWith('files/tocReviewed', { + data: { + fileId: 'fileId', + }, + headers: {}, + }); + expect(store.getActions()).toEqual(expectedActions); + }) + .then(done) + .catch(done.fail); + }); +}); diff --git a/app/react/ToggledFeatures/tocGeneration/specs/utils.spec.ts b/app/react/ToggledFeatures/tocGeneration/specs/utils.spec.ts new file mode 100644 index 0000000000..bfdcd09eb5 --- /dev/null +++ b/app/react/ToggledFeatures/tocGeneration/specs/utils.spec.ts @@ -0,0 +1,19 @@ +import Immutable from 'immutable'; +import { processQuery } from 'app/Library/helpers/requestState'; + +describe('Library/Uploads processQuery()', () => { + it('should add aggregateGeneratedToc if feature activated', () => { + const params = { q: '(order:desc,sort:creationDate)' }; + const globalResources = { + settings: { + collection: Immutable.fromJS({ features: { tocGeneration: {} } }), + }, + }; + const query = processQuery(params, globalResources, 'library'); + expect(query).toEqual({ + order: 'desc', + sort: 'creationDate', + aggregateGeneratedToc: true, + }); + }); +}); diff --git a/app/react/ToggledFeatures/tocGeneration/utils.ts b/app/react/ToggledFeatures/tocGeneration/utils.ts new file mode 100644 index 0000000000..42cef93ce9 --- /dev/null +++ b/app/react/ToggledFeatures/tocGeneration/utils.ts @@ -0,0 +1,11 @@ +import { Settings } from 'shared/types/settingsType'; +import { SearchParams } from 'shared/types/searchParams.d.ts'; + +export const tocGenerationUtils = { + aggregations(params: SearchParams, settings: Settings) { + return { + ...params, + ...(settings?.features?.tocGeneration ? { aggregateGeneratedToc: true } : {}), + }; + }, +}; diff --git a/app/react/UI/Icon/library.js b/app/react/UI/Icon/library.js index 17dfade388..e1f01cbcdb 100644 --- a/app/react/UI/Icon/library.js +++ b/app/react/UI/Icon/library.js @@ -89,6 +89,7 @@ import { faUsers } from '@fortawesome/free-solid-svg-icons/faUsers'; import { faUserTimes } from '@fortawesome/free-solid-svg-icons/faUserTimes'; import { faHandPaper } from '@fortawesome/free-solid-svg-icons/faHandPaper'; import { faExternalLinkAlt } from '@fortawesome/free-solid-svg-icons/faExternalLinkAlt'; +import { faTasks } from '@fortawesome/free-solid-svg-icons/faTasks'; import { saveAndNext } from './save-and-next'; import { exportCsv } from './export-csv'; import { copyFrom } from './copy-from'; @@ -186,6 +187,7 @@ const icons = { faHandPaper, faExternalLinkAlt, saveAndNext, + faTasks, exportCsv, copyFrom, funnelFilter, diff --git a/app/react/Uploads/specs/UploadsRoute.spec.js b/app/react/Uploads/specs/UploadsRoute.spec.js index 18889e06cb..644c98994d 100644 --- a/app/react/Uploads/specs/UploadsRoute.spec.js +++ b/app/react/Uploads/specs/UploadsRoute.spec.js @@ -26,6 +26,7 @@ describe('UploadsRoute', () => { ]; const globalResources = { templates: Immutable(templates), + settings: { collection: Immutable({ features: {} }) }, thesauris: Immutable([]), relationTypes: Immutable([]), }; diff --git a/app/react/istore.d.ts b/app/react/istore.d.ts index 39366d8aba..955b58df2b 100644 --- a/app/react/istore.d.ts +++ b/app/react/istore.d.ts @@ -8,6 +8,7 @@ import { TemplateSchema } from 'shared/types/templateType'; import { EntitySchema } from 'shared/types/entityType'; import { UserGroupSchema } from 'shared/types/userGroupType'; import { ConnectionSchema } from 'shared/types/connectionType'; +import { FileType } from 'shared/types/fileType'; export interface TasksState { SyncState?: TaskStatus; @@ -83,8 +84,12 @@ interface ClientTemplateSchema extends TemplateSchema { _id: string; } +export interface ClientFile extends FileType { + _id: string; +} + export interface ClientEntitySchema extends EntitySchema { - documents?: []; + documents?: ClientFile[]; } export interface IStore { @@ -110,8 +115,8 @@ export interface IStore { documentViewer: { references: IImmutable; targetDocReferences: IImmutable; - doc: IImmutable; - targetDoc: IImmutable; + doc: IImmutable; + targetDoc: IImmutable; uiState: IImmutable<{ activeReference: string; }>; diff --git a/app/server.js b/app/server.js index bd7cf6c511..1befa11243 100644 --- a/app/server.js +++ b/app/server.js @@ -30,6 +30,7 @@ import { tenants } from './api/tenants/tenantContext'; import { multitenantMiddleware } from './api/utils/multitenantMiddleware'; import { staticFilesMiddleware } from './api/utils/staticFilesMiddleware'; import { customUploadsPath, uploadsPath } from './api/files/filesystem'; +import { tocService } from './api/toc_generation/tocService'; mongoose.Promise = Promise; @@ -40,7 +41,7 @@ const http = Server(app); const uncaughtError = error => { handleError(error, { uncaught: true }); - process.exit(1); + throw error; }; process.on('unhandledRejection', uncaughtError); @@ -116,15 +117,21 @@ DB.connect(config.DBHOST, dbAuth).then(async () => { if (!config.multiTenant && !config.clusterMode) { syncWorker.start(); - const { evidencesVault } = await settings.get(); + const { evidencesVault, features } = await settings.get(); if (evidencesVault && evidencesVault.token && evidencesVault.template) { - console.info('==> 📥 evidences vault config detected, started sync ....'); + console.info('==> 📥 evidences vault config detected, started sync ....'); repeater.start( () => vaultSync.sync(evidencesVault.token, evidencesVault.template), 10000 ); } + if (features && features.tocGeneration && features.tocGeneration.url) { + console.info('==> 🗂️ automatically generating TOCs using external service'); + const service = tocService(features.tocGeneration.url); + repeater.start(() => service.processNext(), 10000); + } + repeater.start( () => TaskProvider.runAndWait('TopicClassificationSync', 'TopicClassificationSync', { diff --git a/app/shared/JSONRequest.js b/app/shared/JSONRequest.js index bf975c2f7f..cb13e6e345 100644 --- a/app/shared/JSONRequest.js +++ b/app/shared/JSONRequest.js @@ -117,7 +117,7 @@ export default { delete: (url, data, headers) => _fetch(url, data, 'DELETE', headers), - // TEST!!! Fully untested function + // TEST!!!! Fully untested function uploadFile: (url, filename, file) => new Promise((resolve, reject) => { superagent @@ -126,8 +126,8 @@ export default { .set('X-Requested-With', 'XMLHttpRequest') .set('Cookie', cookie || '') .attach('file', file, filename) - .then(() => { - resolve(); + .then(response => { + resolve(response); }) .catch(err => { reject(err); diff --git a/app/shared/types/Aggregations.d.ts b/app/shared/types/Aggregations.d.ts new file mode 100644 index 0000000000..e1c059a581 --- /dev/null +++ b/app/shared/types/Aggregations.d.ts @@ -0,0 +1,13 @@ +export interface Aggregations { + all: { + [key: string]: { + buckets: Array<{ + key: string; + filtered: { + // eslint-disable-next-line camelcase + doc_count: number; + }; + }>; + }; + }; +} diff --git a/app/shared/types/commonSchemas.ts b/app/shared/types/commonSchemas.ts index 67b411f6b4..f7e037d87c 100644 --- a/app/shared/types/commonSchemas.ts +++ b/app/shared/types/commonSchemas.ts @@ -83,6 +83,7 @@ export const propertyValueSchema = { { type: 'null' }, { type: 'string' }, { type: 'number' }, + { type: 'boolean' }, linkSchema, dateRangeSchema, latLonSchema, diff --git a/app/shared/types/commonTypes.d.ts b/app/shared/types/commonTypes.d.ts index c2596d0e9c..0c7fcf2340 100644 --- a/app/shared/types/commonTypes.d.ts +++ b/app/shared/types/commonTypes.d.ts @@ -52,7 +52,15 @@ export type GeolocationSchema = { lon: number; }[]; -export type PropertyValueSchema = null | string | number | LinkSchema | DateRangeSchema | LatLonSchema | LatLonSchema[]; +export type PropertyValueSchema = + | null + | string + | number + | boolean + | LinkSchema + | DateRangeSchema + | LatLonSchema + | LatLonSchema[]; export interface MetadataObjectSchema { value: PropertyValueSchema; diff --git a/app/shared/types/connectionType.d.ts b/app/shared/types/connectionType.d.ts index 0bcd15b543..1a026f5731 100644 --- a/app/shared/types/connectionType.d.ts +++ b/app/shared/types/connectionType.d.ts @@ -19,6 +19,7 @@ export interface ConnectionSchema { title?: string; template?: ObjectIdSchema; published?: boolean; + generatedToc?: boolean; icon?: { _id?: string | null; label?: string; @@ -42,6 +43,7 @@ export interface ConnectionSchema { | null | string | number + | boolean | { label?: string | null; url?: string | null; @@ -75,6 +77,7 @@ export interface ConnectionSchema { | null | string | number + | boolean | { label?: string | null; url?: string | null; diff --git a/app/shared/types/entitySchema.ts b/app/shared/types/entitySchema.ts index c9ab3a6122..ffe6441f67 100644 --- a/app/shared/types/entitySchema.ts +++ b/app/shared/types/entitySchema.ts @@ -128,6 +128,7 @@ export const entitySchema = { title: { type: 'string', minLength: 1, stringMeetsLuceneMaxLimit: true }, template: objectIdSchema, published: { type: 'boolean' }, + generatedToc: { type: 'boolean' }, icon: { type: 'object', additionalProperties: false, diff --git a/app/shared/types/entityType.d.ts b/app/shared/types/entityType.d.ts index 5c76a73570..337cfdc0aa 100644 --- a/app/shared/types/entityType.d.ts +++ b/app/shared/types/entityType.d.ts @@ -11,6 +11,7 @@ export interface EntitySchema { title?: string; template?: ObjectIdSchema; published?: boolean; + generatedToc?: boolean; icon?: { _id?: string | null; label?: string; diff --git a/app/shared/types/fileSchema.ts b/app/shared/types/fileSchema.ts index 9962c6906b..0a9d4a3e49 100644 --- a/app/shared/types/fileSchema.ts +++ b/app/shared/types/fileSchema.ts @@ -26,6 +26,7 @@ export const fileSchema = { type: { type: 'string', enum: ['custom', 'document', 'thumbnail'] }, status: { type: 'string', enum: ['processing', 'failed', 'ready'] }, totalPages: { type: 'number' }, + generatedToc: { type: 'boolean' }, fullText: { type: 'object', additionalProperties: false, diff --git a/app/shared/types/fileType.d.ts b/app/shared/types/fileType.d.ts index 8ca9b2f47d..a3bfcc02d7 100644 --- a/app/shared/types/fileType.d.ts +++ b/app/shared/types/fileType.d.ts @@ -15,6 +15,7 @@ export interface FileType { type?: 'custom' | 'document' | 'thumbnail'; status?: 'processing' | 'failed' | 'ready'; totalPages?: number; + generatedToc?: boolean; fullText?: { /** * This interface was referenced by `undefined`'s JSON-Schema definition diff --git a/app/shared/types/searchParams.d.ts b/app/shared/types/searchParams.d.ts new file mode 100644 index 0000000000..e756602d11 --- /dev/null +++ b/app/shared/types/searchParams.d.ts @@ -0,0 +1,33 @@ +/* eslint-disable */ +/**AUTO-GENERATED. RUN yarn emit-types to update.*/ + +export interface SearchParams { + query?: { + aggregateGeneratedToc?: boolean; + filters?: { + [k: string]: unknown | undefined; + }; + customFilters?: { + generatedToc?: { + values?: [] | [boolean]; + }; + }; + types?: [] | [string]; + _types?: [] | [string]; + fields?: [] | [string]; + allAggregations?: boolean; + aggregations?: string; + order?: 'asc' | 'desc'; + sort?: string; + limit?: number; + from?: number; + searchTerm?: string | number; + includeUnpublished?: boolean; + userSelectedSorting?: boolean; + treatAs?: string; + unpublished?: boolean; + select?: [] | [string]; + geolocation?: boolean; + }; + [k: string]: unknown | undefined; +} diff --git a/app/api/search/searchSchema.ts b/app/shared/types/searchParams.ts similarity index 61% rename from app/api/search/searchSchema.ts rename to app/shared/types/searchParams.ts index 26b92ecfd9..38be2b30da 100644 --- a/app/api/search/searchSchema.ts +++ b/app/shared/types/searchParams.ts @@ -1,8 +1,25 @@ -export const searchSchema = { +export const emitSchemaTypes = true; +export const searchParamsSchema = { + title: 'searchParams', properties: { query: { + additionalProperties: false, properties: { + aggregateGeneratedToc: { type: 'boolean' }, filters: { type: 'object' }, + customFilters: { + additionalProperties: false, + type: 'object', + properties: { + generatedToc: { + type: 'object', + additionalProperties: false, + properties: { + values: { type: 'array', items: [{ type: 'boolean' }] }, + }, + }, + }, + }, types: { type: 'array', items: [{ type: 'string' }] }, _types: { type: 'array', items: [{ type: 'string' }] }, fields: { type: 'array', items: [{ type: 'string' }] }, diff --git a/app/shared/types/settingsSchema.ts b/app/shared/types/settingsSchema.ts index afbac4a7d4..d2657be092 100644 --- a/app/shared/types/settingsSchema.ts +++ b/app/shared/types/settingsSchema.ts @@ -133,7 +133,14 @@ export const settingsSchema = { type: 'object', properties: { _id: { type: 'string' }, - semanticSearch: { type: 'boolean' }, + tocGeneration: { + type: 'object', + required: ['url'], + additionalProperties: false, + properties: { + url: { type: 'string' }, + }, + }, topicClassification: { type: 'boolean' }, favorites: { type: 'boolean' }, }, diff --git a/app/shared/types/settingsType.d.ts b/app/shared/types/settingsType.d.ts index 842b7507d3..a882056e3d 100644 --- a/app/shared/types/settingsType.d.ts +++ b/app/shared/types/settingsType.d.ts @@ -84,7 +84,9 @@ export interface Settings { links?: SettingsLinkSchema[]; features?: { _id?: string; - semanticSearch?: boolean; + tocGeneration?: { + url: string; + }; topicClassification?: boolean; favorites?: boolean; [k: string]: unknown | undefined; diff --git a/database/elastic_mapping/base_properties.js b/database/elastic_mapping/base_properties.js index 1ad1d5efbc..e37004fea4 100644 --- a/database/elastic_mapping/base_properties.js +++ b/database/elastic_mapping/base_properties.js @@ -64,6 +64,9 @@ const properties = { sort: { type: 'keyword' }, }, }, + generatedToc: { + type: 'keyword', + }, type: { type: 'keyword', }, diff --git a/database/elastic_mapping/elasticMapFactory.ts b/database/elastic_mapping/elasticMapFactory.ts index fcd3a9369d..d141033d0e 100644 --- a/database/elastic_mapping/elasticMapFactory.ts +++ b/database/elastic_mapping/elasticMapFactory.ts @@ -25,7 +25,6 @@ export default { const fieldMapping = propertyMappings[property.type](); map.properties.metadata.properties[property.name] = { properties: fieldMapping }; map.properties.suggestedMetadata.properties[property.name] = { properties: fieldMapping }; - return map; }, baseMapping), baseMappingObject diff --git a/emitSchemaTypes.js b/emitSchemaTypes.js index 9f0c2df6fe..bef0a65946 100644 --- a/emitSchemaTypes.js +++ b/emitSchemaTypes.js @@ -61,30 +61,36 @@ const writeTypeFile = (file, commonImport, snippets) => { } }; -const emitSchemaTypes = async file => { - try { - if (!file.match(/Schema/) || file.match(/spec/)) { - return; - } +const writeSchema = async (schemas, file) => { + const snippets = await Promise.all( + Object.entries(schemas).map(([name, schema]) => { + if (!name.match(/Schema$/)) { + return ''; + } + return compile(schema, schema.title || firstUp(name), opts); + }) + ); + + const contents = fs.readFileSync(file).toString(); - const schemas = require(`./${file}`); + writeTypeFile(file, typeImports(contents.match(typeImportFindRegex)), snippets); +}; - if (!schemas.emitSchemaTypes) { +const emitSchemaTypes = async file => { + try { + if (file.match(/spec/)) { return; } - const snippets = await Promise.all( - Object.entries(schemas).map(([name, schema]) => { - if (!name.match(/Schema$/)) { - return ''; - } - return compile(schema, schema.title || firstUp(name), opts); - }) - ); + if (file.match(/shared\/types/) || file.match(/Schema/)) { + const schemas = require(`./${file}`); - const contents = fs.readFileSync(file).toString(); + if (!schemas.emitSchemaTypes) { + return; + } - writeTypeFile(file, typeImports(contents.match(typeImportFindRegex)), snippets); + writeSchema(schemas, file); + } } catch (err) { console.error(`Failed emitting types from ${file}: ${err}.`); } diff --git a/nightmare/helpers/selectors.js b/nightmare/helpers/selectors.js index 2ea352ce11..1d7f651c26 100644 --- a/nightmare/helpers/selectors.js +++ b/nightmare/helpers/selectors.js @@ -223,7 +223,7 @@ export default { sidePanelFirstDocumentTitle: '#app > div.content > div > div > aside.side-panel.metadata-sidepanel.is-active > div.sidepanel-body > div > div.tab-content-visible > div > div.filelist > ul > li > div > div.file-originalname', sidePanelFirstDocumentEditButton: - '#app > div.content > div > div > aside.side-panel.metadata-sidepanel.is-active > div.sidepanel-body > div > div.tab-content-visible > div > div.filelist > ul > li > div > div:nth-child(2) > button', + '#app > div.content > div > div > aside.side-panel.metadata-sidepanel.is-active > div.sidepanel-body > div > div.metadata.tab-content-visible > div > div.filelist > ul > li > div > div:nth-child(2) > div:nth-child(2) > button', fileFormInput: '#originalname', fileFormSubmit: '#app > div.content > div > div > aside.side-panel.metadata-sidepanel.is-active > div.sidepanel-body > div > div.tab-content-visible > div > div.filelist > ul > li > form > div > div:nth-child(4) > button',