From 8ad80a4860ecf686a57adda720072b14e173ea39 Mon Sep 17 00:00:00 2001 From: Daneryl Date: Mon, 25 Jan 2021 16:24:33 -0500 Subject: [PATCH 01/31] toc_generation service, WIP --- app/api/entities/entitiesModel.ts | 1 + app/api/search/documentQueryBuilder.js | 8 ++- app/api/search/metadataAggregations.js | 7 ++ app/api/toc_generation/specs/fixtures.ts | 48 ++++++++++++++ .../toc_generation/specs/tocService.spec.ts | 66 +++++++++++++++++++ app/api/toc_generation/tocService.ts | 37 +++++++++++ app/shared/types/commonSchemas.ts | 1 + app/shared/types/commonTypes.d.ts | 10 ++- app/shared/types/entitySchema.ts | 1 + app/shared/types/entityType.d.ts | 1 + app/shared/types/fileSchema.ts | 1 + app/shared/types/fileType.d.ts | 1 + 12 files changed, 180 insertions(+), 2 deletions(-) create mode 100644 app/api/toc_generation/specs/fixtures.ts create mode 100644 app/api/toc_generation/specs/tocService.spec.ts create mode 100644 app/api/toc_generation/tocService.ts diff --git a/app/api/entities/entitiesModel.ts b/app/api/entities/entitiesModel.ts index 87143cf71c..2230b711d8 100644 --- a/app/api/entities/entitiesModel.ts +++ b/app/api/entities/entitiesModel.ts @@ -32,6 +32,7 @@ const mongoSchema = new mongoose.Schema( creationDate: Number, editDate: Number, metadata: mongoose.Schema.Types.Mixed, + systemMetadata: mongoose.Schema.Types.Mixed, suggestedMetadata: mongoose.Schema.Types.Mixed, user: { type: mongoose.Schema.Types.ObjectId, ref: 'users' }, }, diff --git a/app/api/search/documentQueryBuilder.js b/app/api/search/documentQueryBuilder.js index 19d2b77b48..e96e5dbafd 100644 --- a/app/api/search/documentQueryBuilder.js +++ b/app/api/search/documentQueryBuilder.js @@ -2,7 +2,7 @@ import { preloadOptionsSearch } from 'shared/config'; import filterToMatch, { multiselectFilter } from './metadataMatchers'; -import { propertyToAggregation } from './metadataAggregations'; +import { propertyToAggregation, generatedTocAggregations } from './metadataAggregations'; export default function() { const baseQuery = { @@ -40,6 +40,7 @@ export default function() { aggregations: { all: { global: {}, + customAggregations: {}, aggregations: { _types: { terms: { @@ -241,8 +242,13 @@ export default function() { return this; }, + generatedTOCAggregations() { + baseQuery.aggregations.all.customAggregations.generatedToc = generatedTocAggregations(baseQuery); + }, + aggregations(properties, dictionaries) { properties.forEach(property => { + // console.log(JSON.stringify(propertyToAggregation(property, dictionaries, baseQuery), null, ' ')); baseQuery.aggregations.all.aggregations[property.name] = propertyToAggregation( property, dictionaries, diff --git a/app/api/search/metadataAggregations.js b/app/api/search/metadataAggregations.js index a44f6acae5..740e975f10 100644 --- a/app/api/search/metadataAggregations.js +++ b/app/api/search/metadataAggregations.js @@ -167,3 +167,10 @@ export const propertyToAggregation = (property, dictionaries, baseQuery, suggest return aggregation(path, should, filters); }; + +export const generatedTocAggregations = baseQuery => { + const path = 'generatedToc.raw'; + const filters = extractFilters(baseQuery, path); + const { should } = baseQuery.query.bool; + return aggregation(path, should, filters); +}; diff --git a/app/api/toc_generation/specs/fixtures.ts b/app/api/toc_generation/specs/fixtures.ts new file mode 100644 index 0000000000..49eea0d873 --- /dev/null +++ b/app/api/toc_generation/specs/fixtures.ts @@ -0,0 +1,48 @@ +import { testingDB, DBFixture } from 'api/utils/testing_db'; + +const fixtures: DBFixture = { + entities: [ + { + sharedId: 'shared1', + title: 'pdf1entity', + }, + { + sharedId: 'shared3', + title: 'pdf3entity', + }, + ], + files: [ + { + _id: testingDB.id(), + entity: 'shared1', + filename: 'pdf1.pdf', + originalname: 'originalPdf1.pdf', + type: 'document', + }, + { + _id: testingDB.id(), + type: 'document', + }, + { + _id: testingDB.id(), + type: 'custom', + filename: 'background.jpg', + }, + { + _id: testingDB.id(), + type: 'document', + filename: 'pdf2.pdf', + originalname: 'originalPdf2.pdf', + toc: [{}], + }, + { + _id: testingDB.id(), + entity: 'shared3', + type: 'document', + filename: 'pdf3.pdf', + originalname: 'originalPdf4.pdf', + }, + ], +}; + +export { fixtures }; diff --git a/app/api/toc_generation/specs/tocService.spec.ts b/app/api/toc_generation/specs/tocService.spec.ts new file mode 100644 index 0000000000..e38663e23a --- /dev/null +++ b/app/api/toc_generation/specs/tocService.spec.ts @@ -0,0 +1,66 @@ +import { testingDB } from 'api/utils/testing_db'; +import request from 'shared/JSONRequest'; +import { files } from 'api/files'; +import { elasticTesting } from 'api/utils/elastic_testing'; +import { fixtures } from './fixtures'; +import { tocService } from '../tocService'; + +describe('tocService', () => { + beforeAll(async () => { + // const elasticIndex = 'toc.service.index'; + // await testingDB.clearAllAndLoad(fixtures, elasticIndex); + spyOn(request, 'uploadFile').and.callFake(async (_url, filename) => { + if (filename === 'pdf1.pdf') { + return Promise.resolve([{ label: 'section1 pdf1' }]); + } + if (filename === 'pdf3.pdf') { + return Promise.resolve([{ label: 'section1 pdf3' }]); + } + throw new Error(`this file is not supposed to be sent for toc generation ${filename}`); + }); + }); + + afterAll(async () => { + await testingDB.disconnect(); + }); + + describe('processNext', () => { + beforeAll(async () => { + const elasticIndex = 'toc.service.index'; + await testingDB.clearAllAndLoad(fixtures, elasticIndex); + await elasticTesting.resetIndex(); + await tocService.processNext(); + await tocService.processNext(); + await elasticTesting.refresh(); + }); + + it('should not fail when there is no more to process', async () => { + await expect(tocService.processNext()).resolves.not.toThrow(); + }); + + it('should send the next pdfFile and save toc generated', async () => { + let [fileProcessed] = await files.get({ filename: 'pdf1.pdf' }); + expect(fileProcessed.toc).toEqual([{ label: 'section1 pdf1' }]); + expect(fileProcessed.generatedToc).toEqual(true); + + [fileProcessed] = await files.get({ filename: 'pdf3.pdf' }); + expect(fileProcessed.toc).toEqual([{ label: 'section1 pdf3' }]); + expect(fileProcessed.generatedToc).toEqual(true); + }); + + it('should reindex the affected entities', async () => { + const entitiesIndexed = await elasticTesting.getIndexedEntities(); + + expect(entitiesIndexed).toEqual([ + expect.objectContaining({ + title: 'pdf1entity', + systemMetadata: { generatedToc: [{ value: true }] }, + }), + expect.objectContaining({ + title: 'pdf3entity', + systemMetadata: { generatedToc: [{ value: true }] }, + }), + ]); + }); + }); +}); diff --git a/app/api/toc_generation/tocService.ts b/app/api/toc_generation/tocService.ts new file mode 100644 index 0000000000..0243afc87e --- /dev/null +++ b/app/api/toc_generation/tocService.ts @@ -0,0 +1,37 @@ +import { files, uploadsPath } from 'api/files'; +import request from 'shared/JSONRequest'; +import entities from 'api/entities'; + +const tocService = { + async processNext() { + const [nextFile] = await files.get( + { toc: { $exists: false }, type: 'document', filename: { $exists: true } }, + '', + { + sort: { _id: 1 }, + limit: 1, + } + ); + + if (!nextFile) { + return null; + } + + const toc = await request.uploadFile( + 'url_toc_service', + nextFile.filename, + uploadsPath(nextFile.filename) + ); + + await files.save({ ...nextFile, toc, generatedToc: true }); + const parentEntities = await entities.get({ sharedId: nextFile.entity }, { language: 1 }); + return entities.saveMultiple( + parentEntities.map(entity => ({ + ...entity, + systemMetadata: { ...(entity.systemMetadata || {}), generatedToc: [{ value: true }] }, + })) + ); + }, +}; + +export { tocService }; diff --git a/app/shared/types/commonSchemas.ts b/app/shared/types/commonSchemas.ts index 67b411f6b4..f7e037d87c 100644 --- a/app/shared/types/commonSchemas.ts +++ b/app/shared/types/commonSchemas.ts @@ -83,6 +83,7 @@ export const propertyValueSchema = { { type: 'null' }, { type: 'string' }, { type: 'number' }, + { type: 'boolean' }, linkSchema, dateRangeSchema, latLonSchema, diff --git a/app/shared/types/commonTypes.d.ts b/app/shared/types/commonTypes.d.ts index c2596d0e9c..0c7fcf2340 100644 --- a/app/shared/types/commonTypes.d.ts +++ b/app/shared/types/commonTypes.d.ts @@ -52,7 +52,15 @@ export type GeolocationSchema = { lon: number; }[]; -export type PropertyValueSchema = null | string | number | LinkSchema | DateRangeSchema | LatLonSchema | LatLonSchema[]; +export type PropertyValueSchema = + | null + | string + | number + | boolean + | LinkSchema + | DateRangeSchema + | LatLonSchema + | LatLonSchema[]; export interface MetadataObjectSchema { value: PropertyValueSchema; diff --git a/app/shared/types/entitySchema.ts b/app/shared/types/entitySchema.ts index c9ab3a6122..11edd2771d 100644 --- a/app/shared/types/entitySchema.ts +++ b/app/shared/types/entitySchema.ts @@ -144,6 +144,7 @@ export const entitySchema = { creationDate: { type: 'number' }, user: objectIdSchema, metadata: metadataSchema, + systemMetadata: metadataSchema, suggestedMetadata: metadataSchema, }, }; diff --git a/app/shared/types/entityType.d.ts b/app/shared/types/entityType.d.ts index 5c76a73570..dd2cb3f471 100644 --- a/app/shared/types/entityType.d.ts +++ b/app/shared/types/entityType.d.ts @@ -20,6 +20,7 @@ export interface EntitySchema { creationDate?: number; user?: ObjectIdSchema; metadata?: MetadataSchema; + systemMetadata?: MetadataSchema; suggestedMetadata?: MetadataSchema; [k: string]: unknown | undefined; } diff --git a/app/shared/types/fileSchema.ts b/app/shared/types/fileSchema.ts index 9962c6906b..0a9d4a3e49 100644 --- a/app/shared/types/fileSchema.ts +++ b/app/shared/types/fileSchema.ts @@ -26,6 +26,7 @@ export const fileSchema = { type: { type: 'string', enum: ['custom', 'document', 'thumbnail'] }, status: { type: 'string', enum: ['processing', 'failed', 'ready'] }, totalPages: { type: 'number' }, + generatedToc: { type: 'boolean' }, fullText: { type: 'object', additionalProperties: false, diff --git a/app/shared/types/fileType.d.ts b/app/shared/types/fileType.d.ts index 8ca9b2f47d..a3bfcc02d7 100644 --- a/app/shared/types/fileType.d.ts +++ b/app/shared/types/fileType.d.ts @@ -15,6 +15,7 @@ export interface FileType { type?: 'custom' | 'document' | 'thumbnail'; status?: 'processing' | 'failed' | 'ready'; totalPages?: number; + generatedToc?: boolean; fullText?: { /** * This interface was referenced by `undefined`'s JSON-Schema definition From 282a5a0340863a1043dbbc93eae6e7f20ea3b6a3 Mon Sep 17 00:00:00 2001 From: Daneryl Date: Wed, 10 Feb 2021 14:35:31 -0500 Subject: [PATCH 02/31] aggregations for generatedToc property --- app/api/entities/entitiesModel.ts | 2 +- app/api/search/documentQueryBuilder.js | 4 +--- app/api/search/metadataAggregations.js | 6 ++++-- app/api/search/search.js | 12 ++++++++++-- app/api/search/searchSchema.ts | 1 + app/api/search/specs/fixtures_elastic.js | 12 ++++-------- app/api/search/specs/search.spec.js | 9 +++++++++ app/api/toc_generation/specs/tocService.spec.ts | 4 ++-- app/api/toc_generation/tocService.ts | 2 +- app/shared/types/entitySchema.ts | 2 +- app/shared/types/entityType.d.ts | 2 +- database/elastic_mapping/base_properties.js | 3 +++ database/elastic_mapping/elasticMapFactory.ts | 1 - 13 files changed, 38 insertions(+), 22 deletions(-) diff --git a/app/api/entities/entitiesModel.ts b/app/api/entities/entitiesModel.ts index 2230b711d8..9c62693655 100644 --- a/app/api/entities/entitiesModel.ts +++ b/app/api/entities/entitiesModel.ts @@ -15,6 +15,7 @@ const mongoSchema = new mongoose.Schema( title: { type: String, required: true }, template: { type: mongoose.Schema.Types.ObjectId, ref: 'templates', index: true }, published: Boolean, + generatedToc: Boolean, icon: new mongoose.Schema({ _id: String, label: String, @@ -32,7 +33,6 @@ const mongoSchema = new mongoose.Schema( creationDate: Number, editDate: Number, metadata: mongoose.Schema.Types.Mixed, - systemMetadata: mongoose.Schema.Types.Mixed, suggestedMetadata: mongoose.Schema.Types.Mixed, user: { type: mongoose.Schema.Types.ObjectId, ref: 'users' }, }, diff --git a/app/api/search/documentQueryBuilder.js b/app/api/search/documentQueryBuilder.js index e96e5dbafd..65679ce7e4 100644 --- a/app/api/search/documentQueryBuilder.js +++ b/app/api/search/documentQueryBuilder.js @@ -40,7 +40,6 @@ export default function() { aggregations: { all: { global: {}, - customAggregations: {}, aggregations: { _types: { terms: { @@ -243,12 +242,11 @@ export default function() { }, generatedTOCAggregations() { - baseQuery.aggregations.all.customAggregations.generatedToc = generatedTocAggregations(baseQuery); + baseQuery.aggregations.all.aggregations.generatedToc = generatedTocAggregations(baseQuery); }, aggregations(properties, dictionaries) { properties.forEach(property => { - // console.log(JSON.stringify(propertyToAggregation(property, dictionaries, baseQuery), null, ' ')); baseQuery.aggregations.all.aggregations[property.name] = propertyToAggregation( property, dictionaries, diff --git a/app/api/search/metadataAggregations.js b/app/api/search/metadataAggregations.js index 740e975f10..aac071e4a0 100644 --- a/app/api/search/metadataAggregations.js +++ b/app/api/search/metadataAggregations.js @@ -169,8 +169,10 @@ export const propertyToAggregation = (property, dictionaries, baseQuery, suggest }; export const generatedTocAggregations = baseQuery => { - const path = 'generatedToc.raw'; + const path = 'generatedToc'; const filters = extractFilters(baseQuery, path); const { should } = baseQuery.query.bool; - return aggregation(path, should, filters); + const agg = aggregation(path, should, filters); + agg.terms.missing = 'false'; + return agg; }; diff --git a/app/api/search/search.js b/app/api/search/search.js index ad93ead8c4..8475759233 100644 --- a/app/api/search/search.js +++ b/app/api/search/search.js @@ -195,6 +195,9 @@ const indexedDictionaryValues = dictionary => }, {}); const _getAggregationDictionary = async (aggregation, language, property, dictionaries) => { + if (!property) { + return [{values: []}, {}]; + } if (property.type === 'relationship') { const entitiesSharedId = aggregation.buckets.map(bucket => bucket.key); @@ -241,7 +244,7 @@ const _denormalizeAggregations = async (aggregations, templates, dictionaries, l const properties = propertiesHelper.allUniqueProperties(templates); return Object.keys(aggregations).reduce(async (denormaLizedAgregationsPromise, key) => { const denormaLizedAgregations = await denormaLizedAgregationsPromise; - if (!aggregations[key].buckets || key === '_types' || aggregations[key].type === 'nested') { + if (!aggregations[key].buckets || key === '_types' || aggregations[key].type === 'nested' || key === 'generatedToc') { return Object.assign(denormaLizedAgregations, { [key]: aggregations[key] }); } @@ -377,6 +380,7 @@ const processResponse = async (response, templates, dictionaries, language, filt result._id = hit._id; return result; }); + const sanitizedAggregations = await _sanitizeAggregations( response.body.aggregations.all, templates, @@ -618,6 +622,10 @@ const search = { searchGeolocation(queryBuilder, templates); } + if (query.aggregateGeneratedToc) { + queryBuilder.generatedTOCAggregations(); + } + // queryBuilder.query() is the actual call return elastic .search({ body: queryBuilder.query() }) @@ -675,7 +683,7 @@ const search = { return snippetsFromSearchHit(response.body.hits.hits[0]); }, - async indexEntities(query, select = '', limit, batchCallback = () => {}) { + async indexEntities(query, select = '', limit, batchCallback = () => { }) { return indexEntities({ query, select, diff --git a/app/api/search/searchSchema.ts b/app/api/search/searchSchema.ts index 26b92ecfd9..9b67c6a1b1 100644 --- a/app/api/search/searchSchema.ts +++ b/app/api/search/searchSchema.ts @@ -2,6 +2,7 @@ export const searchSchema = { properties: { query: { properties: { + aggregateGeneratedToc: { type: 'boolean' }, filters: { type: 'object' }, types: { type: 'array', items: [{ type: 'string' }] }, _types: { type: 'array', items: [{ type: 'string' }] }, diff --git a/app/api/search/specs/fixtures_elastic.js b/app/api/search/specs/fixtures_elastic.js index 25d0432e2f..86cb00f2c7 100644 --- a/app/api/search/specs/fixtures_elastic.js +++ b/app/api/search/specs/fixtures_elastic.js @@ -92,6 +92,7 @@ export const fixtures = { title: 'Batman finishes es', published: true, user: userId, + generatedToc: true, metadata: { relationship: [ { value: batmanBegins, label: 'Batman begins es' }, @@ -115,6 +116,7 @@ export const fixtures = { language: 'es', title: 'Batman begins es', published: true, + generatedToc: false, user: userId, }, { @@ -176,10 +178,7 @@ export const fixtures = { field2: [{ value: 'bane' }], select1: [{ value: 'EgyptID', label: 'Egypt' }], rich_text: [{ value: 'rich' }], - multiselect1: [ - { value: 'EgyptID', label: 'Egypt' }, - { value: 'SpainID', label: 'Spain' }, - ], + multiselect1: [{ value: 'EgyptID', label: 'Egypt' }, { value: 'SpainID', label: 'Spain' }], groupedDictionary: [{ value: 'GermanyID' }], nestedField_nested: [{ value: { nested1: ['1', '2', '3'] } }], city_geolocation: [{ value: { lat: 1, lon: 2 } }], @@ -437,10 +436,7 @@ export const fixtures = { { label: 'Europe', id: 'EuropeID', - values: [ - { label: 'Germany', id: 'GermanyID' }, - { label: 'France', id: 'franceID' }, - ], + values: [{ label: 'Germany', id: 'GermanyID' }, { label: 'France', id: 'franceID' }], }, ], }, diff --git a/app/api/search/specs/search.spec.js b/app/api/search/specs/search.spec.js index 63a9bafdf6..e7152ea72a 100644 --- a/app/api/search/specs/search.spec.js +++ b/app/api/search/specs/search.spec.js @@ -163,6 +163,15 @@ describe('search', () => { .catch(catchErrors(done)); }); + it('should return generatedToc aggregations when requested for', async () => { + const response = await search.search({ aggregateGeneratedToc: true }, 'es'); + + const aggregations = response.aggregations.all.generatedToc.buckets; + + expect(aggregations.find(a => a.key === 'false').filtered.doc_count).toBe(3); + expect(aggregations.find(a => a.key === 'true').filtered.doc_count).toBe(1); + }); + it('should return aggregations when searching by 2 terms', done => { search .search({ searchTerm: 'english document' }, 'es') diff --git a/app/api/toc_generation/specs/tocService.spec.ts b/app/api/toc_generation/specs/tocService.spec.ts index e38663e23a..f9b2285b9a 100644 --- a/app/api/toc_generation/specs/tocService.spec.ts +++ b/app/api/toc_generation/specs/tocService.spec.ts @@ -54,11 +54,11 @@ describe('tocService', () => { expect(entitiesIndexed).toEqual([ expect.objectContaining({ title: 'pdf1entity', - systemMetadata: { generatedToc: [{ value: true }] }, + generatedToc: true, }), expect.objectContaining({ title: 'pdf3entity', - systemMetadata: { generatedToc: [{ value: true }] }, + generatedToc: true, }), ]); }); diff --git a/app/api/toc_generation/tocService.ts b/app/api/toc_generation/tocService.ts index 0243afc87e..80a5bdecd0 100644 --- a/app/api/toc_generation/tocService.ts +++ b/app/api/toc_generation/tocService.ts @@ -28,7 +28,7 @@ const tocService = { return entities.saveMultiple( parentEntities.map(entity => ({ ...entity, - systemMetadata: { ...(entity.systemMetadata || {}), generatedToc: [{ value: true }] }, + generatedToc: true, })) ); }, diff --git a/app/shared/types/entitySchema.ts b/app/shared/types/entitySchema.ts index 11edd2771d..ffe6441f67 100644 --- a/app/shared/types/entitySchema.ts +++ b/app/shared/types/entitySchema.ts @@ -128,6 +128,7 @@ export const entitySchema = { title: { type: 'string', minLength: 1, stringMeetsLuceneMaxLimit: true }, template: objectIdSchema, published: { type: 'boolean' }, + generatedToc: { type: 'boolean' }, icon: { type: 'object', additionalProperties: false, @@ -144,7 +145,6 @@ export const entitySchema = { creationDate: { type: 'number' }, user: objectIdSchema, metadata: metadataSchema, - systemMetadata: metadataSchema, suggestedMetadata: metadataSchema, }, }; diff --git a/app/shared/types/entityType.d.ts b/app/shared/types/entityType.d.ts index dd2cb3f471..337cfdc0aa 100644 --- a/app/shared/types/entityType.d.ts +++ b/app/shared/types/entityType.d.ts @@ -11,6 +11,7 @@ export interface EntitySchema { title?: string; template?: ObjectIdSchema; published?: boolean; + generatedToc?: boolean; icon?: { _id?: string | null; label?: string; @@ -20,7 +21,6 @@ export interface EntitySchema { creationDate?: number; user?: ObjectIdSchema; metadata?: MetadataSchema; - systemMetadata?: MetadataSchema; suggestedMetadata?: MetadataSchema; [k: string]: unknown | undefined; } diff --git a/database/elastic_mapping/base_properties.js b/database/elastic_mapping/base_properties.js index 1ad1d5efbc..e37004fea4 100644 --- a/database/elastic_mapping/base_properties.js +++ b/database/elastic_mapping/base_properties.js @@ -64,6 +64,9 @@ const properties = { sort: { type: 'keyword' }, }, }, + generatedToc: { + type: 'keyword', + }, type: { type: 'keyword', }, diff --git a/database/elastic_mapping/elasticMapFactory.ts b/database/elastic_mapping/elasticMapFactory.ts index fcd3a9369d..d141033d0e 100644 --- a/database/elastic_mapping/elasticMapFactory.ts +++ b/database/elastic_mapping/elasticMapFactory.ts @@ -25,7 +25,6 @@ export default { const fieldMapping = propertyMappings[property.type](); map.properties.metadata.properties[property.name] = { properties: fieldMapping }; map.properties.suggestedMetadata.properties[property.name] = { properties: fieldMapping }; - return map; }, baseMapping), baseMappingObject From 60263ad4f63748ee99ab9ba7f7672af8642082bc Mon Sep 17 00:00:00 2001 From: Daneryl Date: Thu, 11 Feb 2021 08:35:12 -0500 Subject: [PATCH 03/31] review toc endpoint --- app/api/entities/entities.js | 4 +++ app/api/entities/specs/entities.spec.js | 5 ++- app/api/files/files.ts | 25 +++++++++++++++ app/api/files/routes.ts | 23 ++++++++++++++ app/api/files/specs/fixtures.ts | 40 ++++++++++++++++++++---- app/api/files/specs/routes.spec.ts | 36 +++++++++++++++++++-- app/api/files/specs/uploadRoutes.spec.ts | 17 +++++----- app/api/toc_generation/specs/fixtures.ts | 15 +++++++++ app/api/toc_generation/tocService.ts | 10 +++--- 9 files changed, 155 insertions(+), 20 deletions(-) diff --git a/app/api/entities/entities.js b/app/api/entities/entities.js index 753d02ddd4..b8e7c1c6c4 100644 --- a/app/api/entities/entities.js +++ b/app/api/entities/entities.js @@ -163,6 +163,10 @@ async function updateEntity(entity, _template) { if (typeof entity.template !== 'undefined') { d.template = entity.template; } + + if (typeof entity.template !== 'undefined') { + d.generatedToc = entity.generatedToc; + } return model.save(d); }) ); diff --git a/app/api/entities/specs/entities.spec.js b/app/api/entities/specs/entities.spec.js index 3769354886..94166f627c 100644 --- a/app/api/entities/specs/entities.spec.js +++ b/app/api/entities/specs/entities.spec.js @@ -340,7 +340,7 @@ describe('entities', () => { }); }); - describe('when published/template property changes', () => { + describe('when published/template/generatedToc property changes', () => { it('should replicate the change for all the languages', done => { const doc = { _id: batmanFinishesId, @@ -348,6 +348,7 @@ describe('entities', () => { metadata: {}, published: false, template: templateId, + generatedToc: true, }; entities @@ -364,8 +365,10 @@ describe('entities', () => { expect(docES.template).toBeDefined(); expect(docES.published).toBe(false); + expect(docES.generatedToc).toBe(true); expect(docES.template.equals(templateId)).toBe(true); expect(docEN.published).toBe(false); + expect(docEN.generatedToc).toBe(true); expect(docEN.template.equals(templateId)).toBe(true); done(); }) diff --git a/app/api/files/files.ts b/app/api/files/files.ts index c6ecd6ee7c..9e4108e558 100644 --- a/app/api/files/files.ts +++ b/app/api/files/files.ts @@ -1,6 +1,7 @@ import { deleteUploadedFiles } from 'api/files/filesystem'; import connections from 'api/relationships'; import { search } from 'api/search'; +import entities from 'api/entities'; import model from './filesModel'; import { validateFile } from '../../shared/types/fileSchema'; @@ -30,4 +31,28 @@ export const files = { return toDeleteFiles; }, + + async tocReviewed(_id: string) { + const savedFile = await files.save({ _id, generatedToc: false }); + const sameEntityFiles = await files.get({ entity: savedFile.entity }, { generatedToc: 1 }); + const [entity] = await entities.get({ + sharedId: savedFile.entity, + language: savedFile.language, + }); + + await entities.save( + { + _id: entity._id, + sharedId: entity.sharedId, + template: entity.template, + generatedToc: sameEntityFiles.reduce( + (generated, file) => generated || Boolean(file.generatedToc), + false + ), + }, + { user: {}, language: savedFile.language } + ); + + return savedFile; + }, }; diff --git a/app/api/files/routes.ts b/app/api/files/routes.ts index 5c4bb0fff2..4c5de41b3c 100644 --- a/app/api/files/routes.ts +++ b/app/api/files/routes.ts @@ -10,6 +10,7 @@ import activitylogMiddleware from 'api/activitylog/activitylogMiddleware'; import { CSVLoader } from 'api/csv'; import { files } from './files'; import { validation, createError, handleError } from '../utils'; +import entities from 'api/entities'; export default (app: Application) => { app.post( @@ -57,6 +58,28 @@ export default (app: Application) => { .catch(next); }); + app.post( + '/api/files/tocReviewed', + needsAuthorization(['admin', 'editor']), + validation.validateRequest({ + properties: { + body: { + required: ['fileId'], + properties: { + fileId: { type: 'string' }, + }, + }, + }, + }), + async (req, res, next) => { + try { + res.json(await files.tocReviewed(req.body.fileId)); + } catch (e) { + next(e); + } + } + ); + app.get( '/api/files/:filename', validation.validateRequest({ diff --git a/app/api/files/specs/fixtures.ts b/app/api/files/specs/fixtures.ts index 786593a43a..1bab9ad9da 100644 --- a/app/api/files/specs/fixtures.ts +++ b/app/api/files/specs/fixtures.ts @@ -5,22 +5,31 @@ const entityEnId = db.id(); const uploadId = db.id(); const uploadId2 = db.id(); const templateId = db.id(); +const importTemplate = db.id(); const fileName1 = 'f2082bf51b6ef839690485d7153e847a.pdf'; const fixtures: DBFixture = { files: [ { _id: uploadId, - entity: 'entity', + entity: 'sharedId1', + language: 'es', + generatedToc: true, originalname: 'upload1', filename: fileName1, type: 'custom', }, { _id: uploadId2, - entity: 'entity', + generatedToc: true, + language: 'es', + entity: 'sharedId1', filename: 'fileNotInDisk', }, + { + entity: 'sharedId1', + filename: 'fileWithoutTocFlag', + }, { _id: db.id(), filename: 'fileNotOnDisk' }, { _id: db.id(), originalname: 'upload2', type: 'custom' }, { _id: db.id(), originalname: 'upload3', type: 'document' }, @@ -36,11 +45,21 @@ const fixtures: DBFixture = { sharedId: 'sharedId1', language: 'es', title: 'Gadgets 01 ES', - toc: [{ _id: db.id(), label: 'existingToc' }], + generatedToc: true, + template: templateId, + }, + { + _id: entityEnId, + template: templateId, + sharedId: 'sharedId1', + language: 'en', + title: 'Gadgets 01 EN', }, - { _id: entityEnId, sharedId: 'sharedId1', language: 'en', title: 'Gadgets 01 EN' }, ], - templates: [{ _id: templateId, default: true, name: 'mydoc', properties: [] }], + templates: [ + { _id: templateId, default: true, name: 'mydoc', properties: [] }, + { _id: importTemplate, default: true, name: 'import', properties: [] }, + ], settings: [ { _id: db.id(), @@ -51,4 +70,13 @@ const fixtures: DBFixture = { ], }; -export { fixtures, entityId, entityEnId, fileName1, uploadId, uploadId2, templateId }; +export { + fixtures, + entityId, + entityEnId, + fileName1, + uploadId, + uploadId2, + templateId, + importTemplate, +}; diff --git a/app/api/files/specs/routes.spec.ts b/app/api/files/specs/routes.spec.ts index 88b63e1262..28af682930 100644 --- a/app/api/files/specs/routes.spec.ts +++ b/app/api/files/specs/routes.spec.ts @@ -12,6 +12,7 @@ import { FileType } from 'shared/types/fileType'; import { fixtures, uploadId, uploadId2 } from './fixtures'; import { files } from '../files'; import uploadRoutes from '../routes'; +import entities from 'api/entities'; jest.mock( '../../auth/authMiddleware.ts', @@ -47,7 +48,7 @@ describe('files routes', () => { }); it('should reindex all entities that are related to the saved file', async () => { - expect(search.indexEntities).toHaveBeenCalledWith({ sharedId: 'entity' }, '+fullText'); + expect(search.indexEntities).toHaveBeenCalledWith({ sharedId: 'sharedId1' }, '+fullText'); }); }); @@ -85,7 +86,7 @@ describe('files routes', () => { .query({ _id: uploadId2.toString() }); expect(search.indexEntities).toHaveBeenCalledWith( - { sharedId: { $in: ['entity'] } }, + { sharedId: { $in: ['sharedId1'] } }, '+fullText' ); }); @@ -107,5 +108,36 @@ describe('files routes', () => { expect(response.body.errors[0].message).toBe('should be string'); }); + + describe('api/files/tocReviewed', () => { + it('should set tocGenerated to false on the file', async () => { + const response: SuperTestResponse = await request(app) + .post('/api/files/tocReviewed') + .set('content-language', 'es') + .send({ fileId: uploadId.toString() }); + + const [file] = await files.get({ _id: uploadId }); + expect(file.generatedToc).toBe(false); + expect(response.body.entity).toBe('sharedId1'); + }); + + it('should set tocGenerated to false on the entity when all associated files are false', async () => { + await request(app) + .post('/api/files/tocReviewed') + .send({ fileId: uploadId.toString() }) + .expect(200); + + let [entity] = await entities.get({ sharedId: 'sharedId1' }); + expect(entity.generatedToc).toBe(true); + + await request(app) + .post('/api/files/tocReviewed') + .send({ fileId: uploadId2.toString() }) + .expect(200); + + [entity] = await entities.get({ sharedId: 'sharedId1' }); + expect(entity.generatedToc).toBe(false); + }); + }); }); }); diff --git a/app/api/files/specs/uploadRoutes.spec.ts b/app/api/files/specs/uploadRoutes.spec.ts index b8a7fb44a5..3cb15c61dd 100644 --- a/app/api/files/specs/uploadRoutes.spec.ts +++ b/app/api/files/specs/uploadRoutes.spec.ts @@ -16,7 +16,7 @@ import { setUpApp, socketEmit, iosocket } from 'api/utils/testingRoutes'; import { FileType } from 'shared/types/fileType'; import entities from 'api/entities'; -import { fixtures, templateId } from './fixtures'; +import { fixtures, templateId, importTemplate } from './fixtures'; import { files } from '../files'; import uploadRoutes from '../routes'; @@ -72,7 +72,10 @@ describe('upload routes', () => { expect(iosocket.emit).toHaveBeenCalledWith('conversionStart', 'sharedId1'); expect(iosocket.emit).toHaveBeenCalledWith('documentProcessed', 'sharedId1'); - const [upload] = await files.get({ entity: 'sharedId1' }, '+fullText'); + const [upload] = await files.get( + { originalname: 'f2082bf51b6ef839690485d7153e847a.pdf' }, + '+fullText' + ); expect(upload).toEqual( expect.objectContaining({ @@ -105,14 +108,14 @@ describe('upload routes', () => { it('should detect English documents and store the result', async () => { await uploadDocument('uploads/eng.pdf'); - const [upload] = await files.get({ entity: 'sharedId1' }); + const [upload] = await files.get({ originalname: 'eng.pdf' }); expect(upload.language).toBe('eng'); }, 10000); it('should detect Spanish documents and store the result', async () => { await uploadDocument('uploads/spn.pdf'); - const [upload] = await files.get({ entity: 'sharedId1' }); + const [upload] = await files.get({ originalname: 'spn.pdf' }); expect(upload.language).toBe('spa'); }); }); @@ -126,7 +129,7 @@ describe('upload routes', () => { .attach('file', path.join(__dirname, 'uploads/invalid_document.txt')) ); - const [upload] = await files.get({ entity: 'sharedId1' }, '+fullText'); + const [upload] = await files.get({ originalname: 'invalid_document.txt' }, '+fullText'); expect(upload.status).toBe('failed'); }); @@ -176,7 +179,7 @@ describe('upload routes', () => { await socketEmit('IMPORT_CSV_END', async () => request(app) .post('/api/import') - .field('template', templateId.toString()) + .field('template', importTemplate.toString()) .attach('file', `${__dirname}/uploads/importcsv.csv`) ); @@ -184,7 +187,7 @@ describe('upload routes', () => { expect(iosocket.emit).toHaveBeenCalledWith('IMPORT_CSV_PROGRESS', 1); expect(iosocket.emit).toHaveBeenCalledWith('IMPORT_CSV_PROGRESS', 2); - const imported = await entities.get({ template: templateId }); + const imported = await entities.get({ template: importTemplate }); expect(imported).toEqual([ expect.objectContaining({ title: 'imported entity one' }), expect.objectContaining({ title: 'imported entity two' }), diff --git a/app/api/toc_generation/specs/fixtures.ts b/app/api/toc_generation/specs/fixtures.ts index 49eea0d873..6c7639cb9e 100644 --- a/app/api/toc_generation/specs/fixtures.ts +++ b/app/api/toc_generation/specs/fixtures.ts @@ -1,14 +1,24 @@ import { testingDB, DBFixture } from 'api/utils/testing_db'; +const templateId = testingDB.id(); + const fixtures: DBFixture = { + templates: [ + { + _id: templateId, + properties: [], + }, + ], entities: [ { sharedId: 'shared1', title: 'pdf1entity', + template: templateId, }, { sharedId: 'shared3', title: 'pdf3entity', + template: templateId, }, ], files: [ @@ -16,17 +26,20 @@ const fixtures: DBFixture = { _id: testingDB.id(), entity: 'shared1', filename: 'pdf1.pdf', + language: 'es', originalname: 'originalPdf1.pdf', type: 'document', }, { _id: testingDB.id(), type: 'document', + language: 'es', }, { _id: testingDB.id(), type: 'custom', filename: 'background.jpg', + language: 'es', }, { _id: testingDB.id(), @@ -34,6 +47,7 @@ const fixtures: DBFixture = { filename: 'pdf2.pdf', originalname: 'originalPdf2.pdf', toc: [{}], + language: 'es', }, { _id: testingDB.id(), @@ -41,6 +55,7 @@ const fixtures: DBFixture = { type: 'document', filename: 'pdf3.pdf', originalname: 'originalPdf4.pdf', + language: 'es', }, ], }; diff --git a/app/api/toc_generation/tocService.ts b/app/api/toc_generation/tocService.ts index 80a5bdecd0..d64bbeeed0 100644 --- a/app/api/toc_generation/tocService.ts +++ b/app/api/toc_generation/tocService.ts @@ -24,12 +24,14 @@ const tocService = { ); await files.save({ ...nextFile, toc, generatedToc: true }); - const parentEntities = await entities.get({ sharedId: nextFile.entity }, { language: 1 }); - return entities.saveMultiple( - parentEntities.map(entity => ({ + const [entity] = await entities.get({ sharedId: nextFile.entity }, {}); + return entities.save( + { ...entity, generatedToc: true, - })) + }, + { user: {}, language: nextFile.language }, + false ); }, }; From 77b05321d71230fa8fde778217ab5afae2527dca Mon Sep 17 00:00:00 2001 From: Daneryl Date: Thu, 11 Feb 2021 11:50:40 -0500 Subject: [PATCH 04/31] tocService as a feature and cronjob --- app/api/toc_generation/specs/fixtures.ts | 1 + app/api/toc_generation/specs/tocService.spec.ts | 12 ++++++------ app/api/toc_generation/tocService.ts | 13 ++++++++----- app/server.js | 11 +++++++++-- app/shared/JSONRequest.js | 4 ++-- app/shared/types/settingsSchema.ts | 9 ++++++++- app/shared/types/settingsType.d.ts | 4 +++- 7 files changed, 37 insertions(+), 17 deletions(-) diff --git a/app/api/toc_generation/specs/fixtures.ts b/app/api/toc_generation/specs/fixtures.ts index 6c7639cb9e..2370acac38 100644 --- a/app/api/toc_generation/specs/fixtures.ts +++ b/app/api/toc_generation/specs/fixtures.ts @@ -54,6 +54,7 @@ const fixtures: DBFixture = { entity: 'shared3', type: 'document', filename: 'pdf3.pdf', + toc: [], originalname: 'originalPdf4.pdf', language: 'es', }, diff --git a/app/api/toc_generation/specs/tocService.spec.ts b/app/api/toc_generation/specs/tocService.spec.ts index f9b2285b9a..d77ec52000 100644 --- a/app/api/toc_generation/specs/tocService.spec.ts +++ b/app/api/toc_generation/specs/tocService.spec.ts @@ -6,10 +6,10 @@ import { fixtures } from './fixtures'; import { tocService } from '../tocService'; describe('tocService', () => { + const service = tocService('url'); beforeAll(async () => { - // const elasticIndex = 'toc.service.index'; - // await testingDB.clearAllAndLoad(fixtures, elasticIndex); - spyOn(request, 'uploadFile').and.callFake(async (_url, filename) => { + spyOn(request, 'uploadFile').and.callFake(async (url, filename) => { + expect(url).toBe('url'); if (filename === 'pdf1.pdf') { return Promise.resolve([{ label: 'section1 pdf1' }]); } @@ -29,13 +29,13 @@ describe('tocService', () => { const elasticIndex = 'toc.service.index'; await testingDB.clearAllAndLoad(fixtures, elasticIndex); await elasticTesting.resetIndex(); - await tocService.processNext(); - await tocService.processNext(); + await service.processNext(); + await service.processNext(); await elasticTesting.refresh(); }); it('should not fail when there is no more to process', async () => { - await expect(tocService.processNext()).resolves.not.toThrow(); + await expect(service.processNext()).resolves.not.toThrow(); }); it('should send the next pdfFile and save toc generated', async () => { diff --git a/app/api/toc_generation/tocService.ts b/app/api/toc_generation/tocService.ts index d64bbeeed0..69df4c7c59 100644 --- a/app/api/toc_generation/tocService.ts +++ b/app/api/toc_generation/tocService.ts @@ -2,23 +2,26 @@ import { files, uploadsPath } from 'api/files'; import request from 'shared/JSONRequest'; import entities from 'api/entities'; -const tocService = { +const tocService = (serviceUrl: string) => ({ async processNext() { const [nextFile] = await files.get( - { toc: { $exists: false }, type: 'document', filename: { $exists: true } }, + { + $or: [{ toc: { $size: 0 } }, { toc: { $exists: false } }], + type: 'document', + filename: { $exists: true }, + }, '', { sort: { _id: 1 }, limit: 1, } ); - if (!nextFile) { return null; } const toc = await request.uploadFile( - 'url_toc_service', + serviceUrl, nextFile.filename, uploadsPath(nextFile.filename) ); @@ -34,6 +37,6 @@ const tocService = { false ); }, -}; +}); export { tocService }; diff --git a/app/server.js b/app/server.js index bd7cf6c511..1cc383b1bb 100644 --- a/app/server.js +++ b/app/server.js @@ -30,6 +30,7 @@ import { tenants } from './api/tenants/tenantContext'; import { multitenantMiddleware } from './api/utils/multitenantMiddleware'; import { staticFilesMiddleware } from './api/utils/staticFilesMiddleware'; import { customUploadsPath, uploadsPath } from './api/files/filesystem'; +import { tocService } from './api/toc_generation/tocService'; mongoose.Promise = Promise; @@ -40,7 +41,7 @@ const http = Server(app); const uncaughtError = error => { handleError(error, { uncaught: true }); - process.exit(1); + throw error; }; process.on('unhandledRejection', uncaughtError); @@ -116,7 +117,7 @@ DB.connect(config.DBHOST, dbAuth).then(async () => { if (!config.multiTenant && !config.clusterMode) { syncWorker.start(); - const { evidencesVault } = await settings.get(); + const { evidencesVault, features } = await settings.get(); if (evidencesVault && evidencesVault.token && evidencesVault.template) { console.info('==> 📥 evidences vault config detected, started sync ....'); repeater.start( @@ -125,6 +126,12 @@ DB.connect(config.DBHOST, dbAuth).then(async () => { ); } + if (features && features.tocGeneration && features.tocGeneration.url) { + console.info('==> 🗂️ automatically generating TOCs using external service'); + const service = tocService(features.tocGeneration.url); + repeater.start(() => service.processNext(), 10000); + } + repeater.start( () => TaskProvider.runAndWait('TopicClassificationSync', 'TopicClassificationSync', { diff --git a/app/shared/JSONRequest.js b/app/shared/JSONRequest.js index bf975c2f7f..1473fcafbd 100644 --- a/app/shared/JSONRequest.js +++ b/app/shared/JSONRequest.js @@ -126,8 +126,8 @@ export default { .set('X-Requested-With', 'XMLHttpRequest') .set('Cookie', cookie || '') .attach('file', file, filename) - .then(() => { - resolve(); + .then(response => { + resolve(response.body); }) .catch(err => { reject(err); diff --git a/app/shared/types/settingsSchema.ts b/app/shared/types/settingsSchema.ts index afbac4a7d4..c8e0bd3f9b 100644 --- a/app/shared/types/settingsSchema.ts +++ b/app/shared/types/settingsSchema.ts @@ -133,7 +133,14 @@ export const settingsSchema = { type: 'object', properties: { _id: { type: 'string' }, - semanticSearch: { type: 'boolean' }, + tocGeneration: { + type: 'object', + required: ['active', 'url'], + additionalProperties: false, + properties: { + url: { type: 'string' }, + }, + }, topicClassification: { type: 'boolean' }, favorites: { type: 'boolean' }, }, diff --git a/app/shared/types/settingsType.d.ts b/app/shared/types/settingsType.d.ts index 842b7507d3..a882056e3d 100644 --- a/app/shared/types/settingsType.d.ts +++ b/app/shared/types/settingsType.d.ts @@ -84,7 +84,9 @@ export interface Settings { links?: SettingsLinkSchema[]; features?: { _id?: string; - semanticSearch?: boolean; + tocGeneration?: { + url: string; + }; topicClassification?: boolean; favorites?: boolean; [k: string]: unknown | undefined; From 2883177649839cb84ea5161b4d10db05f8254026 Mon Sep 17 00:00:00 2001 From: Daneryl Date: Wed, 17 Feb 2021 08:39:22 -0500 Subject: [PATCH 05/31] customFilters for search endpoint --- app/api/search/documentQueryBuilder.js | 13 ++++++++++++- app/api/search/metadataAggregations.js | 3 ++- app/api/search/search.js | 2 +- app/api/search/searchSchema.ts | 11 +++++++++++ app/api/search/specs/fixtures_elastic.js | 2 ++ app/api/search/specs/search.spec.js | 17 ++++++++++++++++- 6 files changed, 44 insertions(+), 4 deletions(-) diff --git a/app/api/search/documentQueryBuilder.js b/app/api/search/documentQueryBuilder.js index 65679ce7e4..ee0631acc3 100644 --- a/app/api/search/documentQueryBuilder.js +++ b/app/api/search/documentQueryBuilder.js @@ -1,7 +1,7 @@ /* eslint-disable camelcase, max-lines */ import { preloadOptionsSearch } from 'shared/config'; -import filterToMatch, { multiselectFilter } from './metadataMatchers'; +import filterToMatch, { textFilter, multiselectFilter } from './metadataMatchers'; import { propertyToAggregation, generatedTocAggregations } from './metadataAggregations'; export default function() { @@ -231,6 +231,17 @@ export default function() { } }, + customFilters(filters = {}) { + Object.keys(filters).forEach(key => { + if (filters[key].values.length) { + addFilter({ + terms: { [key]: filters[key].values }, + }); + } + }); + return this; + }, + filterMetadata(filters = []) { filters.forEach(filter => { const match = filterToMatch(filter, filter.suggested ? 'suggestedMetadata' : 'metadata'); diff --git a/app/api/search/metadataAggregations.js b/app/api/search/metadataAggregations.js index aac071e4a0..03766ee3c8 100644 --- a/app/api/search/metadataAggregations.js +++ b/app/api/search/metadataAggregations.js @@ -137,6 +137,7 @@ const extractFilters = (baseQuery, path) => { (!match.terms || (match.terms && !match.terms[path])) && (!match.bool || !match.bool.should || + !match.bool.should[1] || !match.bool.should[1].terms || !match.bool.should[1].terms[path]) ); @@ -173,6 +174,6 @@ export const generatedTocAggregations = baseQuery => { const filters = extractFilters(baseQuery, path); const { should } = baseQuery.query.bool; const agg = aggregation(path, should, filters); - agg.terms.missing = 'false'; + // agg.terms.missing = 'false'; return agg; }; diff --git a/app/api/search/search.js b/app/api/search/search.js index 8475759233..b3d80184d3 100644 --- a/app/api/search/search.js +++ b/app/api/search/search.js @@ -607,6 +607,7 @@ const buildQuery = async (query, language, user, resources) => { const filters = processFilters(query.filters, [...allUniqueProps, ...properties]); // this is where the query filters are built queryBuilder.filterMetadata(filters); + queryBuilder.customFilters(query.customFilters); // this is where the query aggregations are built queryBuilder.aggregations(aggregations, dictionaries); @@ -626,7 +627,6 @@ const search = { queryBuilder.generatedTOCAggregations(); } - // queryBuilder.query() is the actual call return elastic .search({ body: queryBuilder.query() }) .then(response => processResponse(response, templates, dictionaries, language, query.filters)) diff --git a/app/api/search/searchSchema.ts b/app/api/search/searchSchema.ts index 9b67c6a1b1..abae216f28 100644 --- a/app/api/search/searchSchema.ts +++ b/app/api/search/searchSchema.ts @@ -4,6 +4,17 @@ export const searchSchema = { properties: { aggregateGeneratedToc: { type: 'boolean' }, filters: { type: 'object' }, + customFilters: { + type: 'object', + properties: { + generatedToc: { + type: 'object', + properties: { + values: { type: 'array', items: [{ type: 'boolean' }] }, + }, + }, + }, + }, types: { type: 'array', items: [{ type: 'string' }] }, _types: { type: 'array', items: [{ type: 'string' }] }, fields: { type: 'array', items: [{ type: 'string' }] }, diff --git a/app/api/search/specs/fixtures_elastic.js b/app/api/search/specs/fixtures_elastic.js index 86cb00f2c7..3981a52437 100644 --- a/app/api/search/specs/fixtures_elastic.js +++ b/app/api/search/specs/fixtures_elastic.js @@ -72,6 +72,7 @@ export const fixtures = { language: 'en', title: 'Batman finishes en', published: true, + generatedToc: true, user: userId, metadata: { relationship: [ @@ -153,6 +154,7 @@ export const fixtures = { language: 'es', title: 'template1 title es', published: true, + generatedToc: false, user: userId, }, { diff --git a/app/api/search/specs/search.spec.js b/app/api/search/specs/search.spec.js index e7152ea72a..8b94c670b8 100644 --- a/app/api/search/specs/search.spec.js +++ b/app/api/search/specs/search.spec.js @@ -168,7 +168,7 @@ describe('search', () => { const aggregations = response.aggregations.all.generatedToc.buckets; - expect(aggregations.find(a => a.key === 'false').filtered.doc_count).toBe(3); + expect(aggregations.find(a => a.key === 'false').filtered.doc_count).toBe(2); expect(aggregations.find(a => a.key === 'true').filtered.doc_count).toBe(1); }); @@ -778,6 +778,21 @@ describe('search', () => { }); }); + describe('customFilters', () => { + it('should filter by the values passed', async () => { + const query = { + customFilters: { + generatedToc: { + values: ['true'], + }, + }, + }; + + const { rows } = await search.search(query, 'en'); + expect(rows).toEqual([expect.objectContaining({ title: 'Batman finishes en' })]); + }); + }); + describe('autocompleteAggregations()', () => { it('should return a list of options matching by label and options related to the matching one', async () => { const query = { From bcdb188b2b2e73def838f354a55767f0b8760ce3 Mon Sep 17 00:00:00 2001 From: Daneryl Date: Thu, 11 Feb 2021 17:20:59 -0500 Subject: [PATCH 06/31] UI toc generation components, WIP --- app/api/files/files.ts | 5 +- app/api/files/routes.ts | 2 +- app/api/files/specs/fixtures.ts | 2 - app/api/search/deprecatedRoutes.js | 4 +- app/api/search/elasticTypes.ts | 3 +- app/api/search/specs/searchSchema.spec.ts | 9 +- app/react/App/scss/modules/_toc.scss | 7 +- app/react/Attachments/components/File.tsx | 11 ++- .../components/specs/File.spec.tsx | 3 +- .../Documents/components/DocumentSidePanel.js | 15 ++- app/react/Documents/components/ShowToc.js | 7 +- app/react/Library/actions/libraryActions.js | 3 + .../actions/specs/filterActions.spec.js | 8 +- .../actions/specs/libraryActions.spec.js | 43 +++++--- app/react/Library/components/FiltersForm.js | 7 +- .../Library/components/ViewMetadataPanel.js | 12 ++- app/react/Library/helpers/libraryFilters.js | 1 + app/react/Library/helpers/requestState.js | 10 +- .../helpers/specs/libraryFilters.spec.js | 24 ++--- .../helpers/specs/resquestState.spec.js | 1 + .../tocGeneration/FilterTocGeneration.tsx | 49 +++++++++ .../tocGeneration/ReviewTocButton.tsx | 34 +++++++ .../tocGeneration/TocGeneratedLabel.tsx | 14 +++ .../ToggledFeatures/tocGeneration/actions.ts | 36 +++++++ .../ToggledFeatures/tocGeneration/index.ts | 4 + .../specs/FilterTocGeneration.spec.tsx | 68 +++++++++++++ .../specs/ReviewTocButton.spec.tsx | 37 +++++++ .../tocGeneration/specs/actions.spec.ts | 99 +++++++++++++++++++ .../tocGeneration/specs/utils.spec.ts | 19 ++++ .../ToggledFeatures/tocGeneration/utils.ts | 11 +++ app/react/Uploads/specs/UploadsRoute.spec.js | 1 + app/react/istore.d.ts | 11 ++- app/server.js | 2 +- app/shared/types/Aggregations.d.ts | 13 +++ app/shared/types/connectionType.d.ts | 3 + app/shared/types/searchParams.d.ts | 33 +++++++ .../types/searchParams.ts} | 7 +- emitSchemaTypes.js | 32 +++--- 38 files changed, 574 insertions(+), 76 deletions(-) create mode 100644 app/react/ToggledFeatures/tocGeneration/FilterTocGeneration.tsx create mode 100644 app/react/ToggledFeatures/tocGeneration/ReviewTocButton.tsx create mode 100644 app/react/ToggledFeatures/tocGeneration/TocGeneratedLabel.tsx create mode 100644 app/react/ToggledFeatures/tocGeneration/actions.ts create mode 100644 app/react/ToggledFeatures/tocGeneration/index.ts create mode 100644 app/react/ToggledFeatures/tocGeneration/specs/FilterTocGeneration.spec.tsx create mode 100644 app/react/ToggledFeatures/tocGeneration/specs/ReviewTocButton.spec.tsx create mode 100644 app/react/ToggledFeatures/tocGeneration/specs/actions.spec.ts create mode 100644 app/react/ToggledFeatures/tocGeneration/specs/utils.spec.ts create mode 100644 app/react/ToggledFeatures/tocGeneration/utils.ts create mode 100644 app/shared/types/Aggregations.d.ts create mode 100644 app/shared/types/searchParams.d.ts rename app/{api/search/searchSchema.ts => shared/types/searchParams.ts} (85%) diff --git a/app/api/files/files.ts b/app/api/files/files.ts index 9e4108e558..2667669584 100644 --- a/app/api/files/files.ts +++ b/app/api/files/files.ts @@ -32,12 +32,11 @@ export const files = { return toDeleteFiles; }, - async tocReviewed(_id: string) { + async tocReviewed(_id: string, language: string) { const savedFile = await files.save({ _id, generatedToc: false }); const sameEntityFiles = await files.get({ entity: savedFile.entity }, { generatedToc: 1 }); const [entity] = await entities.get({ sharedId: savedFile.entity, - language: savedFile.language, }); await entities.save( @@ -50,7 +49,7 @@ export const files = { false ), }, - { user: {}, language: savedFile.language } + { user: {}, language } ); return savedFile; diff --git a/app/api/files/routes.ts b/app/api/files/routes.ts index 4c5de41b3c..9e3170aea2 100644 --- a/app/api/files/routes.ts +++ b/app/api/files/routes.ts @@ -73,7 +73,7 @@ export default (app: Application) => { }), async (req, res, next) => { try { - res.json(await files.tocReviewed(req.body.fileId)); + res.json(await files.tocReviewed(req.body.fileId, req.language)); } catch (e) { next(e); } diff --git a/app/api/files/specs/fixtures.ts b/app/api/files/specs/fixtures.ts index 1bab9ad9da..94b88d6f77 100644 --- a/app/api/files/specs/fixtures.ts +++ b/app/api/files/specs/fixtures.ts @@ -13,7 +13,6 @@ const fixtures: DBFixture = { { _id: uploadId, entity: 'sharedId1', - language: 'es', generatedToc: true, originalname: 'upload1', filename: fileName1, @@ -22,7 +21,6 @@ const fixtures: DBFixture = { { _id: uploadId2, generatedToc: true, - language: 'es', entity: 'sharedId1', filename: 'fileNotInDisk', }, diff --git a/app/api/search/deprecatedRoutes.js b/app/api/search/deprecatedRoutes.js index 9526181f88..e5d369dbeb 100644 --- a/app/api/search/deprecatedRoutes.js +++ b/app/api/search/deprecatedRoutes.js @@ -1,6 +1,6 @@ import Joi from 'joi'; import entities from 'api/entities'; -import { searchSchema } from 'api/search/searchSchema'; +import { searchParamsSchema } from 'shared/types/searchParams'; import { search } from './search'; import { validation, parseQuery } from '../utils'; @@ -25,7 +25,7 @@ export default app => { app.get( '/api/search', parseQuery, - validation.validateRequest(searchSchema), + validation.validateRequest(searchParamsSchema), (req, res, next) => { const action = req.query.geolocation ? 'searchGeolocations' : 'search'; diff --git a/app/api/search/elasticTypes.ts b/app/api/search/elasticTypes.ts index efee29aab9..b5351ca1fa 100644 --- a/app/api/search/elasticTypes.ts +++ b/app/api/search/elasticTypes.ts @@ -1,5 +1,6 @@ import { RequestParams } from '@elastic/elasticsearch'; import { RequestBody } from '@elastic/elasticsearch/lib/Transport'; +import { Aggregations } from 'shared/types/Aggregations.d.ts'; interface ShardsResponse { total: number; @@ -42,7 +43,7 @@ export interface SearchResponse { sort?: string[]; }>; }; - aggregations?: any; + aggregations?: Aggregations; } export type IndicesDelete = Omit; diff --git a/app/api/search/specs/searchSchema.spec.ts b/app/api/search/specs/searchSchema.spec.ts index 33722cd602..8a23458c1b 100644 --- a/app/api/search/specs/searchSchema.spec.ts +++ b/app/api/search/specs/searchSchema.spec.ts @@ -1,6 +1,6 @@ import { ValidationError } from 'ajv'; import { validation } from 'api/utils'; -import { searchSchema } from '../searchSchema'; +import { searchParamsSchema } from 'shared/types/searchParams'; describe('search schema', () => { const validQuery = { @@ -27,11 +27,12 @@ describe('search schema', () => { it('should not have validation errors for valid search', async () => { const validSearch = { validQuery }; - await validation.validateRequest(searchSchema)(validSearch, null, expectValidSchema); + await validation.validateRequest(searchParamsSchema)(validSearch, null, expectValidSchema); }); + it('should support a number as a search term', async () => { const validSearch = { query: { ...validQuery, searchTerm: 3 } }; - await validation.validateRequest(searchSchema)(validSearch, null, expectValidSchema); + await validation.validateRequest(searchParamsSchema)(validSearch, null, expectValidSchema); }); }); @@ -42,7 +43,7 @@ describe('search schema', () => { async function testInvalidProperty(invalidProperty: any) { const invalidSearch = { query: { ...validQuery, ...invalidProperty } }; - await validation.validateRequest(searchSchema)(invalidSearch, null, expectInvalidSchema); + await validation.validateRequest(searchParamsSchema)(invalidSearch, null, expectInvalidSchema); } it('should be invalid if allAgregations is not a boolean value', async () => { diff --git a/app/react/App/scss/modules/_toc.scss b/app/react/App/scss/modules/_toc.scss index 3f195ab5ed..9f06fca32e 100644 --- a/app/react/App/scss/modules/_toc.scss +++ b/app/react/App/scss/modules/_toc.scss @@ -1,5 +1,10 @@ .toc { - padding: 15px; + padding: 0px 15px 15px 15px; +} + +div.tocHeader { + padding-left: 15px; + border-bottom: 1px solid #f4f4f4; } .toc-view{ diff --git a/app/react/Attachments/components/File.tsx b/app/react/Attachments/components/File.tsx index 21fd3c5cb1..e18dd4bc0a 100644 --- a/app/react/Attachments/components/File.tsx +++ b/app/react/Attachments/components/File.tsx @@ -10,6 +10,7 @@ import { APIURL } from 'app/config.js'; import { LocalForm, Control } from 'react-redux-form'; import { updateFile, deleteFile } from 'app/Attachments/actions/actions'; import { wrapDispatch } from 'app/Multireducer'; +import { TocGeneratedLabel } from 'app/ToggledFeatures/tocGeneration'; import { NeedAuthorization } from 'app/Auth'; import { EntitySchema } from 'shared/types/entityType'; import { ViewDocumentLink } from './ViewDocumentLink'; @@ -102,7 +103,10 @@ export class File extends Component {
{language ? transformLanguage(language) || '' : ''} -
{' '} +
+ + ML TOC + { const mapDispatchToProps = (dispatch: Dispatch<{}>, props: FileProps) => bindActionCreators({ updateFile, deleteFile }, wrapDispatch(dispatch, props.storeKey)); -export const ConnectedFile = connect(null, mapDispatchToProps)(File); +export const ConnectedFile = connect( + null, + mapDispatchToProps +)(File); diff --git a/app/react/Attachments/components/specs/File.spec.tsx b/app/react/Attachments/components/specs/File.spec.tsx index 997ef2a635..95ca643fa4 100644 --- a/app/react/Attachments/components/specs/File.spec.tsx +++ b/app/react/Attachments/components/specs/File.spec.tsx @@ -2,8 +2,8 @@ import React from 'react'; import { shallow, ShallowWrapper } from 'enzyme'; import { LocalForm } from 'react-redux-form'; import { FileType } from 'shared/types/fileType'; -import { File, FileProps } from '../File'; import { Translate } from 'app/I18N'; +import { File, FileProps } from '../File'; describe('file', () => { let component: ShallowWrapper; @@ -32,6 +32,7 @@ describe('file', () => { }); const render = () => { + //eslint-disable-next-line react/jsx-props-no-spreading component = shallow(, { context }); }; diff --git a/app/react/Documents/components/DocumentSidePanel.js b/app/react/Documents/components/DocumentSidePanel.js index d476d6d047..9d9cf7d75e 100644 --- a/app/react/Documents/components/DocumentSidePanel.js +++ b/app/react/Documents/components/DocumentSidePanel.js @@ -16,6 +16,8 @@ import ShowIf from 'app/App/ShowIf'; import SidePanel from 'app/Layout/SidePanel'; import DocumentSemanticSearchResults from 'app/SemanticSearch/components/DocumentResults'; import { CopyFromEntity } from 'app/Metadata/components/CopyFromEntity'; +import { FeatureToggle } from 'app/components/Elements/FeatureToggle'; +import { TocGeneratedLabel, ReviewTocButton } from 'app/ToggledFeatures/tocGeneration'; import { Icon } from 'UI'; import * as viewerModule from 'app/Viewer'; @@ -321,11 +323,14 @@ export class DocumentSidePanel extends Component {
+ + Mark as Reviewed +
@@ -340,6 +345,14 @@ export class DocumentSidePanel extends Component { /> +
+

+ Table of contents +

+ + auto-created ⓘ + +
{ let filtersState; beforeEach(() => { - libraryFilters = [ - { name: 'author', filter: true }, - { name: 'country', filter: true }, - ]; + libraryFilters = [{ name: 'author', filter: true }, { name: 'country', filter: true }]; search = { searchTerm: '', filters: { author: 'RR Martin', country: '' } }; filtersState = { documentTypes, @@ -36,6 +33,9 @@ describe('filterActions', () => { }; store = { + settings: { + collection: Immutable.Map({}), + }, library: { filters: Immutable.fromJS(filtersState), search, diff --git a/app/react/Library/actions/specs/libraryActions.spec.js b/app/react/Library/actions/specs/libraryActions.spec.js index 71d771da5a..d1d94de2cf 100644 --- a/app/react/Library/actions/specs/libraryActions.spec.js +++ b/app/react/Library/actions/specs/libraryActions.spec.js @@ -48,9 +48,12 @@ describe('libraryActions', () => { beforeEach(() => { dispatch = jasmine.createSpy('dispatch'); - getState = jasmine - .createSpy('getState') - .and.returnValue({ library: { filters: Immutable.fromJS(filters), search: {} } }); + getState = jasmine.createSpy('getState').and.returnValue({ + settings: { + collection: Immutable.Map({}), + }, + library: { filters: Immutable.fromJS(filters), search: {} }, + }); }); it('should dispatch a SET_LIBRARY_TEMPLATES action ', () => { @@ -178,15 +181,24 @@ describe('libraryActions', () => { { name: 'relationshipfilter', type: 'relationshipfilter', - filters: [ - { name: 'status', type: 'select' }, - { name: 'empty', type: 'date' }, - ], + filters: [{ name: 'status', type: 'select' }, { name: 'empty', type: 'date' }], }, ], documentTypes: ['decision'], }; - store = { library: { filters: Immutable.fromJS(state), search: { searchTerm: 'batman' } } }; + store = { + settings: { + collection: Immutable.Map({}), + }, + library: { + filters: Immutable.fromJS(state), + search: { + searchTerm: 'batman', + customFilters: { property: { values: ['value'] } }, + filters: {}, + }, + }, + }; spyOn(browserHistory, 'getCurrentLocation').and.returnValue({ pathname: '/library', query: { view: 'chart' }, @@ -261,6 +273,16 @@ describe('libraryActions', () => { ); }); + it('should use customFilters from the current search on the store', () => { + const limit = 60; + spyOn(browserHistory, 'push'); + actions.searchDocuments({}, storeKey, limit)(dispatch, getState); + + expect(browserHistory.push).toHaveBeenCalledWith( + "/library/?view=chart&q=(customFilters:(property:(values:!(value))),filters:(),from:0,limit:60,searchTerm:'batman',sort:_score,types:!(decision))" //eslint-disable-line + ); + }); + it('should set the storeKey selectedSorting if user has selected a custom sorting', () => { const expectedDispatch = { type: 'library.selectedSorting/SET', @@ -365,10 +387,7 @@ describe('libraryActions', () => { }, { type: types.UPDATE_DOCUMENTS, - docs: [ - { sharedId: '1', metadataResponse }, - { sharedId: '2', metadataResponse }, - ], + docs: [{ sharedId: '1', metadataResponse }, { sharedId: '2', metadataResponse }], }, ]; const store = mockStore({}); diff --git a/app/react/Library/components/FiltersForm.js b/app/react/Library/components/FiltersForm.js index 6f773961c4..a06985851a 100644 --- a/app/react/Library/components/FiltersForm.js +++ b/app/react/Library/components/FiltersForm.js @@ -11,6 +11,7 @@ import { t } from 'app/I18N'; import { wrapDispatch } from 'app/Multireducer'; import debounce from 'app/utils/debounce'; import libraryHelper from 'app/Library/helpers/libraryFilters'; +import { FilterTocGeneration } from 'app/ToggledFeatures/tocGeneration'; import Filters from './FiltersFromProperties'; @@ -97,6 +98,7 @@ export class FiltersForm extends Component { translationContext={translationContext} storeKey={this.props.storeKey} /> + ); @@ -126,4 +128,7 @@ function mapDispatchToProps(dispatch, props) { return bindActionCreators({ searchDocuments }, wrapDispatch(dispatch, props.storeKey)); } -export default connect(mapStateToProps, mapDispatchToProps)(FiltersForm); +export default connect( + mapStateToProps, + mapDispatchToProps +)(FiltersForm); diff --git a/app/react/Library/components/ViewMetadataPanel.js b/app/react/Library/components/ViewMetadataPanel.js index 6f9ec1138f..f3487a6278 100644 --- a/app/react/Library/components/ViewMetadataPanel.js +++ b/app/react/Library/components/ViewMetadataPanel.js @@ -9,6 +9,7 @@ import { actions } from 'app/Metadata'; import { deleteDocument, searchSnippets } from 'app/Library/actions/libraryActions'; import { deleteEntity } from 'app/Entities/actions/actions'; import { wrapDispatch } from 'app/Multireducer'; +import { entityDefaultDocument } from 'shared/entityDefaultDocument'; import modals from 'app/Modals'; import { @@ -22,9 +23,18 @@ const getTemplates = state => state.templates; const mapStateToProps = (state, props) => { const library = state[props.storeKey]; + const doc = library.ui.get('selectedDocuments').first() || Immutable.fromJS({ documents: [] }); + const defaultLanguage = state.settings.collection.get('languages').find(l => l.get('defautl')); + const file = entityDefaultDocument( + doc.get('documents').toJS(), + doc.get('language'), + defaultLanguage + ); + return { open: library.ui.get('selectedDocuments').size === 1, - doc: library.ui.get('selectedDocuments').first() || Immutable.fromJS({}), + doc, + file, references: library.sidepanel.references, tab: library.sidepanel.tab, docBeingEdited: !!Object.keys(library.sidepanel.metadata).length, diff --git a/app/react/Library/helpers/libraryFilters.js b/app/react/Library/helpers/libraryFilters.js index 79695577b3..9f39daa6fb 100644 --- a/app/react/Library/helpers/libraryFilters.js +++ b/app/react/Library/helpers/libraryFilters.js @@ -56,6 +56,7 @@ function URLQueryToState(query, templates, _thesauris, _relationTypes, forcedPro search: { searchTerm, filters, + customFilters: query.customFilters, sort, order, userSelectedSorting, diff --git a/app/react/Library/helpers/requestState.js b/app/react/Library/helpers/requestState.js index dc66ccd084..144f01f5b9 100644 --- a/app/react/Library/helpers/requestState.js +++ b/app/react/Library/helpers/requestState.js @@ -5,6 +5,7 @@ import prioritySortingCriteria from 'app/utils/prioritySortingCriteria'; import rison from 'rison-node'; import { getThesaurusPropertyNames } from 'shared/commonTopicClassification'; import { setTableViewColumns } from 'app/Library/actions/libraryActions'; +import { tocGenerationUtils } from 'app/ToggledFeatures/tocGeneration'; import { wrapDispatch } from 'app/Multireducer'; import { getTableColumns } from './tableColumns'; import setReduxState from './setReduxState.js'; @@ -36,12 +37,17 @@ export function processQuery(params, globalResources, key) { } const { userSelectedSorting, ...sanitizedQuery } = query; - return sanitizedQuery; + return tocGenerationUtils.aggregations( + sanitizedQuery, + globalResources.settings.collection.toJS() + ); } export default function requestState(request, globalResources, calculateTableColumns = false) { const docsQuery = processQuery(request.data, globalResources, 'library'); - const documentsRequest = request.set(docsQuery); + const documentsRequest = request.set( + tocGenerationUtils.aggregations(docsQuery, globalResources.settings.collection.toJS()) + ); const markersRequest = request.set({ ...docsQuery, geolocation: true }); return Promise.all([api.search(documentsRequest), api.search(markersRequest)]).then( diff --git a/app/react/Library/helpers/specs/libraryFilters.spec.js b/app/react/Library/helpers/specs/libraryFilters.spec.js index fa91a8bc15..72225d6450 100644 --- a/app/react/Library/helpers/specs/libraryFilters.spec.js +++ b/app/react/Library/helpers/specs/libraryFilters.spec.js @@ -33,26 +33,17 @@ describe('library helper', () => { const thesauris = [ { _id: 'abc1', - values: [ - { id: 1, value: 'value1' }, - { id: 2, value: 'value2' }, - ], + values: [{ id: 1, value: 'value1' }, { id: 2, value: 'value2' }], }, { _id: 'thesauri2', type: 'template', - values: [ - { id: 3, value: 'value3' }, - { id: 4, value: 'value4' }, - ], + values: [{ id: 3, value: 'value3' }, { id: 4, value: 'value4' }], }, { _id: 'thesauri3', type: 'template', - values: [ - { id: 5, value: 'value5' }, - { id: 6, value: 'value6' }, - ], + values: [{ id: 5, value: 'value5' }, { id: 6, value: 'value6' }], }, ]; @@ -83,10 +74,14 @@ describe('library helper', () => { sort: 'sort', types: ['3'], filters: { country: 'countryValue', rich: 'search' }, + customFilters: { + property: { values: ['value'] }, + }, }; const state = libraryHelper.URLQueryToState(query, templates); expect(state.properties.length).toBe(1); + expect(state.search.customFilters).toEqual(query.customFilters); expect(state.search.filters.country).toBe('countryValue'); expect(state.search.filters.rich).toBe('search'); expect(state.search.searchTerm).toBe('searchTerm'); @@ -146,10 +141,7 @@ describe('library helper', () => { filter: true, type: 'select', content: 'abc1', - options: [ - { id: 1, value: 'value1' }, - { id: 2, value: 'value2' }, - ], + options: [{ id: 1, value: 'value1' }, { id: 2, value: 'value2' }], }, { name: 'date', filter: true, type: 'text' }, ]; diff --git a/app/react/Library/helpers/specs/resquestState.spec.js b/app/react/Library/helpers/specs/resquestState.spec.js index 7df7599f3c..6377d1f40c 100644 --- a/app/react/Library/helpers/specs/resquestState.spec.js +++ b/app/react/Library/helpers/specs/resquestState.spec.js @@ -30,6 +30,7 @@ describe('static requestState()', () => { }; const globalResources = { templates: Immutable.fromJS(templates), + settings: { collection: Immutable.fromJS({ features: {} }) }, thesauris: Immutable.fromJS(thesauris), relationTypes: Immutable.fromJS(relationTypes), }; diff --git a/app/react/ToggledFeatures/tocGeneration/FilterTocGeneration.tsx b/app/react/ToggledFeatures/tocGeneration/FilterTocGeneration.tsx new file mode 100644 index 0000000000..2afe82a2ff --- /dev/null +++ b/app/react/ToggledFeatures/tocGeneration/FilterTocGeneration.tsx @@ -0,0 +1,49 @@ +import React from 'react'; +import { Aggregations } from 'shared/types/Aggregations.d.ts'; +import { FeatureToggle } from 'app/components/Elements/FeatureToggle'; +import SelectFilter from 'app/Library/components/SelectFilter'; +import FormGroup from 'app/DocumentForm/components/FormGroup'; +import { t } from 'app/I18N'; +import { NeedAuthorization } from 'app/Auth'; + +export interface FilterTocGenerationProps { + onChange: () => void; + aggregations: Aggregations; +} + +const filteredAggregation = (aggregations: Aggregations, key: string) => { + const bucket = (aggregations?.all?.generatedToc?.buckets || []).find(a => a.key === key) || { + filtered: { doc_count: 0 }, + }; + return bucket.filtered.doc_count; +}; + +const options = (aggregations: Aggregations = { all: {} }) => [ + { + label: 'Automatically generated', + value: true, + results: filteredAggregation(aggregations, 'true'), + }, + { + label: 'Reviewed', + value: false, + results: filteredAggregation(aggregations, 'false'), + }, +]; + +export const FilterTocGeneration = ({ onChange, aggregations }: FilterTocGenerationProps) => ( + + + + + + + +); diff --git a/app/react/ToggledFeatures/tocGeneration/ReviewTocButton.tsx b/app/react/ToggledFeatures/tocGeneration/ReviewTocButton.tsx new file mode 100644 index 0000000000..8c5878b62c --- /dev/null +++ b/app/react/ToggledFeatures/tocGeneration/ReviewTocButton.tsx @@ -0,0 +1,34 @@ +import React from 'react'; +import { Icon } from 'UI'; +import { FeatureToggle } from 'app/components/Elements/FeatureToggle'; +import { connect, ConnectedProps } from 'react-redux'; +import { bindActionCreators, Dispatch } from 'redux'; +import { ClientFile } from 'app/istore'; +import { tocGenerationActions } from './actions'; + +export interface ReviewTocButtonProps { + file: ClientFile; + children: JSX.Element | string; +} + +const mapDispatchToProps = (dispatch: Dispatch<{}>) => + bindActionCreators({ onClick: tocGenerationActions.reviewToc }, dispatch); + +const connector = connect(null, mapDispatchToProps); + +type MappedProps = ConnectedProps; +type ComponentProps = ReviewTocButtonProps & MappedProps; + +const ReviewTocButton = ({ file, onClick, children }: ComponentProps) => ( + + {file.generatedToc && ( + + )} + +); + +const container = connector(ReviewTocButton); +export { container as ReviewTocButton }; diff --git a/app/react/ToggledFeatures/tocGeneration/TocGeneratedLabel.tsx b/app/react/ToggledFeatures/tocGeneration/TocGeneratedLabel.tsx new file mode 100644 index 0000000000..5ef7e63b17 --- /dev/null +++ b/app/react/ToggledFeatures/tocGeneration/TocGeneratedLabel.tsx @@ -0,0 +1,14 @@ +import React from 'react'; +import { FeatureToggle } from 'app/components/Elements/FeatureToggle'; +import { FileType } from 'shared/types/fileType'; + +export interface TocGeneratedLabelProps { + file: FileType; + children: JSX.Element | string; +} + +export const TocGeneratedLabel = ({ file, children }: TocGeneratedLabelProps) => ( + + {file.generatedToc && {children}} + +); diff --git a/app/react/ToggledFeatures/tocGeneration/actions.ts b/app/react/ToggledFeatures/tocGeneration/actions.ts new file mode 100644 index 0000000000..8ab8f5b5fc --- /dev/null +++ b/app/react/ToggledFeatures/tocGeneration/actions.ts @@ -0,0 +1,36 @@ +import { actions } from 'app/BasicReducer/reducer'; +import { actions as formActions } from 'react-redux-form'; +import { RequestParams } from 'app/utils/RequestParams'; +import api from 'app/utils/api'; +import { notificationActions } from 'app/Notifications'; +import { IStore } from 'app/istore'; +import { Dispatch } from 'redux'; +import { ensure } from 'shared/tsUtils'; +import { FileType } from 'shared/types/fileType'; + +const tocGenerationActions = { + reviewToc(fileId: string) { + return async (dispatch: Dispatch, getState: () => IStore) => { + const currentDoc = getState().documentViewer.doc.toJS(); + dispatch(formActions.reset('documentViewer.sidepanel.metadata')); + + const updatedFile = (await api.post('files/tocReviewed', new RequestParams({ fileId }))).json; + const doc = { + ...currentDoc, + defaultDoc: updatedFile, + documents: ensure(currentDoc.documents).map(d => { + if (d._id === updatedFile._id) { + return updatedFile; + } + return d; + }), + }; + + dispatch(notificationActions.notify('Document updated', 'success')); + dispatch(formActions.reset('documentViewer.sidepanel.metadata')); + dispatch(actions.set('viewer/doc', doc)); + }; + }, +}; + +export { tocGenerationActions }; diff --git a/app/react/ToggledFeatures/tocGeneration/index.ts b/app/react/ToggledFeatures/tocGeneration/index.ts new file mode 100644 index 0000000000..7785be5636 --- /dev/null +++ b/app/react/ToggledFeatures/tocGeneration/index.ts @@ -0,0 +1,4 @@ +export { TocGeneratedLabel } from './TocGeneratedLabel'; +export { ReviewTocButton } from './ReviewTocButton'; +export { FilterTocGeneration } from './FilterTocGeneration'; +export { tocGenerationUtils } from './utils'; diff --git a/app/react/ToggledFeatures/tocGeneration/specs/FilterTocGeneration.spec.tsx b/app/react/ToggledFeatures/tocGeneration/specs/FilterTocGeneration.spec.tsx new file mode 100644 index 0000000000..a2532c54ba --- /dev/null +++ b/app/react/ToggledFeatures/tocGeneration/specs/FilterTocGeneration.spec.tsx @@ -0,0 +1,68 @@ +import React from 'react'; +import { shallow, ShallowWrapper } from 'enzyme'; +import SelectFilter from 'app/Library/components/SelectFilter'; +import { FilterTocGeneration } from '../FilterTocGeneration'; +import { Aggregations } from 'shared/types/Aggregations'; + +describe('ReviewTocButton', () => { + let component: ShallowWrapper; + const aggregations = { + all: { + generatedToc: { + buckets: [ + { key: 'false', filtered: { doc_count: 2 } }, + { key: 'true', filtered: { doc_count: 5 } }, + ], + }, + }, + }; + + const render = (aggs: Aggregations = aggregations) => { + component = shallow( {}} aggregations={aggs} />); + }; + + it('should render nothing if file generatedToc is false', () => { + render(); + const options = component.find(SelectFilter).prop('options'); + expect(options).toEqual([ + expect.objectContaining({ value: true, results: 5 }), + expect.objectContaining({ value: false, results: 2 }), + ]); + }); + + describe('when there is bucket missing', () => { + it('should not fail (render blank state)', () => { + render({ + all: { + generatedToc: { + buckets: [{ key: 'false', filtered: { doc_count: 2 } }], + }, + }, + }); + const options = component.find(SelectFilter).prop('options'); + expect(options).toEqual([ + expect.objectContaining({ value: true, results: 0 }), + expect.objectContaining({ value: false, results: 2 }), + ]); + }); + }); + + describe('when aggregations are not defined/complete', () => { + it('should not fail (render blank state)', () => { + //@ts-ignore + render({}); + let options = component.find(SelectFilter).prop('options'); + expect(options).toEqual([ + expect.objectContaining({ value: true, results: 0 }), + expect.objectContaining({ value: false, results: 0 }), + ]); + + render({ all: {} }); + options = component.find(SelectFilter).prop('options'); + expect(options).toEqual([ + expect.objectContaining({ value: true, results: 0 }), + expect.objectContaining({ value: false, results: 0 }), + ]); + }); + }); +}); diff --git a/app/react/ToggledFeatures/tocGeneration/specs/ReviewTocButton.spec.tsx b/app/react/ToggledFeatures/tocGeneration/specs/ReviewTocButton.spec.tsx new file mode 100644 index 0000000000..43b032cc8d --- /dev/null +++ b/app/react/ToggledFeatures/tocGeneration/specs/ReviewTocButton.spec.tsx @@ -0,0 +1,37 @@ +import React from 'react'; +import { shallow, ShallowWrapper } from 'enzyme'; +import configureStore, { MockStore, MockStoreCreator } from 'redux-mock-store'; +import { Provider } from 'react-redux'; +import { ClientFile } from 'app/istore'; +import { ReviewTocButton } from '../ReviewTocButton'; + +describe('ReviewTocButton', () => { + let component: ShallowWrapper; + + const mockStoreCreator: MockStoreCreator = configureStore([]); + const render = (file: Partial) => { + const store: MockStore = mockStoreCreator({}); + component = shallow( + + + test + + + ) + .dive() + .dive(); + }; + + it('should render nothing if file generatedToc is false', () => { + render({ generatedToc: false }); + expect(component.find('button').length).toEqual(0); + + render({}); + expect(component.find('button').length).toEqual(0); + }); + + it('should render when generatedToc is true', () => { + render({ generatedToc: true }); + expect(component.find('button').length).toEqual(1); + }); +}); diff --git a/app/react/ToggledFeatures/tocGeneration/specs/actions.spec.ts b/app/react/ToggledFeatures/tocGeneration/specs/actions.spec.ts new file mode 100644 index 0000000000..8950ca6b73 --- /dev/null +++ b/app/react/ToggledFeatures/tocGeneration/specs/actions.spec.ts @@ -0,0 +1,99 @@ +import api from 'app/utils/api'; +import backend from 'fetch-mock'; +import * as notificationsTypes from 'app/Notifications/actions/actionTypes'; +import { actions as relationshipActions } from 'app/Relationships'; +import { APIURL } from 'app/config'; +import configureMockStore from 'redux-mock-store'; +import thunk from 'redux-thunk'; +import Immutable from 'immutable'; +import { mockID } from 'shared/uniqueID.js'; +import { ClientEntitySchema, IStore } from 'app/istore'; +import { tocGenerationActions } from '../actions'; + +const middlewares = [thunk]; +const mockStore = configureMockStore(middlewares); + +describe('reviewToc', () => { + it('should store the document with the response of reviewToc', done => { + mockID(); + const fileId = 'fileId'; + + backend.post(`${APIURL}files/tocReviewed`, { + body: JSON.stringify({ _id: fileId, generatedToc: false }), + }); + + spyOn(relationshipActions, 'reloadRelationships').and.returnValue({ + type: 'reloadRelationships', + }); + + const doc: ClientEntitySchema = { + name: 'doc', + _id: 'id', + sharedId: 'sharedId', + defaultDoc: { + _id: fileId, + generatedToc: true, + }, + documents: [ + { + _id: fileId, + generatedToc: true, + }, + ], + }; + + const updatedEntity = { + name: 'doc', + _id: 'id', + sharedId: 'sharedId', + defaultDoc: { + _id: fileId, + generatedToc: false, + }, + documents: [ + { + _id: fileId, + generatedToc: false, + }, + ], + }; + + const expectedActions = [ + { type: 'rrf/reset', model: 'documentViewer.sidepanel.metadata' }, + { + type: notificationsTypes.NOTIFY, + notification: { message: 'Document updated', type: 'success', id: 'unique_id' }, + }, + { type: 'rrf/reset', model: 'documentViewer.sidepanel.metadata' }, + { type: 'viewer/doc/SET', value: updatedEntity }, + ]; + + const store = mockStore({ + documentViewer: { + doc: Immutable.fromJS(doc), + references: Immutable.fromJS([]), + targetDocReferences: Immutable.fromJS([]), + targetDoc: Immutable.fromJS(doc), + uiState: Immutable.fromJS({}), + }, + }); + + spyOn(api, 'post').and.callThrough(); + store + //fot this to be properly typed, redux, redux-thunk need to be updated (and probably others), + //producing hundreds of type errors + //@ts-ignore + .dispatch(tocGenerationActions.reviewToc(fileId)) + .then(() => { + expect(api.post).toHaveBeenCalledWith('files/tocReviewed', { + data: { + fileId: 'fileId', + }, + headers: {}, + }); + expect(store.getActions()).toEqual(expectedActions); + }) + .then(done) + .catch(done.fail); + }); +}); diff --git a/app/react/ToggledFeatures/tocGeneration/specs/utils.spec.ts b/app/react/ToggledFeatures/tocGeneration/specs/utils.spec.ts new file mode 100644 index 0000000000..bfdcd09eb5 --- /dev/null +++ b/app/react/ToggledFeatures/tocGeneration/specs/utils.spec.ts @@ -0,0 +1,19 @@ +import Immutable from 'immutable'; +import { processQuery } from 'app/Library/helpers/requestState'; + +describe('Library/Uploads processQuery()', () => { + it('should add aggregateGeneratedToc if feature activated', () => { + const params = { q: '(order:desc,sort:creationDate)' }; + const globalResources = { + settings: { + collection: Immutable.fromJS({ features: { tocGeneration: {} } }), + }, + }; + const query = processQuery(params, globalResources, 'library'); + expect(query).toEqual({ + order: 'desc', + sort: 'creationDate', + aggregateGeneratedToc: true, + }); + }); +}); diff --git a/app/react/ToggledFeatures/tocGeneration/utils.ts b/app/react/ToggledFeatures/tocGeneration/utils.ts new file mode 100644 index 0000000000..42cef93ce9 --- /dev/null +++ b/app/react/ToggledFeatures/tocGeneration/utils.ts @@ -0,0 +1,11 @@ +import { Settings } from 'shared/types/settingsType'; +import { SearchParams } from 'shared/types/searchParams.d.ts'; + +export const tocGenerationUtils = { + aggregations(params: SearchParams, settings: Settings) { + return { + ...params, + ...(settings?.features?.tocGeneration ? { aggregateGeneratedToc: true } : {}), + }; + }, +}; diff --git a/app/react/Uploads/specs/UploadsRoute.spec.js b/app/react/Uploads/specs/UploadsRoute.spec.js index 18889e06cb..644c98994d 100644 --- a/app/react/Uploads/specs/UploadsRoute.spec.js +++ b/app/react/Uploads/specs/UploadsRoute.spec.js @@ -26,6 +26,7 @@ describe('UploadsRoute', () => { ]; const globalResources = { templates: Immutable(templates), + settings: { collection: Immutable({ features: {} }) }, thesauris: Immutable([]), relationTypes: Immutable([]), }; diff --git a/app/react/istore.d.ts b/app/react/istore.d.ts index 39366d8aba..955b58df2b 100644 --- a/app/react/istore.d.ts +++ b/app/react/istore.d.ts @@ -8,6 +8,7 @@ import { TemplateSchema } from 'shared/types/templateType'; import { EntitySchema } from 'shared/types/entityType'; import { UserGroupSchema } from 'shared/types/userGroupType'; import { ConnectionSchema } from 'shared/types/connectionType'; +import { FileType } from 'shared/types/fileType'; export interface TasksState { SyncState?: TaskStatus; @@ -83,8 +84,12 @@ interface ClientTemplateSchema extends TemplateSchema { _id: string; } +export interface ClientFile extends FileType { + _id: string; +} + export interface ClientEntitySchema extends EntitySchema { - documents?: []; + documents?: ClientFile[]; } export interface IStore { @@ -110,8 +115,8 @@ export interface IStore { documentViewer: { references: IImmutable; targetDocReferences: IImmutable; - doc: IImmutable; - targetDoc: IImmutable; + doc: IImmutable; + targetDoc: IImmutable; uiState: IImmutable<{ activeReference: string; }>; diff --git a/app/server.js b/app/server.js index 1cc383b1bb..1befa11243 100644 --- a/app/server.js +++ b/app/server.js @@ -119,7 +119,7 @@ DB.connect(config.DBHOST, dbAuth).then(async () => { const { evidencesVault, features } = await settings.get(); if (evidencesVault && evidencesVault.token && evidencesVault.template) { - console.info('==> 📥 evidences vault config detected, started sync ....'); + console.info('==> 📥 evidences vault config detected, started sync ....'); repeater.start( () => vaultSync.sync(evidencesVault.token, evidencesVault.template), 10000 diff --git a/app/shared/types/Aggregations.d.ts b/app/shared/types/Aggregations.d.ts new file mode 100644 index 0000000000..e1c059a581 --- /dev/null +++ b/app/shared/types/Aggregations.d.ts @@ -0,0 +1,13 @@ +export interface Aggregations { + all: { + [key: string]: { + buckets: Array<{ + key: string; + filtered: { + // eslint-disable-next-line camelcase + doc_count: number; + }; + }>; + }; + }; +} diff --git a/app/shared/types/connectionType.d.ts b/app/shared/types/connectionType.d.ts index 0bcd15b543..1a026f5731 100644 --- a/app/shared/types/connectionType.d.ts +++ b/app/shared/types/connectionType.d.ts @@ -19,6 +19,7 @@ export interface ConnectionSchema { title?: string; template?: ObjectIdSchema; published?: boolean; + generatedToc?: boolean; icon?: { _id?: string | null; label?: string; @@ -42,6 +43,7 @@ export interface ConnectionSchema { | null | string | number + | boolean | { label?: string | null; url?: string | null; @@ -75,6 +77,7 @@ export interface ConnectionSchema { | null | string | number + | boolean | { label?: string | null; url?: string | null; diff --git a/app/shared/types/searchParams.d.ts b/app/shared/types/searchParams.d.ts new file mode 100644 index 0000000000..53c1826b95 --- /dev/null +++ b/app/shared/types/searchParams.d.ts @@ -0,0 +1,33 @@ +/* eslint-disable */ +/**AUTO-GENERATED. RUN yarn emit-types to update.*/ + +export interface SearchParams { + query?: { + aggregateGeneratedToc?: boolean; + filters?: { + [k: string]: unknown | undefined; + }; + customFilters?: { + generatedToc?: { + values?: [] | [string]; + }; + }; + types?: [] | [string]; + _types?: [] | [string]; + fields?: [] | [string]; + allAggregations?: boolean; + aggregations?: string; + order?: 'asc' | 'desc'; + sort?: string; + limit?: number; + from?: number; + searchTerm?: string | number; + includeUnpublished?: boolean; + userSelectedSorting?: boolean; + treatAs?: string; + unpublished?: boolean; + select?: [] | [string]; + geolocation?: boolean; + }; + [k: string]: unknown | undefined; +} diff --git a/app/api/search/searchSchema.ts b/app/shared/types/searchParams.ts similarity index 85% rename from app/api/search/searchSchema.ts rename to app/shared/types/searchParams.ts index abae216f28..38be2b30da 100644 --- a/app/api/search/searchSchema.ts +++ b/app/shared/types/searchParams.ts @@ -1,14 +1,19 @@ -export const searchSchema = { +export const emitSchemaTypes = true; +export const searchParamsSchema = { + title: 'searchParams', properties: { query: { + additionalProperties: false, properties: { aggregateGeneratedToc: { type: 'boolean' }, filters: { type: 'object' }, customFilters: { + additionalProperties: false, type: 'object', properties: { generatedToc: { type: 'object', + additionalProperties: false, properties: { values: { type: 'array', items: [{ type: 'boolean' }] }, }, diff --git a/emitSchemaTypes.js b/emitSchemaTypes.js index 9f0c2df6fe..4b626ea540 100644 --- a/emitSchemaTypes.js +++ b/emitSchemaTypes.js @@ -63,28 +63,30 @@ const writeTypeFile = (file, commonImport, snippets) => { const emitSchemaTypes = async file => { try { - if (!file.match(/Schema/) || file.match(/spec/)) { + if (file.match(/spec/)) { return; } - const schemas = require(`./${file}`); + if (file.match(/shared\/types/) || file.match(/Schema/)) { + const schemas = require(`./${file}`); - if (!schemas.emitSchemaTypes) { - return; - } + if (!schemas.emitSchemaTypes) { + return; + } - const snippets = await Promise.all( - Object.entries(schemas).map(([name, schema]) => { - if (!name.match(/Schema$/)) { - return ''; - } - return compile(schema, schema.title || firstUp(name), opts); - }) - ); + const snippets = await Promise.all( + Object.entries(schemas).map(([name, schema]) => { + if (!name.match(/Schema$/)) { + return ''; + } + return compile(schema, schema.title || firstUp(name), opts); + }) + ); - const contents = fs.readFileSync(file).toString(); + const contents = fs.readFileSync(file).toString(); - writeTypeFile(file, typeImports(contents.match(typeImportFindRegex)), snippets); + writeTypeFile(file, typeImports(contents.match(typeImportFindRegex)), snippets); + } } catch (err) { console.error(`Failed emitting types from ${file}: ${err}.`); } From baa08e51e9be316c01e3ab90be4a8cec4063f280 Mon Sep 17 00:00:00 2001 From: Kevin Nderitu Date: Fri, 26 Feb 2021 14:44:22 +0300 Subject: [PATCH 07/31] added basic styles on autogenerated toc --- .../Documents/components/DocumentSidePanel.js | 9 ++--- app/react/Documents/components/ShowToc.js | 12 ++++--- .../Documents/components/scss/showToc.scss | 34 +++++++++++++++++++ app/react/Documents/components/scss/toc.scss | 18 ++++++++++ 4 files changed, 64 insertions(+), 9 deletions(-) create mode 100644 app/react/Documents/components/scss/showToc.scss create mode 100644 app/react/Documents/components/scss/toc.scss diff --git a/app/react/Documents/components/DocumentSidePanel.js b/app/react/Documents/components/DocumentSidePanel.js index 9d9cf7d75e..e4aed13429 100644 --- a/app/react/Documents/components/DocumentSidePanel.js +++ b/app/react/Documents/components/DocumentSidePanel.js @@ -1,9 +1,11 @@ +/* eslint-disable max-lines */ import { Tabs, TabLink, TabContent } from 'react-tabs-redux'; import { browserHistory } from 'react-router'; import { connect } from 'react-redux'; import Immutable from 'immutable'; import PropTypes from 'prop-types'; import React, { Component } from 'react'; +import './scss/toc.scss'; import { MetadataFormButtons, ShowMetadata } from 'app/Metadata'; import { NeedAuthorization } from 'app/Auth'; @@ -16,7 +18,6 @@ import ShowIf from 'app/App/ShowIf'; import SidePanel from 'app/Layout/SidePanel'; import DocumentSemanticSearchResults from 'app/SemanticSearch/components/DocumentResults'; import { CopyFromEntity } from 'app/Metadata/components/CopyFromEntity'; -import { FeatureToggle } from 'app/components/Elements/FeatureToggle'; import { TocGeneratedLabel, ReviewTocButton } from 'app/ToggledFeatures/tocGeneration'; import { Icon } from 'UI'; @@ -348,10 +349,10 @@ export class DocumentSidePanel extends Component {

Table of contents + + auto-created ⓘ +

- - auto-created ⓘ -
{tocElement.get('label')} + + {tocElement.getIn(['selectionRectangles', 0]).get('page')} + @@ -65,7 +70,4 @@ function mapDispatchToProps() { return { scrollToToc }; } -export default connect( - null, - mapDispatchToProps -)(ShowToc); +export default connect(null, mapDispatchToProps)(ShowToc); diff --git a/app/react/Documents/components/scss/showToc.scss b/app/react/Documents/components/scss/showToc.scss new file mode 100644 index 0000000000..702d875114 --- /dev/null +++ b/app/react/Documents/components/scss/showToc.scss @@ -0,0 +1,34 @@ +@import "../../../App/scss/config/_colors.scss"; + +.toc { + padding-left: 5px !important; + padding-right: 5px !important; + + ul.toc-view { + $indents: 5; + + @for $i from 0 through $indents { + li.toc-indent-#{$i} { + border-radius: 2px; + height: 28px; + + &:hover { + background-color: $c-info-light; + } + a:hover { + text-decoration: none; + }; + a { + padding-left: 4px; + } + } + } + } +} + +.page-number { + position: relative; + float: right; + font-weight: normal; + padding-right: 4px; +} \ No newline at end of file diff --git a/app/react/Documents/components/scss/toc.scss b/app/react/Documents/components/scss/toc.scss new file mode 100644 index 0000000000..e02c88b19b --- /dev/null +++ b/app/react/Documents/components/scss/toc.scss @@ -0,0 +1,18 @@ +@import "../../../App/scss/config/_colors.scss"; + +.tocHeader { + & { + padding-left: 5px; + } + + h1 { + span.label-generatedToc { + margin-left: 10px; + padding: 1px 6px 1px 6px; + font-size: 8px; + border: 1px solid $c-info-dark; + border-radius: 20px; + background-color: $c-info-light; + } + } +} \ No newline at end of file From 42e7dfd6b9c372357ec09cbaffa5af812bc7f0ef Mon Sep 17 00:00:00 2001 From: Daneryl Date: Mon, 25 Jan 2021 16:24:33 -0500 Subject: [PATCH 08/31] toc_generation service, WIP --- app/api/entities/entitiesModel.ts | 1 + app/api/search/documentQueryBuilder.js | 8 ++- app/api/search/metadataAggregations.js | 7 ++ app/api/toc_generation/specs/fixtures.ts | 48 ++++++++++++++ .../toc_generation/specs/tocService.spec.ts | 66 +++++++++++++++++++ app/api/toc_generation/tocService.ts | 37 +++++++++++ app/shared/types/commonSchemas.ts | 1 + app/shared/types/commonTypes.d.ts | 10 ++- app/shared/types/entitySchema.ts | 1 + app/shared/types/entityType.d.ts | 1 + app/shared/types/fileSchema.ts | 1 + app/shared/types/fileType.d.ts | 1 + 12 files changed, 180 insertions(+), 2 deletions(-) create mode 100644 app/api/toc_generation/specs/fixtures.ts create mode 100644 app/api/toc_generation/specs/tocService.spec.ts create mode 100644 app/api/toc_generation/tocService.ts diff --git a/app/api/entities/entitiesModel.ts b/app/api/entities/entitiesModel.ts index 87143cf71c..2230b711d8 100644 --- a/app/api/entities/entitiesModel.ts +++ b/app/api/entities/entitiesModel.ts @@ -32,6 +32,7 @@ const mongoSchema = new mongoose.Schema( creationDate: Number, editDate: Number, metadata: mongoose.Schema.Types.Mixed, + systemMetadata: mongoose.Schema.Types.Mixed, suggestedMetadata: mongoose.Schema.Types.Mixed, user: { type: mongoose.Schema.Types.ObjectId, ref: 'users' }, }, diff --git a/app/api/search/documentQueryBuilder.js b/app/api/search/documentQueryBuilder.js index 19d2b77b48..e96e5dbafd 100644 --- a/app/api/search/documentQueryBuilder.js +++ b/app/api/search/documentQueryBuilder.js @@ -2,7 +2,7 @@ import { preloadOptionsSearch } from 'shared/config'; import filterToMatch, { multiselectFilter } from './metadataMatchers'; -import { propertyToAggregation } from './metadataAggregations'; +import { propertyToAggregation, generatedTocAggregations } from './metadataAggregations'; export default function() { const baseQuery = { @@ -40,6 +40,7 @@ export default function() { aggregations: { all: { global: {}, + customAggregations: {}, aggregations: { _types: { terms: { @@ -241,8 +242,13 @@ export default function() { return this; }, + generatedTOCAggregations() { + baseQuery.aggregations.all.customAggregations.generatedToc = generatedTocAggregations(baseQuery); + }, + aggregations(properties, dictionaries) { properties.forEach(property => { + // console.log(JSON.stringify(propertyToAggregation(property, dictionaries, baseQuery), null, ' ')); baseQuery.aggregations.all.aggregations[property.name] = propertyToAggregation( property, dictionaries, diff --git a/app/api/search/metadataAggregations.js b/app/api/search/metadataAggregations.js index a44f6acae5..740e975f10 100644 --- a/app/api/search/metadataAggregations.js +++ b/app/api/search/metadataAggregations.js @@ -167,3 +167,10 @@ export const propertyToAggregation = (property, dictionaries, baseQuery, suggest return aggregation(path, should, filters); }; + +export const generatedTocAggregations = baseQuery => { + const path = 'generatedToc.raw'; + const filters = extractFilters(baseQuery, path); + const { should } = baseQuery.query.bool; + return aggregation(path, should, filters); +}; diff --git a/app/api/toc_generation/specs/fixtures.ts b/app/api/toc_generation/specs/fixtures.ts new file mode 100644 index 0000000000..49eea0d873 --- /dev/null +++ b/app/api/toc_generation/specs/fixtures.ts @@ -0,0 +1,48 @@ +import { testingDB, DBFixture } from 'api/utils/testing_db'; + +const fixtures: DBFixture = { + entities: [ + { + sharedId: 'shared1', + title: 'pdf1entity', + }, + { + sharedId: 'shared3', + title: 'pdf3entity', + }, + ], + files: [ + { + _id: testingDB.id(), + entity: 'shared1', + filename: 'pdf1.pdf', + originalname: 'originalPdf1.pdf', + type: 'document', + }, + { + _id: testingDB.id(), + type: 'document', + }, + { + _id: testingDB.id(), + type: 'custom', + filename: 'background.jpg', + }, + { + _id: testingDB.id(), + type: 'document', + filename: 'pdf2.pdf', + originalname: 'originalPdf2.pdf', + toc: [{}], + }, + { + _id: testingDB.id(), + entity: 'shared3', + type: 'document', + filename: 'pdf3.pdf', + originalname: 'originalPdf4.pdf', + }, + ], +}; + +export { fixtures }; diff --git a/app/api/toc_generation/specs/tocService.spec.ts b/app/api/toc_generation/specs/tocService.spec.ts new file mode 100644 index 0000000000..e38663e23a --- /dev/null +++ b/app/api/toc_generation/specs/tocService.spec.ts @@ -0,0 +1,66 @@ +import { testingDB } from 'api/utils/testing_db'; +import request from 'shared/JSONRequest'; +import { files } from 'api/files'; +import { elasticTesting } from 'api/utils/elastic_testing'; +import { fixtures } from './fixtures'; +import { tocService } from '../tocService'; + +describe('tocService', () => { + beforeAll(async () => { + // const elasticIndex = 'toc.service.index'; + // await testingDB.clearAllAndLoad(fixtures, elasticIndex); + spyOn(request, 'uploadFile').and.callFake(async (_url, filename) => { + if (filename === 'pdf1.pdf') { + return Promise.resolve([{ label: 'section1 pdf1' }]); + } + if (filename === 'pdf3.pdf') { + return Promise.resolve([{ label: 'section1 pdf3' }]); + } + throw new Error(`this file is not supposed to be sent for toc generation ${filename}`); + }); + }); + + afterAll(async () => { + await testingDB.disconnect(); + }); + + describe('processNext', () => { + beforeAll(async () => { + const elasticIndex = 'toc.service.index'; + await testingDB.clearAllAndLoad(fixtures, elasticIndex); + await elasticTesting.resetIndex(); + await tocService.processNext(); + await tocService.processNext(); + await elasticTesting.refresh(); + }); + + it('should not fail when there is no more to process', async () => { + await expect(tocService.processNext()).resolves.not.toThrow(); + }); + + it('should send the next pdfFile and save toc generated', async () => { + let [fileProcessed] = await files.get({ filename: 'pdf1.pdf' }); + expect(fileProcessed.toc).toEqual([{ label: 'section1 pdf1' }]); + expect(fileProcessed.generatedToc).toEqual(true); + + [fileProcessed] = await files.get({ filename: 'pdf3.pdf' }); + expect(fileProcessed.toc).toEqual([{ label: 'section1 pdf3' }]); + expect(fileProcessed.generatedToc).toEqual(true); + }); + + it('should reindex the affected entities', async () => { + const entitiesIndexed = await elasticTesting.getIndexedEntities(); + + expect(entitiesIndexed).toEqual([ + expect.objectContaining({ + title: 'pdf1entity', + systemMetadata: { generatedToc: [{ value: true }] }, + }), + expect.objectContaining({ + title: 'pdf3entity', + systemMetadata: { generatedToc: [{ value: true }] }, + }), + ]); + }); + }); +}); diff --git a/app/api/toc_generation/tocService.ts b/app/api/toc_generation/tocService.ts new file mode 100644 index 0000000000..0243afc87e --- /dev/null +++ b/app/api/toc_generation/tocService.ts @@ -0,0 +1,37 @@ +import { files, uploadsPath } from 'api/files'; +import request from 'shared/JSONRequest'; +import entities from 'api/entities'; + +const tocService = { + async processNext() { + const [nextFile] = await files.get( + { toc: { $exists: false }, type: 'document', filename: { $exists: true } }, + '', + { + sort: { _id: 1 }, + limit: 1, + } + ); + + if (!nextFile) { + return null; + } + + const toc = await request.uploadFile( + 'url_toc_service', + nextFile.filename, + uploadsPath(nextFile.filename) + ); + + await files.save({ ...nextFile, toc, generatedToc: true }); + const parentEntities = await entities.get({ sharedId: nextFile.entity }, { language: 1 }); + return entities.saveMultiple( + parentEntities.map(entity => ({ + ...entity, + systemMetadata: { ...(entity.systemMetadata || {}), generatedToc: [{ value: true }] }, + })) + ); + }, +}; + +export { tocService }; diff --git a/app/shared/types/commonSchemas.ts b/app/shared/types/commonSchemas.ts index 67b411f6b4..f7e037d87c 100644 --- a/app/shared/types/commonSchemas.ts +++ b/app/shared/types/commonSchemas.ts @@ -83,6 +83,7 @@ export const propertyValueSchema = { { type: 'null' }, { type: 'string' }, { type: 'number' }, + { type: 'boolean' }, linkSchema, dateRangeSchema, latLonSchema, diff --git a/app/shared/types/commonTypes.d.ts b/app/shared/types/commonTypes.d.ts index c2596d0e9c..0c7fcf2340 100644 --- a/app/shared/types/commonTypes.d.ts +++ b/app/shared/types/commonTypes.d.ts @@ -52,7 +52,15 @@ export type GeolocationSchema = { lon: number; }[]; -export type PropertyValueSchema = null | string | number | LinkSchema | DateRangeSchema | LatLonSchema | LatLonSchema[]; +export type PropertyValueSchema = + | null + | string + | number + | boolean + | LinkSchema + | DateRangeSchema + | LatLonSchema + | LatLonSchema[]; export interface MetadataObjectSchema { value: PropertyValueSchema; diff --git a/app/shared/types/entitySchema.ts b/app/shared/types/entitySchema.ts index c9ab3a6122..11edd2771d 100644 --- a/app/shared/types/entitySchema.ts +++ b/app/shared/types/entitySchema.ts @@ -144,6 +144,7 @@ export const entitySchema = { creationDate: { type: 'number' }, user: objectIdSchema, metadata: metadataSchema, + systemMetadata: metadataSchema, suggestedMetadata: metadataSchema, }, }; diff --git a/app/shared/types/entityType.d.ts b/app/shared/types/entityType.d.ts index 5c76a73570..dd2cb3f471 100644 --- a/app/shared/types/entityType.d.ts +++ b/app/shared/types/entityType.d.ts @@ -20,6 +20,7 @@ export interface EntitySchema { creationDate?: number; user?: ObjectIdSchema; metadata?: MetadataSchema; + systemMetadata?: MetadataSchema; suggestedMetadata?: MetadataSchema; [k: string]: unknown | undefined; } diff --git a/app/shared/types/fileSchema.ts b/app/shared/types/fileSchema.ts index 9962c6906b..0a9d4a3e49 100644 --- a/app/shared/types/fileSchema.ts +++ b/app/shared/types/fileSchema.ts @@ -26,6 +26,7 @@ export const fileSchema = { type: { type: 'string', enum: ['custom', 'document', 'thumbnail'] }, status: { type: 'string', enum: ['processing', 'failed', 'ready'] }, totalPages: { type: 'number' }, + generatedToc: { type: 'boolean' }, fullText: { type: 'object', additionalProperties: false, diff --git a/app/shared/types/fileType.d.ts b/app/shared/types/fileType.d.ts index 8ca9b2f47d..a3bfcc02d7 100644 --- a/app/shared/types/fileType.d.ts +++ b/app/shared/types/fileType.d.ts @@ -15,6 +15,7 @@ export interface FileType { type?: 'custom' | 'document' | 'thumbnail'; status?: 'processing' | 'failed' | 'ready'; totalPages?: number; + generatedToc?: boolean; fullText?: { /** * This interface was referenced by `undefined`'s JSON-Schema definition From 97405a3cfc0e5efa345ce541fc65d34b099ea4fd Mon Sep 17 00:00:00 2001 From: Daneryl Date: Wed, 10 Feb 2021 14:35:31 -0500 Subject: [PATCH 09/31] aggregations for generatedToc property --- app/api/entities/entitiesModel.ts | 2 +- app/api/search/documentQueryBuilder.js | 4 +--- app/api/search/metadataAggregations.js | 6 ++++-- app/api/search/search.js | 12 ++++++++++-- app/api/search/searchSchema.ts | 1 + app/api/search/specs/fixtures_elastic.js | 12 ++++-------- app/api/search/specs/search.spec.js | 9 +++++++++ app/api/toc_generation/specs/tocService.spec.ts | 4 ++-- app/api/toc_generation/tocService.ts | 2 +- app/shared/types/entitySchema.ts | 2 +- app/shared/types/entityType.d.ts | 2 +- database/elastic_mapping/base_properties.js | 3 +++ database/elastic_mapping/elasticMapFactory.ts | 1 - 13 files changed, 38 insertions(+), 22 deletions(-) diff --git a/app/api/entities/entitiesModel.ts b/app/api/entities/entitiesModel.ts index 2230b711d8..9c62693655 100644 --- a/app/api/entities/entitiesModel.ts +++ b/app/api/entities/entitiesModel.ts @@ -15,6 +15,7 @@ const mongoSchema = new mongoose.Schema( title: { type: String, required: true }, template: { type: mongoose.Schema.Types.ObjectId, ref: 'templates', index: true }, published: Boolean, + generatedToc: Boolean, icon: new mongoose.Schema({ _id: String, label: String, @@ -32,7 +33,6 @@ const mongoSchema = new mongoose.Schema( creationDate: Number, editDate: Number, metadata: mongoose.Schema.Types.Mixed, - systemMetadata: mongoose.Schema.Types.Mixed, suggestedMetadata: mongoose.Schema.Types.Mixed, user: { type: mongoose.Schema.Types.ObjectId, ref: 'users' }, }, diff --git a/app/api/search/documentQueryBuilder.js b/app/api/search/documentQueryBuilder.js index e96e5dbafd..65679ce7e4 100644 --- a/app/api/search/documentQueryBuilder.js +++ b/app/api/search/documentQueryBuilder.js @@ -40,7 +40,6 @@ export default function() { aggregations: { all: { global: {}, - customAggregations: {}, aggregations: { _types: { terms: { @@ -243,12 +242,11 @@ export default function() { }, generatedTOCAggregations() { - baseQuery.aggregations.all.customAggregations.generatedToc = generatedTocAggregations(baseQuery); + baseQuery.aggregations.all.aggregations.generatedToc = generatedTocAggregations(baseQuery); }, aggregations(properties, dictionaries) { properties.forEach(property => { - // console.log(JSON.stringify(propertyToAggregation(property, dictionaries, baseQuery), null, ' ')); baseQuery.aggregations.all.aggregations[property.name] = propertyToAggregation( property, dictionaries, diff --git a/app/api/search/metadataAggregations.js b/app/api/search/metadataAggregations.js index 740e975f10..aac071e4a0 100644 --- a/app/api/search/metadataAggregations.js +++ b/app/api/search/metadataAggregations.js @@ -169,8 +169,10 @@ export const propertyToAggregation = (property, dictionaries, baseQuery, suggest }; export const generatedTocAggregations = baseQuery => { - const path = 'generatedToc.raw'; + const path = 'generatedToc'; const filters = extractFilters(baseQuery, path); const { should } = baseQuery.query.bool; - return aggregation(path, should, filters); + const agg = aggregation(path, should, filters); + agg.terms.missing = 'false'; + return agg; }; diff --git a/app/api/search/search.js b/app/api/search/search.js index ad93ead8c4..8475759233 100644 --- a/app/api/search/search.js +++ b/app/api/search/search.js @@ -195,6 +195,9 @@ const indexedDictionaryValues = dictionary => }, {}); const _getAggregationDictionary = async (aggregation, language, property, dictionaries) => { + if (!property) { + return [{values: []}, {}]; + } if (property.type === 'relationship') { const entitiesSharedId = aggregation.buckets.map(bucket => bucket.key); @@ -241,7 +244,7 @@ const _denormalizeAggregations = async (aggregations, templates, dictionaries, l const properties = propertiesHelper.allUniqueProperties(templates); return Object.keys(aggregations).reduce(async (denormaLizedAgregationsPromise, key) => { const denormaLizedAgregations = await denormaLizedAgregationsPromise; - if (!aggregations[key].buckets || key === '_types' || aggregations[key].type === 'nested') { + if (!aggregations[key].buckets || key === '_types' || aggregations[key].type === 'nested' || key === 'generatedToc') { return Object.assign(denormaLizedAgregations, { [key]: aggregations[key] }); } @@ -377,6 +380,7 @@ const processResponse = async (response, templates, dictionaries, language, filt result._id = hit._id; return result; }); + const sanitizedAggregations = await _sanitizeAggregations( response.body.aggregations.all, templates, @@ -618,6 +622,10 @@ const search = { searchGeolocation(queryBuilder, templates); } + if (query.aggregateGeneratedToc) { + queryBuilder.generatedTOCAggregations(); + } + // queryBuilder.query() is the actual call return elastic .search({ body: queryBuilder.query() }) @@ -675,7 +683,7 @@ const search = { return snippetsFromSearchHit(response.body.hits.hits[0]); }, - async indexEntities(query, select = '', limit, batchCallback = () => {}) { + async indexEntities(query, select = '', limit, batchCallback = () => { }) { return indexEntities({ query, select, diff --git a/app/api/search/searchSchema.ts b/app/api/search/searchSchema.ts index 26b92ecfd9..9b67c6a1b1 100644 --- a/app/api/search/searchSchema.ts +++ b/app/api/search/searchSchema.ts @@ -2,6 +2,7 @@ export const searchSchema = { properties: { query: { properties: { + aggregateGeneratedToc: { type: 'boolean' }, filters: { type: 'object' }, types: { type: 'array', items: [{ type: 'string' }] }, _types: { type: 'array', items: [{ type: 'string' }] }, diff --git a/app/api/search/specs/fixtures_elastic.js b/app/api/search/specs/fixtures_elastic.js index 25d0432e2f..86cb00f2c7 100644 --- a/app/api/search/specs/fixtures_elastic.js +++ b/app/api/search/specs/fixtures_elastic.js @@ -92,6 +92,7 @@ export const fixtures = { title: 'Batman finishes es', published: true, user: userId, + generatedToc: true, metadata: { relationship: [ { value: batmanBegins, label: 'Batman begins es' }, @@ -115,6 +116,7 @@ export const fixtures = { language: 'es', title: 'Batman begins es', published: true, + generatedToc: false, user: userId, }, { @@ -176,10 +178,7 @@ export const fixtures = { field2: [{ value: 'bane' }], select1: [{ value: 'EgyptID', label: 'Egypt' }], rich_text: [{ value: 'rich' }], - multiselect1: [ - { value: 'EgyptID', label: 'Egypt' }, - { value: 'SpainID', label: 'Spain' }, - ], + multiselect1: [{ value: 'EgyptID', label: 'Egypt' }, { value: 'SpainID', label: 'Spain' }], groupedDictionary: [{ value: 'GermanyID' }], nestedField_nested: [{ value: { nested1: ['1', '2', '3'] } }], city_geolocation: [{ value: { lat: 1, lon: 2 } }], @@ -437,10 +436,7 @@ export const fixtures = { { label: 'Europe', id: 'EuropeID', - values: [ - { label: 'Germany', id: 'GermanyID' }, - { label: 'France', id: 'franceID' }, - ], + values: [{ label: 'Germany', id: 'GermanyID' }, { label: 'France', id: 'franceID' }], }, ], }, diff --git a/app/api/search/specs/search.spec.js b/app/api/search/specs/search.spec.js index 63a9bafdf6..e7152ea72a 100644 --- a/app/api/search/specs/search.spec.js +++ b/app/api/search/specs/search.spec.js @@ -163,6 +163,15 @@ describe('search', () => { .catch(catchErrors(done)); }); + it('should return generatedToc aggregations when requested for', async () => { + const response = await search.search({ aggregateGeneratedToc: true }, 'es'); + + const aggregations = response.aggregations.all.generatedToc.buckets; + + expect(aggregations.find(a => a.key === 'false').filtered.doc_count).toBe(3); + expect(aggregations.find(a => a.key === 'true').filtered.doc_count).toBe(1); + }); + it('should return aggregations when searching by 2 terms', done => { search .search({ searchTerm: 'english document' }, 'es') diff --git a/app/api/toc_generation/specs/tocService.spec.ts b/app/api/toc_generation/specs/tocService.spec.ts index e38663e23a..f9b2285b9a 100644 --- a/app/api/toc_generation/specs/tocService.spec.ts +++ b/app/api/toc_generation/specs/tocService.spec.ts @@ -54,11 +54,11 @@ describe('tocService', () => { expect(entitiesIndexed).toEqual([ expect.objectContaining({ title: 'pdf1entity', - systemMetadata: { generatedToc: [{ value: true }] }, + generatedToc: true, }), expect.objectContaining({ title: 'pdf3entity', - systemMetadata: { generatedToc: [{ value: true }] }, + generatedToc: true, }), ]); }); diff --git a/app/api/toc_generation/tocService.ts b/app/api/toc_generation/tocService.ts index 0243afc87e..80a5bdecd0 100644 --- a/app/api/toc_generation/tocService.ts +++ b/app/api/toc_generation/tocService.ts @@ -28,7 +28,7 @@ const tocService = { return entities.saveMultiple( parentEntities.map(entity => ({ ...entity, - systemMetadata: { ...(entity.systemMetadata || {}), generatedToc: [{ value: true }] }, + generatedToc: true, })) ); }, diff --git a/app/shared/types/entitySchema.ts b/app/shared/types/entitySchema.ts index 11edd2771d..ffe6441f67 100644 --- a/app/shared/types/entitySchema.ts +++ b/app/shared/types/entitySchema.ts @@ -128,6 +128,7 @@ export const entitySchema = { title: { type: 'string', minLength: 1, stringMeetsLuceneMaxLimit: true }, template: objectIdSchema, published: { type: 'boolean' }, + generatedToc: { type: 'boolean' }, icon: { type: 'object', additionalProperties: false, @@ -144,7 +145,6 @@ export const entitySchema = { creationDate: { type: 'number' }, user: objectIdSchema, metadata: metadataSchema, - systemMetadata: metadataSchema, suggestedMetadata: metadataSchema, }, }; diff --git a/app/shared/types/entityType.d.ts b/app/shared/types/entityType.d.ts index dd2cb3f471..337cfdc0aa 100644 --- a/app/shared/types/entityType.d.ts +++ b/app/shared/types/entityType.d.ts @@ -11,6 +11,7 @@ export interface EntitySchema { title?: string; template?: ObjectIdSchema; published?: boolean; + generatedToc?: boolean; icon?: { _id?: string | null; label?: string; @@ -20,7 +21,6 @@ export interface EntitySchema { creationDate?: number; user?: ObjectIdSchema; metadata?: MetadataSchema; - systemMetadata?: MetadataSchema; suggestedMetadata?: MetadataSchema; [k: string]: unknown | undefined; } diff --git a/database/elastic_mapping/base_properties.js b/database/elastic_mapping/base_properties.js index 1ad1d5efbc..e37004fea4 100644 --- a/database/elastic_mapping/base_properties.js +++ b/database/elastic_mapping/base_properties.js @@ -64,6 +64,9 @@ const properties = { sort: { type: 'keyword' }, }, }, + generatedToc: { + type: 'keyword', + }, type: { type: 'keyword', }, diff --git a/database/elastic_mapping/elasticMapFactory.ts b/database/elastic_mapping/elasticMapFactory.ts index fcd3a9369d..d141033d0e 100644 --- a/database/elastic_mapping/elasticMapFactory.ts +++ b/database/elastic_mapping/elasticMapFactory.ts @@ -25,7 +25,6 @@ export default { const fieldMapping = propertyMappings[property.type](); map.properties.metadata.properties[property.name] = { properties: fieldMapping }; map.properties.suggestedMetadata.properties[property.name] = { properties: fieldMapping }; - return map; }, baseMapping), baseMappingObject From ed1565ae8446f28c09f62d88dfa59ea886740266 Mon Sep 17 00:00:00 2001 From: Daneryl Date: Thu, 11 Feb 2021 08:35:12 -0500 Subject: [PATCH 10/31] review toc endpoint --- app/api/entities/entities.js | 4 +++ app/api/entities/specs/entities.spec.js | 5 ++- app/api/files/files.ts | 25 +++++++++++++++ app/api/files/routes.ts | 23 ++++++++++++++ app/api/files/specs/fixtures.ts | 40 ++++++++++++++++++++---- app/api/files/specs/routes.spec.ts | 36 +++++++++++++++++++-- app/api/files/specs/uploadRoutes.spec.ts | 17 +++++----- app/api/toc_generation/specs/fixtures.ts | 15 +++++++++ app/api/toc_generation/tocService.ts | 10 +++--- 9 files changed, 155 insertions(+), 20 deletions(-) diff --git a/app/api/entities/entities.js b/app/api/entities/entities.js index 753d02ddd4..b8e7c1c6c4 100644 --- a/app/api/entities/entities.js +++ b/app/api/entities/entities.js @@ -163,6 +163,10 @@ async function updateEntity(entity, _template) { if (typeof entity.template !== 'undefined') { d.template = entity.template; } + + if (typeof entity.template !== 'undefined') { + d.generatedToc = entity.generatedToc; + } return model.save(d); }) ); diff --git a/app/api/entities/specs/entities.spec.js b/app/api/entities/specs/entities.spec.js index 3769354886..94166f627c 100644 --- a/app/api/entities/specs/entities.spec.js +++ b/app/api/entities/specs/entities.spec.js @@ -340,7 +340,7 @@ describe('entities', () => { }); }); - describe('when published/template property changes', () => { + describe('when published/template/generatedToc property changes', () => { it('should replicate the change for all the languages', done => { const doc = { _id: batmanFinishesId, @@ -348,6 +348,7 @@ describe('entities', () => { metadata: {}, published: false, template: templateId, + generatedToc: true, }; entities @@ -364,8 +365,10 @@ describe('entities', () => { expect(docES.template).toBeDefined(); expect(docES.published).toBe(false); + expect(docES.generatedToc).toBe(true); expect(docES.template.equals(templateId)).toBe(true); expect(docEN.published).toBe(false); + expect(docEN.generatedToc).toBe(true); expect(docEN.template.equals(templateId)).toBe(true); done(); }) diff --git a/app/api/files/files.ts b/app/api/files/files.ts index c6ecd6ee7c..9e4108e558 100644 --- a/app/api/files/files.ts +++ b/app/api/files/files.ts @@ -1,6 +1,7 @@ import { deleteUploadedFiles } from 'api/files/filesystem'; import connections from 'api/relationships'; import { search } from 'api/search'; +import entities from 'api/entities'; import model from './filesModel'; import { validateFile } from '../../shared/types/fileSchema'; @@ -30,4 +31,28 @@ export const files = { return toDeleteFiles; }, + + async tocReviewed(_id: string) { + const savedFile = await files.save({ _id, generatedToc: false }); + const sameEntityFiles = await files.get({ entity: savedFile.entity }, { generatedToc: 1 }); + const [entity] = await entities.get({ + sharedId: savedFile.entity, + language: savedFile.language, + }); + + await entities.save( + { + _id: entity._id, + sharedId: entity.sharedId, + template: entity.template, + generatedToc: sameEntityFiles.reduce( + (generated, file) => generated || Boolean(file.generatedToc), + false + ), + }, + { user: {}, language: savedFile.language } + ); + + return savedFile; + }, }; diff --git a/app/api/files/routes.ts b/app/api/files/routes.ts index 5c4bb0fff2..4c5de41b3c 100644 --- a/app/api/files/routes.ts +++ b/app/api/files/routes.ts @@ -10,6 +10,7 @@ import activitylogMiddleware from 'api/activitylog/activitylogMiddleware'; import { CSVLoader } from 'api/csv'; import { files } from './files'; import { validation, createError, handleError } from '../utils'; +import entities from 'api/entities'; export default (app: Application) => { app.post( @@ -57,6 +58,28 @@ export default (app: Application) => { .catch(next); }); + app.post( + '/api/files/tocReviewed', + needsAuthorization(['admin', 'editor']), + validation.validateRequest({ + properties: { + body: { + required: ['fileId'], + properties: { + fileId: { type: 'string' }, + }, + }, + }, + }), + async (req, res, next) => { + try { + res.json(await files.tocReviewed(req.body.fileId)); + } catch (e) { + next(e); + } + } + ); + app.get( '/api/files/:filename', validation.validateRequest({ diff --git a/app/api/files/specs/fixtures.ts b/app/api/files/specs/fixtures.ts index 786593a43a..1bab9ad9da 100644 --- a/app/api/files/specs/fixtures.ts +++ b/app/api/files/specs/fixtures.ts @@ -5,22 +5,31 @@ const entityEnId = db.id(); const uploadId = db.id(); const uploadId2 = db.id(); const templateId = db.id(); +const importTemplate = db.id(); const fileName1 = 'f2082bf51b6ef839690485d7153e847a.pdf'; const fixtures: DBFixture = { files: [ { _id: uploadId, - entity: 'entity', + entity: 'sharedId1', + language: 'es', + generatedToc: true, originalname: 'upload1', filename: fileName1, type: 'custom', }, { _id: uploadId2, - entity: 'entity', + generatedToc: true, + language: 'es', + entity: 'sharedId1', filename: 'fileNotInDisk', }, + { + entity: 'sharedId1', + filename: 'fileWithoutTocFlag', + }, { _id: db.id(), filename: 'fileNotOnDisk' }, { _id: db.id(), originalname: 'upload2', type: 'custom' }, { _id: db.id(), originalname: 'upload3', type: 'document' }, @@ -36,11 +45,21 @@ const fixtures: DBFixture = { sharedId: 'sharedId1', language: 'es', title: 'Gadgets 01 ES', - toc: [{ _id: db.id(), label: 'existingToc' }], + generatedToc: true, + template: templateId, + }, + { + _id: entityEnId, + template: templateId, + sharedId: 'sharedId1', + language: 'en', + title: 'Gadgets 01 EN', }, - { _id: entityEnId, sharedId: 'sharedId1', language: 'en', title: 'Gadgets 01 EN' }, ], - templates: [{ _id: templateId, default: true, name: 'mydoc', properties: [] }], + templates: [ + { _id: templateId, default: true, name: 'mydoc', properties: [] }, + { _id: importTemplate, default: true, name: 'import', properties: [] }, + ], settings: [ { _id: db.id(), @@ -51,4 +70,13 @@ const fixtures: DBFixture = { ], }; -export { fixtures, entityId, entityEnId, fileName1, uploadId, uploadId2, templateId }; +export { + fixtures, + entityId, + entityEnId, + fileName1, + uploadId, + uploadId2, + templateId, + importTemplate, +}; diff --git a/app/api/files/specs/routes.spec.ts b/app/api/files/specs/routes.spec.ts index 88b63e1262..28af682930 100644 --- a/app/api/files/specs/routes.spec.ts +++ b/app/api/files/specs/routes.spec.ts @@ -12,6 +12,7 @@ import { FileType } from 'shared/types/fileType'; import { fixtures, uploadId, uploadId2 } from './fixtures'; import { files } from '../files'; import uploadRoutes from '../routes'; +import entities from 'api/entities'; jest.mock( '../../auth/authMiddleware.ts', @@ -47,7 +48,7 @@ describe('files routes', () => { }); it('should reindex all entities that are related to the saved file', async () => { - expect(search.indexEntities).toHaveBeenCalledWith({ sharedId: 'entity' }, '+fullText'); + expect(search.indexEntities).toHaveBeenCalledWith({ sharedId: 'sharedId1' }, '+fullText'); }); }); @@ -85,7 +86,7 @@ describe('files routes', () => { .query({ _id: uploadId2.toString() }); expect(search.indexEntities).toHaveBeenCalledWith( - { sharedId: { $in: ['entity'] } }, + { sharedId: { $in: ['sharedId1'] } }, '+fullText' ); }); @@ -107,5 +108,36 @@ describe('files routes', () => { expect(response.body.errors[0].message).toBe('should be string'); }); + + describe('api/files/tocReviewed', () => { + it('should set tocGenerated to false on the file', async () => { + const response: SuperTestResponse = await request(app) + .post('/api/files/tocReviewed') + .set('content-language', 'es') + .send({ fileId: uploadId.toString() }); + + const [file] = await files.get({ _id: uploadId }); + expect(file.generatedToc).toBe(false); + expect(response.body.entity).toBe('sharedId1'); + }); + + it('should set tocGenerated to false on the entity when all associated files are false', async () => { + await request(app) + .post('/api/files/tocReviewed') + .send({ fileId: uploadId.toString() }) + .expect(200); + + let [entity] = await entities.get({ sharedId: 'sharedId1' }); + expect(entity.generatedToc).toBe(true); + + await request(app) + .post('/api/files/tocReviewed') + .send({ fileId: uploadId2.toString() }) + .expect(200); + + [entity] = await entities.get({ sharedId: 'sharedId1' }); + expect(entity.generatedToc).toBe(false); + }); + }); }); }); diff --git a/app/api/files/specs/uploadRoutes.spec.ts b/app/api/files/specs/uploadRoutes.spec.ts index b8a7fb44a5..3cb15c61dd 100644 --- a/app/api/files/specs/uploadRoutes.spec.ts +++ b/app/api/files/specs/uploadRoutes.spec.ts @@ -16,7 +16,7 @@ import { setUpApp, socketEmit, iosocket } from 'api/utils/testingRoutes'; import { FileType } from 'shared/types/fileType'; import entities from 'api/entities'; -import { fixtures, templateId } from './fixtures'; +import { fixtures, templateId, importTemplate } from './fixtures'; import { files } from '../files'; import uploadRoutes from '../routes'; @@ -72,7 +72,10 @@ describe('upload routes', () => { expect(iosocket.emit).toHaveBeenCalledWith('conversionStart', 'sharedId1'); expect(iosocket.emit).toHaveBeenCalledWith('documentProcessed', 'sharedId1'); - const [upload] = await files.get({ entity: 'sharedId1' }, '+fullText'); + const [upload] = await files.get( + { originalname: 'f2082bf51b6ef839690485d7153e847a.pdf' }, + '+fullText' + ); expect(upload).toEqual( expect.objectContaining({ @@ -105,14 +108,14 @@ describe('upload routes', () => { it('should detect English documents and store the result', async () => { await uploadDocument('uploads/eng.pdf'); - const [upload] = await files.get({ entity: 'sharedId1' }); + const [upload] = await files.get({ originalname: 'eng.pdf' }); expect(upload.language).toBe('eng'); }, 10000); it('should detect Spanish documents and store the result', async () => { await uploadDocument('uploads/spn.pdf'); - const [upload] = await files.get({ entity: 'sharedId1' }); + const [upload] = await files.get({ originalname: 'spn.pdf' }); expect(upload.language).toBe('spa'); }); }); @@ -126,7 +129,7 @@ describe('upload routes', () => { .attach('file', path.join(__dirname, 'uploads/invalid_document.txt')) ); - const [upload] = await files.get({ entity: 'sharedId1' }, '+fullText'); + const [upload] = await files.get({ originalname: 'invalid_document.txt' }, '+fullText'); expect(upload.status).toBe('failed'); }); @@ -176,7 +179,7 @@ describe('upload routes', () => { await socketEmit('IMPORT_CSV_END', async () => request(app) .post('/api/import') - .field('template', templateId.toString()) + .field('template', importTemplate.toString()) .attach('file', `${__dirname}/uploads/importcsv.csv`) ); @@ -184,7 +187,7 @@ describe('upload routes', () => { expect(iosocket.emit).toHaveBeenCalledWith('IMPORT_CSV_PROGRESS', 1); expect(iosocket.emit).toHaveBeenCalledWith('IMPORT_CSV_PROGRESS', 2); - const imported = await entities.get({ template: templateId }); + const imported = await entities.get({ template: importTemplate }); expect(imported).toEqual([ expect.objectContaining({ title: 'imported entity one' }), expect.objectContaining({ title: 'imported entity two' }), diff --git a/app/api/toc_generation/specs/fixtures.ts b/app/api/toc_generation/specs/fixtures.ts index 49eea0d873..6c7639cb9e 100644 --- a/app/api/toc_generation/specs/fixtures.ts +++ b/app/api/toc_generation/specs/fixtures.ts @@ -1,14 +1,24 @@ import { testingDB, DBFixture } from 'api/utils/testing_db'; +const templateId = testingDB.id(); + const fixtures: DBFixture = { + templates: [ + { + _id: templateId, + properties: [], + }, + ], entities: [ { sharedId: 'shared1', title: 'pdf1entity', + template: templateId, }, { sharedId: 'shared3', title: 'pdf3entity', + template: templateId, }, ], files: [ @@ -16,17 +26,20 @@ const fixtures: DBFixture = { _id: testingDB.id(), entity: 'shared1', filename: 'pdf1.pdf', + language: 'es', originalname: 'originalPdf1.pdf', type: 'document', }, { _id: testingDB.id(), type: 'document', + language: 'es', }, { _id: testingDB.id(), type: 'custom', filename: 'background.jpg', + language: 'es', }, { _id: testingDB.id(), @@ -34,6 +47,7 @@ const fixtures: DBFixture = { filename: 'pdf2.pdf', originalname: 'originalPdf2.pdf', toc: [{}], + language: 'es', }, { _id: testingDB.id(), @@ -41,6 +55,7 @@ const fixtures: DBFixture = { type: 'document', filename: 'pdf3.pdf', originalname: 'originalPdf4.pdf', + language: 'es', }, ], }; diff --git a/app/api/toc_generation/tocService.ts b/app/api/toc_generation/tocService.ts index 80a5bdecd0..d64bbeeed0 100644 --- a/app/api/toc_generation/tocService.ts +++ b/app/api/toc_generation/tocService.ts @@ -24,12 +24,14 @@ const tocService = { ); await files.save({ ...nextFile, toc, generatedToc: true }); - const parentEntities = await entities.get({ sharedId: nextFile.entity }, { language: 1 }); - return entities.saveMultiple( - parentEntities.map(entity => ({ + const [entity] = await entities.get({ sharedId: nextFile.entity }, {}); + return entities.save( + { ...entity, generatedToc: true, - })) + }, + { user: {}, language: nextFile.language }, + false ); }, }; From 2023a2222179ad5e9616deed8da49bbc8a7ee637 Mon Sep 17 00:00:00 2001 From: Daneryl Date: Thu, 11 Feb 2021 11:50:40 -0500 Subject: [PATCH 11/31] tocService as a feature and cronjob --- app/api/toc_generation/specs/fixtures.ts | 1 + app/api/toc_generation/specs/tocService.spec.ts | 12 ++++++------ app/api/toc_generation/tocService.ts | 13 ++++++++----- app/server.js | 11 +++++++++-- app/shared/JSONRequest.js | 4 ++-- app/shared/types/settingsSchema.ts | 9 ++++++++- app/shared/types/settingsType.d.ts | 4 +++- 7 files changed, 37 insertions(+), 17 deletions(-) diff --git a/app/api/toc_generation/specs/fixtures.ts b/app/api/toc_generation/specs/fixtures.ts index 6c7639cb9e..2370acac38 100644 --- a/app/api/toc_generation/specs/fixtures.ts +++ b/app/api/toc_generation/specs/fixtures.ts @@ -54,6 +54,7 @@ const fixtures: DBFixture = { entity: 'shared3', type: 'document', filename: 'pdf3.pdf', + toc: [], originalname: 'originalPdf4.pdf', language: 'es', }, diff --git a/app/api/toc_generation/specs/tocService.spec.ts b/app/api/toc_generation/specs/tocService.spec.ts index f9b2285b9a..d77ec52000 100644 --- a/app/api/toc_generation/specs/tocService.spec.ts +++ b/app/api/toc_generation/specs/tocService.spec.ts @@ -6,10 +6,10 @@ import { fixtures } from './fixtures'; import { tocService } from '../tocService'; describe('tocService', () => { + const service = tocService('url'); beforeAll(async () => { - // const elasticIndex = 'toc.service.index'; - // await testingDB.clearAllAndLoad(fixtures, elasticIndex); - spyOn(request, 'uploadFile').and.callFake(async (_url, filename) => { + spyOn(request, 'uploadFile').and.callFake(async (url, filename) => { + expect(url).toBe('url'); if (filename === 'pdf1.pdf') { return Promise.resolve([{ label: 'section1 pdf1' }]); } @@ -29,13 +29,13 @@ describe('tocService', () => { const elasticIndex = 'toc.service.index'; await testingDB.clearAllAndLoad(fixtures, elasticIndex); await elasticTesting.resetIndex(); - await tocService.processNext(); - await tocService.processNext(); + await service.processNext(); + await service.processNext(); await elasticTesting.refresh(); }); it('should not fail when there is no more to process', async () => { - await expect(tocService.processNext()).resolves.not.toThrow(); + await expect(service.processNext()).resolves.not.toThrow(); }); it('should send the next pdfFile and save toc generated', async () => { diff --git a/app/api/toc_generation/tocService.ts b/app/api/toc_generation/tocService.ts index d64bbeeed0..69df4c7c59 100644 --- a/app/api/toc_generation/tocService.ts +++ b/app/api/toc_generation/tocService.ts @@ -2,23 +2,26 @@ import { files, uploadsPath } from 'api/files'; import request from 'shared/JSONRequest'; import entities from 'api/entities'; -const tocService = { +const tocService = (serviceUrl: string) => ({ async processNext() { const [nextFile] = await files.get( - { toc: { $exists: false }, type: 'document', filename: { $exists: true } }, + { + $or: [{ toc: { $size: 0 } }, { toc: { $exists: false } }], + type: 'document', + filename: { $exists: true }, + }, '', { sort: { _id: 1 }, limit: 1, } ); - if (!nextFile) { return null; } const toc = await request.uploadFile( - 'url_toc_service', + serviceUrl, nextFile.filename, uploadsPath(nextFile.filename) ); @@ -34,6 +37,6 @@ const tocService = { false ); }, -}; +}); export { tocService }; diff --git a/app/server.js b/app/server.js index bd7cf6c511..1cc383b1bb 100644 --- a/app/server.js +++ b/app/server.js @@ -30,6 +30,7 @@ import { tenants } from './api/tenants/tenantContext'; import { multitenantMiddleware } from './api/utils/multitenantMiddleware'; import { staticFilesMiddleware } from './api/utils/staticFilesMiddleware'; import { customUploadsPath, uploadsPath } from './api/files/filesystem'; +import { tocService } from './api/toc_generation/tocService'; mongoose.Promise = Promise; @@ -40,7 +41,7 @@ const http = Server(app); const uncaughtError = error => { handleError(error, { uncaught: true }); - process.exit(1); + throw error; }; process.on('unhandledRejection', uncaughtError); @@ -116,7 +117,7 @@ DB.connect(config.DBHOST, dbAuth).then(async () => { if (!config.multiTenant && !config.clusterMode) { syncWorker.start(); - const { evidencesVault } = await settings.get(); + const { evidencesVault, features } = await settings.get(); if (evidencesVault && evidencesVault.token && evidencesVault.template) { console.info('==> 📥 evidences vault config detected, started sync ....'); repeater.start( @@ -125,6 +126,12 @@ DB.connect(config.DBHOST, dbAuth).then(async () => { ); } + if (features && features.tocGeneration && features.tocGeneration.url) { + console.info('==> 🗂️ automatically generating TOCs using external service'); + const service = tocService(features.tocGeneration.url); + repeater.start(() => service.processNext(), 10000); + } + repeater.start( () => TaskProvider.runAndWait('TopicClassificationSync', 'TopicClassificationSync', { diff --git a/app/shared/JSONRequest.js b/app/shared/JSONRequest.js index bf975c2f7f..1473fcafbd 100644 --- a/app/shared/JSONRequest.js +++ b/app/shared/JSONRequest.js @@ -126,8 +126,8 @@ export default { .set('X-Requested-With', 'XMLHttpRequest') .set('Cookie', cookie || '') .attach('file', file, filename) - .then(() => { - resolve(); + .then(response => { + resolve(response.body); }) .catch(err => { reject(err); diff --git a/app/shared/types/settingsSchema.ts b/app/shared/types/settingsSchema.ts index afbac4a7d4..c8e0bd3f9b 100644 --- a/app/shared/types/settingsSchema.ts +++ b/app/shared/types/settingsSchema.ts @@ -133,7 +133,14 @@ export const settingsSchema = { type: 'object', properties: { _id: { type: 'string' }, - semanticSearch: { type: 'boolean' }, + tocGeneration: { + type: 'object', + required: ['active', 'url'], + additionalProperties: false, + properties: { + url: { type: 'string' }, + }, + }, topicClassification: { type: 'boolean' }, favorites: { type: 'boolean' }, }, diff --git a/app/shared/types/settingsType.d.ts b/app/shared/types/settingsType.d.ts index 842b7507d3..a882056e3d 100644 --- a/app/shared/types/settingsType.d.ts +++ b/app/shared/types/settingsType.d.ts @@ -84,7 +84,9 @@ export interface Settings { links?: SettingsLinkSchema[]; features?: { _id?: string; - semanticSearch?: boolean; + tocGeneration?: { + url: string; + }; topicClassification?: boolean; favorites?: boolean; [k: string]: unknown | undefined; From 5f88a16e2a01cff5cba8eb3709594ead573461a4 Mon Sep 17 00:00:00 2001 From: Daneryl Date: Wed, 17 Feb 2021 08:39:22 -0500 Subject: [PATCH 12/31] customFilters for search endpoint --- app/api/search/documentQueryBuilder.js | 13 ++++++++++++- app/api/search/metadataAggregations.js | 3 ++- app/api/search/search.js | 2 +- app/api/search/searchSchema.ts | 11 +++++++++++ app/api/search/specs/fixtures_elastic.js | 2 ++ app/api/search/specs/search.spec.js | 17 ++++++++++++++++- 6 files changed, 44 insertions(+), 4 deletions(-) diff --git a/app/api/search/documentQueryBuilder.js b/app/api/search/documentQueryBuilder.js index 65679ce7e4..ee0631acc3 100644 --- a/app/api/search/documentQueryBuilder.js +++ b/app/api/search/documentQueryBuilder.js @@ -1,7 +1,7 @@ /* eslint-disable camelcase, max-lines */ import { preloadOptionsSearch } from 'shared/config'; -import filterToMatch, { multiselectFilter } from './metadataMatchers'; +import filterToMatch, { textFilter, multiselectFilter } from './metadataMatchers'; import { propertyToAggregation, generatedTocAggregations } from './metadataAggregations'; export default function() { @@ -231,6 +231,17 @@ export default function() { } }, + customFilters(filters = {}) { + Object.keys(filters).forEach(key => { + if (filters[key].values.length) { + addFilter({ + terms: { [key]: filters[key].values }, + }); + } + }); + return this; + }, + filterMetadata(filters = []) { filters.forEach(filter => { const match = filterToMatch(filter, filter.suggested ? 'suggestedMetadata' : 'metadata'); diff --git a/app/api/search/metadataAggregations.js b/app/api/search/metadataAggregations.js index aac071e4a0..03766ee3c8 100644 --- a/app/api/search/metadataAggregations.js +++ b/app/api/search/metadataAggregations.js @@ -137,6 +137,7 @@ const extractFilters = (baseQuery, path) => { (!match.terms || (match.terms && !match.terms[path])) && (!match.bool || !match.bool.should || + !match.bool.should[1] || !match.bool.should[1].terms || !match.bool.should[1].terms[path]) ); @@ -173,6 +174,6 @@ export const generatedTocAggregations = baseQuery => { const filters = extractFilters(baseQuery, path); const { should } = baseQuery.query.bool; const agg = aggregation(path, should, filters); - agg.terms.missing = 'false'; + // agg.terms.missing = 'false'; return agg; }; diff --git a/app/api/search/search.js b/app/api/search/search.js index 8475759233..b3d80184d3 100644 --- a/app/api/search/search.js +++ b/app/api/search/search.js @@ -607,6 +607,7 @@ const buildQuery = async (query, language, user, resources) => { const filters = processFilters(query.filters, [...allUniqueProps, ...properties]); // this is where the query filters are built queryBuilder.filterMetadata(filters); + queryBuilder.customFilters(query.customFilters); // this is where the query aggregations are built queryBuilder.aggregations(aggregations, dictionaries); @@ -626,7 +627,6 @@ const search = { queryBuilder.generatedTOCAggregations(); } - // queryBuilder.query() is the actual call return elastic .search({ body: queryBuilder.query() }) .then(response => processResponse(response, templates, dictionaries, language, query.filters)) diff --git a/app/api/search/searchSchema.ts b/app/api/search/searchSchema.ts index 9b67c6a1b1..abae216f28 100644 --- a/app/api/search/searchSchema.ts +++ b/app/api/search/searchSchema.ts @@ -4,6 +4,17 @@ export const searchSchema = { properties: { aggregateGeneratedToc: { type: 'boolean' }, filters: { type: 'object' }, + customFilters: { + type: 'object', + properties: { + generatedToc: { + type: 'object', + properties: { + values: { type: 'array', items: [{ type: 'boolean' }] }, + }, + }, + }, + }, types: { type: 'array', items: [{ type: 'string' }] }, _types: { type: 'array', items: [{ type: 'string' }] }, fields: { type: 'array', items: [{ type: 'string' }] }, diff --git a/app/api/search/specs/fixtures_elastic.js b/app/api/search/specs/fixtures_elastic.js index 86cb00f2c7..3981a52437 100644 --- a/app/api/search/specs/fixtures_elastic.js +++ b/app/api/search/specs/fixtures_elastic.js @@ -72,6 +72,7 @@ export const fixtures = { language: 'en', title: 'Batman finishes en', published: true, + generatedToc: true, user: userId, metadata: { relationship: [ @@ -153,6 +154,7 @@ export const fixtures = { language: 'es', title: 'template1 title es', published: true, + generatedToc: false, user: userId, }, { diff --git a/app/api/search/specs/search.spec.js b/app/api/search/specs/search.spec.js index e7152ea72a..8b94c670b8 100644 --- a/app/api/search/specs/search.spec.js +++ b/app/api/search/specs/search.spec.js @@ -168,7 +168,7 @@ describe('search', () => { const aggregations = response.aggregations.all.generatedToc.buckets; - expect(aggregations.find(a => a.key === 'false').filtered.doc_count).toBe(3); + expect(aggregations.find(a => a.key === 'false').filtered.doc_count).toBe(2); expect(aggregations.find(a => a.key === 'true').filtered.doc_count).toBe(1); }); @@ -778,6 +778,21 @@ describe('search', () => { }); }); + describe('customFilters', () => { + it('should filter by the values passed', async () => { + const query = { + customFilters: { + generatedToc: { + values: ['true'], + }, + }, + }; + + const { rows } = await search.search(query, 'en'); + expect(rows).toEqual([expect.objectContaining({ title: 'Batman finishes en' })]); + }); + }); + describe('autocompleteAggregations()', () => { it('should return a list of options matching by label and options related to the matching one', async () => { const query = { From 08bea1fa45873151be3c7924413703a94bc2765a Mon Sep 17 00:00:00 2001 From: Daneryl Date: Thu, 11 Feb 2021 17:20:59 -0500 Subject: [PATCH 13/31] UI toc generation components, WIP --- app/api/files/files.ts | 5 +- app/api/files/routes.ts | 2 +- app/api/files/specs/fixtures.ts | 2 - app/api/search/deprecatedRoutes.js | 4 +- app/api/search/elasticTypes.ts | 3 +- app/api/search/specs/searchSchema.spec.ts | 9 +- app/react/App/scss/modules/_toc.scss | 7 +- app/react/Attachments/components/File.tsx | 11 ++- .../components/specs/File.spec.tsx | 3 +- .../Documents/components/DocumentSidePanel.js | 15 ++- app/react/Documents/components/ShowToc.js | 7 +- app/react/Library/actions/libraryActions.js | 3 + .../actions/specs/filterActions.spec.js | 8 +- .../actions/specs/libraryActions.spec.js | 43 +++++--- app/react/Library/components/FiltersForm.js | 7 +- .../Library/components/ViewMetadataPanel.js | 12 ++- app/react/Library/helpers/libraryFilters.js | 1 + app/react/Library/helpers/requestState.js | 10 +- .../helpers/specs/libraryFilters.spec.js | 24 ++--- .../helpers/specs/resquestState.spec.js | 1 + .../tocGeneration/FilterTocGeneration.tsx | 49 +++++++++ .../tocGeneration/ReviewTocButton.tsx | 34 +++++++ .../tocGeneration/TocGeneratedLabel.tsx | 14 +++ .../ToggledFeatures/tocGeneration/actions.ts | 36 +++++++ .../ToggledFeatures/tocGeneration/index.ts | 4 + .../specs/FilterTocGeneration.spec.tsx | 68 +++++++++++++ .../specs/ReviewTocButton.spec.tsx | 37 +++++++ .../tocGeneration/specs/actions.spec.ts | 99 +++++++++++++++++++ .../tocGeneration/specs/utils.spec.ts | 19 ++++ .../ToggledFeatures/tocGeneration/utils.ts | 11 +++ app/react/Uploads/specs/UploadsRoute.spec.js | 1 + app/react/istore.d.ts | 11 ++- app/server.js | 2 +- app/shared/types/Aggregations.d.ts | 13 +++ app/shared/types/connectionType.d.ts | 3 + app/shared/types/searchParams.d.ts | 33 +++++++ .../types/searchParams.ts} | 7 +- emitSchemaTypes.js | 32 +++--- 38 files changed, 574 insertions(+), 76 deletions(-) create mode 100644 app/react/ToggledFeatures/tocGeneration/FilterTocGeneration.tsx create mode 100644 app/react/ToggledFeatures/tocGeneration/ReviewTocButton.tsx create mode 100644 app/react/ToggledFeatures/tocGeneration/TocGeneratedLabel.tsx create mode 100644 app/react/ToggledFeatures/tocGeneration/actions.ts create mode 100644 app/react/ToggledFeatures/tocGeneration/index.ts create mode 100644 app/react/ToggledFeatures/tocGeneration/specs/FilterTocGeneration.spec.tsx create mode 100644 app/react/ToggledFeatures/tocGeneration/specs/ReviewTocButton.spec.tsx create mode 100644 app/react/ToggledFeatures/tocGeneration/specs/actions.spec.ts create mode 100644 app/react/ToggledFeatures/tocGeneration/specs/utils.spec.ts create mode 100644 app/react/ToggledFeatures/tocGeneration/utils.ts create mode 100644 app/shared/types/Aggregations.d.ts create mode 100644 app/shared/types/searchParams.d.ts rename app/{api/search/searchSchema.ts => shared/types/searchParams.ts} (85%) diff --git a/app/api/files/files.ts b/app/api/files/files.ts index 9e4108e558..2667669584 100644 --- a/app/api/files/files.ts +++ b/app/api/files/files.ts @@ -32,12 +32,11 @@ export const files = { return toDeleteFiles; }, - async tocReviewed(_id: string) { + async tocReviewed(_id: string, language: string) { const savedFile = await files.save({ _id, generatedToc: false }); const sameEntityFiles = await files.get({ entity: savedFile.entity }, { generatedToc: 1 }); const [entity] = await entities.get({ sharedId: savedFile.entity, - language: savedFile.language, }); await entities.save( @@ -50,7 +49,7 @@ export const files = { false ), }, - { user: {}, language: savedFile.language } + { user: {}, language } ); return savedFile; diff --git a/app/api/files/routes.ts b/app/api/files/routes.ts index 4c5de41b3c..9e3170aea2 100644 --- a/app/api/files/routes.ts +++ b/app/api/files/routes.ts @@ -73,7 +73,7 @@ export default (app: Application) => { }), async (req, res, next) => { try { - res.json(await files.tocReviewed(req.body.fileId)); + res.json(await files.tocReviewed(req.body.fileId, req.language)); } catch (e) { next(e); } diff --git a/app/api/files/specs/fixtures.ts b/app/api/files/specs/fixtures.ts index 1bab9ad9da..94b88d6f77 100644 --- a/app/api/files/specs/fixtures.ts +++ b/app/api/files/specs/fixtures.ts @@ -13,7 +13,6 @@ const fixtures: DBFixture = { { _id: uploadId, entity: 'sharedId1', - language: 'es', generatedToc: true, originalname: 'upload1', filename: fileName1, @@ -22,7 +21,6 @@ const fixtures: DBFixture = { { _id: uploadId2, generatedToc: true, - language: 'es', entity: 'sharedId1', filename: 'fileNotInDisk', }, diff --git a/app/api/search/deprecatedRoutes.js b/app/api/search/deprecatedRoutes.js index 9526181f88..e5d369dbeb 100644 --- a/app/api/search/deprecatedRoutes.js +++ b/app/api/search/deprecatedRoutes.js @@ -1,6 +1,6 @@ import Joi from 'joi'; import entities from 'api/entities'; -import { searchSchema } from 'api/search/searchSchema'; +import { searchParamsSchema } from 'shared/types/searchParams'; import { search } from './search'; import { validation, parseQuery } from '../utils'; @@ -25,7 +25,7 @@ export default app => { app.get( '/api/search', parseQuery, - validation.validateRequest(searchSchema), + validation.validateRequest(searchParamsSchema), (req, res, next) => { const action = req.query.geolocation ? 'searchGeolocations' : 'search'; diff --git a/app/api/search/elasticTypes.ts b/app/api/search/elasticTypes.ts index efee29aab9..b5351ca1fa 100644 --- a/app/api/search/elasticTypes.ts +++ b/app/api/search/elasticTypes.ts @@ -1,5 +1,6 @@ import { RequestParams } from '@elastic/elasticsearch'; import { RequestBody } from '@elastic/elasticsearch/lib/Transport'; +import { Aggregations } from 'shared/types/Aggregations.d.ts'; interface ShardsResponse { total: number; @@ -42,7 +43,7 @@ export interface SearchResponse { sort?: string[]; }>; }; - aggregations?: any; + aggregations?: Aggregations; } export type IndicesDelete = Omit; diff --git a/app/api/search/specs/searchSchema.spec.ts b/app/api/search/specs/searchSchema.spec.ts index 33722cd602..8a23458c1b 100644 --- a/app/api/search/specs/searchSchema.spec.ts +++ b/app/api/search/specs/searchSchema.spec.ts @@ -1,6 +1,6 @@ import { ValidationError } from 'ajv'; import { validation } from 'api/utils'; -import { searchSchema } from '../searchSchema'; +import { searchParamsSchema } from 'shared/types/searchParams'; describe('search schema', () => { const validQuery = { @@ -27,11 +27,12 @@ describe('search schema', () => { it('should not have validation errors for valid search', async () => { const validSearch = { validQuery }; - await validation.validateRequest(searchSchema)(validSearch, null, expectValidSchema); + await validation.validateRequest(searchParamsSchema)(validSearch, null, expectValidSchema); }); + it('should support a number as a search term', async () => { const validSearch = { query: { ...validQuery, searchTerm: 3 } }; - await validation.validateRequest(searchSchema)(validSearch, null, expectValidSchema); + await validation.validateRequest(searchParamsSchema)(validSearch, null, expectValidSchema); }); }); @@ -42,7 +43,7 @@ describe('search schema', () => { async function testInvalidProperty(invalidProperty: any) { const invalidSearch = { query: { ...validQuery, ...invalidProperty } }; - await validation.validateRequest(searchSchema)(invalidSearch, null, expectInvalidSchema); + await validation.validateRequest(searchParamsSchema)(invalidSearch, null, expectInvalidSchema); } it('should be invalid if allAgregations is not a boolean value', async () => { diff --git a/app/react/App/scss/modules/_toc.scss b/app/react/App/scss/modules/_toc.scss index 3f195ab5ed..9f06fca32e 100644 --- a/app/react/App/scss/modules/_toc.scss +++ b/app/react/App/scss/modules/_toc.scss @@ -1,5 +1,10 @@ .toc { - padding: 15px; + padding: 0px 15px 15px 15px; +} + +div.tocHeader { + padding-left: 15px; + border-bottom: 1px solid #f4f4f4; } .toc-view{ diff --git a/app/react/Attachments/components/File.tsx b/app/react/Attachments/components/File.tsx index 21fd3c5cb1..e18dd4bc0a 100644 --- a/app/react/Attachments/components/File.tsx +++ b/app/react/Attachments/components/File.tsx @@ -10,6 +10,7 @@ import { APIURL } from 'app/config.js'; import { LocalForm, Control } from 'react-redux-form'; import { updateFile, deleteFile } from 'app/Attachments/actions/actions'; import { wrapDispatch } from 'app/Multireducer'; +import { TocGeneratedLabel } from 'app/ToggledFeatures/tocGeneration'; import { NeedAuthorization } from 'app/Auth'; import { EntitySchema } from 'shared/types/entityType'; import { ViewDocumentLink } from './ViewDocumentLink'; @@ -102,7 +103,10 @@ export class File extends Component {
{language ? transformLanguage(language) || '' : ''} -
{' '} +
+ + ML TOC + { const mapDispatchToProps = (dispatch: Dispatch<{}>, props: FileProps) => bindActionCreators({ updateFile, deleteFile }, wrapDispatch(dispatch, props.storeKey)); -export const ConnectedFile = connect(null, mapDispatchToProps)(File); +export const ConnectedFile = connect( + null, + mapDispatchToProps +)(File); diff --git a/app/react/Attachments/components/specs/File.spec.tsx b/app/react/Attachments/components/specs/File.spec.tsx index 997ef2a635..95ca643fa4 100644 --- a/app/react/Attachments/components/specs/File.spec.tsx +++ b/app/react/Attachments/components/specs/File.spec.tsx @@ -2,8 +2,8 @@ import React from 'react'; import { shallow, ShallowWrapper } from 'enzyme'; import { LocalForm } from 'react-redux-form'; import { FileType } from 'shared/types/fileType'; -import { File, FileProps } from '../File'; import { Translate } from 'app/I18N'; +import { File, FileProps } from '../File'; describe('file', () => { let component: ShallowWrapper; @@ -32,6 +32,7 @@ describe('file', () => { }); const render = () => { + //eslint-disable-next-line react/jsx-props-no-spreading component = shallow(, { context }); }; diff --git a/app/react/Documents/components/DocumentSidePanel.js b/app/react/Documents/components/DocumentSidePanel.js index d476d6d047..9d9cf7d75e 100644 --- a/app/react/Documents/components/DocumentSidePanel.js +++ b/app/react/Documents/components/DocumentSidePanel.js @@ -16,6 +16,8 @@ import ShowIf from 'app/App/ShowIf'; import SidePanel from 'app/Layout/SidePanel'; import DocumentSemanticSearchResults from 'app/SemanticSearch/components/DocumentResults'; import { CopyFromEntity } from 'app/Metadata/components/CopyFromEntity'; +import { FeatureToggle } from 'app/components/Elements/FeatureToggle'; +import { TocGeneratedLabel, ReviewTocButton } from 'app/ToggledFeatures/tocGeneration'; import { Icon } from 'UI'; import * as viewerModule from 'app/Viewer'; @@ -321,11 +323,14 @@ export class DocumentSidePanel extends Component {
+ + Mark as Reviewed +
@@ -340,6 +345,14 @@ export class DocumentSidePanel extends Component { /> +
+

+ Table of contents +

+ + auto-created ⓘ + +
{ let filtersState; beforeEach(() => { - libraryFilters = [ - { name: 'author', filter: true }, - { name: 'country', filter: true }, - ]; + libraryFilters = [{ name: 'author', filter: true }, { name: 'country', filter: true }]; search = { searchTerm: '', filters: { author: 'RR Martin', country: '' } }; filtersState = { documentTypes, @@ -36,6 +33,9 @@ describe('filterActions', () => { }; store = { + settings: { + collection: Immutable.Map({}), + }, library: { filters: Immutable.fromJS(filtersState), search, diff --git a/app/react/Library/actions/specs/libraryActions.spec.js b/app/react/Library/actions/specs/libraryActions.spec.js index 71d771da5a..d1d94de2cf 100644 --- a/app/react/Library/actions/specs/libraryActions.spec.js +++ b/app/react/Library/actions/specs/libraryActions.spec.js @@ -48,9 +48,12 @@ describe('libraryActions', () => { beforeEach(() => { dispatch = jasmine.createSpy('dispatch'); - getState = jasmine - .createSpy('getState') - .and.returnValue({ library: { filters: Immutable.fromJS(filters), search: {} } }); + getState = jasmine.createSpy('getState').and.returnValue({ + settings: { + collection: Immutable.Map({}), + }, + library: { filters: Immutable.fromJS(filters), search: {} }, + }); }); it('should dispatch a SET_LIBRARY_TEMPLATES action ', () => { @@ -178,15 +181,24 @@ describe('libraryActions', () => { { name: 'relationshipfilter', type: 'relationshipfilter', - filters: [ - { name: 'status', type: 'select' }, - { name: 'empty', type: 'date' }, - ], + filters: [{ name: 'status', type: 'select' }, { name: 'empty', type: 'date' }], }, ], documentTypes: ['decision'], }; - store = { library: { filters: Immutable.fromJS(state), search: { searchTerm: 'batman' } } }; + store = { + settings: { + collection: Immutable.Map({}), + }, + library: { + filters: Immutable.fromJS(state), + search: { + searchTerm: 'batman', + customFilters: { property: { values: ['value'] } }, + filters: {}, + }, + }, + }; spyOn(browserHistory, 'getCurrentLocation').and.returnValue({ pathname: '/library', query: { view: 'chart' }, @@ -261,6 +273,16 @@ describe('libraryActions', () => { ); }); + it('should use customFilters from the current search on the store', () => { + const limit = 60; + spyOn(browserHistory, 'push'); + actions.searchDocuments({}, storeKey, limit)(dispatch, getState); + + expect(browserHistory.push).toHaveBeenCalledWith( + "/library/?view=chart&q=(customFilters:(property:(values:!(value))),filters:(),from:0,limit:60,searchTerm:'batman',sort:_score,types:!(decision))" //eslint-disable-line + ); + }); + it('should set the storeKey selectedSorting if user has selected a custom sorting', () => { const expectedDispatch = { type: 'library.selectedSorting/SET', @@ -365,10 +387,7 @@ describe('libraryActions', () => { }, { type: types.UPDATE_DOCUMENTS, - docs: [ - { sharedId: '1', metadataResponse }, - { sharedId: '2', metadataResponse }, - ], + docs: [{ sharedId: '1', metadataResponse }, { sharedId: '2', metadataResponse }], }, ]; const store = mockStore({}); diff --git a/app/react/Library/components/FiltersForm.js b/app/react/Library/components/FiltersForm.js index 6f773961c4..a06985851a 100644 --- a/app/react/Library/components/FiltersForm.js +++ b/app/react/Library/components/FiltersForm.js @@ -11,6 +11,7 @@ import { t } from 'app/I18N'; import { wrapDispatch } from 'app/Multireducer'; import debounce from 'app/utils/debounce'; import libraryHelper from 'app/Library/helpers/libraryFilters'; +import { FilterTocGeneration } from 'app/ToggledFeatures/tocGeneration'; import Filters from './FiltersFromProperties'; @@ -97,6 +98,7 @@ export class FiltersForm extends Component { translationContext={translationContext} storeKey={this.props.storeKey} /> + ); @@ -126,4 +128,7 @@ function mapDispatchToProps(dispatch, props) { return bindActionCreators({ searchDocuments }, wrapDispatch(dispatch, props.storeKey)); } -export default connect(mapStateToProps, mapDispatchToProps)(FiltersForm); +export default connect( + mapStateToProps, + mapDispatchToProps +)(FiltersForm); diff --git a/app/react/Library/components/ViewMetadataPanel.js b/app/react/Library/components/ViewMetadataPanel.js index 6f9ec1138f..f3487a6278 100644 --- a/app/react/Library/components/ViewMetadataPanel.js +++ b/app/react/Library/components/ViewMetadataPanel.js @@ -9,6 +9,7 @@ import { actions } from 'app/Metadata'; import { deleteDocument, searchSnippets } from 'app/Library/actions/libraryActions'; import { deleteEntity } from 'app/Entities/actions/actions'; import { wrapDispatch } from 'app/Multireducer'; +import { entityDefaultDocument } from 'shared/entityDefaultDocument'; import modals from 'app/Modals'; import { @@ -22,9 +23,18 @@ const getTemplates = state => state.templates; const mapStateToProps = (state, props) => { const library = state[props.storeKey]; + const doc = library.ui.get('selectedDocuments').first() || Immutable.fromJS({ documents: [] }); + const defaultLanguage = state.settings.collection.get('languages').find(l => l.get('defautl')); + const file = entityDefaultDocument( + doc.get('documents').toJS(), + doc.get('language'), + defaultLanguage + ); + return { open: library.ui.get('selectedDocuments').size === 1, - doc: library.ui.get('selectedDocuments').first() || Immutable.fromJS({}), + doc, + file, references: library.sidepanel.references, tab: library.sidepanel.tab, docBeingEdited: !!Object.keys(library.sidepanel.metadata).length, diff --git a/app/react/Library/helpers/libraryFilters.js b/app/react/Library/helpers/libraryFilters.js index 79695577b3..9f39daa6fb 100644 --- a/app/react/Library/helpers/libraryFilters.js +++ b/app/react/Library/helpers/libraryFilters.js @@ -56,6 +56,7 @@ function URLQueryToState(query, templates, _thesauris, _relationTypes, forcedPro search: { searchTerm, filters, + customFilters: query.customFilters, sort, order, userSelectedSorting, diff --git a/app/react/Library/helpers/requestState.js b/app/react/Library/helpers/requestState.js index dc66ccd084..144f01f5b9 100644 --- a/app/react/Library/helpers/requestState.js +++ b/app/react/Library/helpers/requestState.js @@ -5,6 +5,7 @@ import prioritySortingCriteria from 'app/utils/prioritySortingCriteria'; import rison from 'rison-node'; import { getThesaurusPropertyNames } from 'shared/commonTopicClassification'; import { setTableViewColumns } from 'app/Library/actions/libraryActions'; +import { tocGenerationUtils } from 'app/ToggledFeatures/tocGeneration'; import { wrapDispatch } from 'app/Multireducer'; import { getTableColumns } from './tableColumns'; import setReduxState from './setReduxState.js'; @@ -36,12 +37,17 @@ export function processQuery(params, globalResources, key) { } const { userSelectedSorting, ...sanitizedQuery } = query; - return sanitizedQuery; + return tocGenerationUtils.aggregations( + sanitizedQuery, + globalResources.settings.collection.toJS() + ); } export default function requestState(request, globalResources, calculateTableColumns = false) { const docsQuery = processQuery(request.data, globalResources, 'library'); - const documentsRequest = request.set(docsQuery); + const documentsRequest = request.set( + tocGenerationUtils.aggregations(docsQuery, globalResources.settings.collection.toJS()) + ); const markersRequest = request.set({ ...docsQuery, geolocation: true }); return Promise.all([api.search(documentsRequest), api.search(markersRequest)]).then( diff --git a/app/react/Library/helpers/specs/libraryFilters.spec.js b/app/react/Library/helpers/specs/libraryFilters.spec.js index fa91a8bc15..72225d6450 100644 --- a/app/react/Library/helpers/specs/libraryFilters.spec.js +++ b/app/react/Library/helpers/specs/libraryFilters.spec.js @@ -33,26 +33,17 @@ describe('library helper', () => { const thesauris = [ { _id: 'abc1', - values: [ - { id: 1, value: 'value1' }, - { id: 2, value: 'value2' }, - ], + values: [{ id: 1, value: 'value1' }, { id: 2, value: 'value2' }], }, { _id: 'thesauri2', type: 'template', - values: [ - { id: 3, value: 'value3' }, - { id: 4, value: 'value4' }, - ], + values: [{ id: 3, value: 'value3' }, { id: 4, value: 'value4' }], }, { _id: 'thesauri3', type: 'template', - values: [ - { id: 5, value: 'value5' }, - { id: 6, value: 'value6' }, - ], + values: [{ id: 5, value: 'value5' }, { id: 6, value: 'value6' }], }, ]; @@ -83,10 +74,14 @@ describe('library helper', () => { sort: 'sort', types: ['3'], filters: { country: 'countryValue', rich: 'search' }, + customFilters: { + property: { values: ['value'] }, + }, }; const state = libraryHelper.URLQueryToState(query, templates); expect(state.properties.length).toBe(1); + expect(state.search.customFilters).toEqual(query.customFilters); expect(state.search.filters.country).toBe('countryValue'); expect(state.search.filters.rich).toBe('search'); expect(state.search.searchTerm).toBe('searchTerm'); @@ -146,10 +141,7 @@ describe('library helper', () => { filter: true, type: 'select', content: 'abc1', - options: [ - { id: 1, value: 'value1' }, - { id: 2, value: 'value2' }, - ], + options: [{ id: 1, value: 'value1' }, { id: 2, value: 'value2' }], }, { name: 'date', filter: true, type: 'text' }, ]; diff --git a/app/react/Library/helpers/specs/resquestState.spec.js b/app/react/Library/helpers/specs/resquestState.spec.js index 7df7599f3c..6377d1f40c 100644 --- a/app/react/Library/helpers/specs/resquestState.spec.js +++ b/app/react/Library/helpers/specs/resquestState.spec.js @@ -30,6 +30,7 @@ describe('static requestState()', () => { }; const globalResources = { templates: Immutable.fromJS(templates), + settings: { collection: Immutable.fromJS({ features: {} }) }, thesauris: Immutable.fromJS(thesauris), relationTypes: Immutable.fromJS(relationTypes), }; diff --git a/app/react/ToggledFeatures/tocGeneration/FilterTocGeneration.tsx b/app/react/ToggledFeatures/tocGeneration/FilterTocGeneration.tsx new file mode 100644 index 0000000000..2afe82a2ff --- /dev/null +++ b/app/react/ToggledFeatures/tocGeneration/FilterTocGeneration.tsx @@ -0,0 +1,49 @@ +import React from 'react'; +import { Aggregations } from 'shared/types/Aggregations.d.ts'; +import { FeatureToggle } from 'app/components/Elements/FeatureToggle'; +import SelectFilter from 'app/Library/components/SelectFilter'; +import FormGroup from 'app/DocumentForm/components/FormGroup'; +import { t } from 'app/I18N'; +import { NeedAuthorization } from 'app/Auth'; + +export interface FilterTocGenerationProps { + onChange: () => void; + aggregations: Aggregations; +} + +const filteredAggregation = (aggregations: Aggregations, key: string) => { + const bucket = (aggregations?.all?.generatedToc?.buckets || []).find(a => a.key === key) || { + filtered: { doc_count: 0 }, + }; + return bucket.filtered.doc_count; +}; + +const options = (aggregations: Aggregations = { all: {} }) => [ + { + label: 'Automatically generated', + value: true, + results: filteredAggregation(aggregations, 'true'), + }, + { + label: 'Reviewed', + value: false, + results: filteredAggregation(aggregations, 'false'), + }, +]; + +export const FilterTocGeneration = ({ onChange, aggregations }: FilterTocGenerationProps) => ( + + + + + + + +); diff --git a/app/react/ToggledFeatures/tocGeneration/ReviewTocButton.tsx b/app/react/ToggledFeatures/tocGeneration/ReviewTocButton.tsx new file mode 100644 index 0000000000..8c5878b62c --- /dev/null +++ b/app/react/ToggledFeatures/tocGeneration/ReviewTocButton.tsx @@ -0,0 +1,34 @@ +import React from 'react'; +import { Icon } from 'UI'; +import { FeatureToggle } from 'app/components/Elements/FeatureToggle'; +import { connect, ConnectedProps } from 'react-redux'; +import { bindActionCreators, Dispatch } from 'redux'; +import { ClientFile } from 'app/istore'; +import { tocGenerationActions } from './actions'; + +export interface ReviewTocButtonProps { + file: ClientFile; + children: JSX.Element | string; +} + +const mapDispatchToProps = (dispatch: Dispatch<{}>) => + bindActionCreators({ onClick: tocGenerationActions.reviewToc }, dispatch); + +const connector = connect(null, mapDispatchToProps); + +type MappedProps = ConnectedProps; +type ComponentProps = ReviewTocButtonProps & MappedProps; + +const ReviewTocButton = ({ file, onClick, children }: ComponentProps) => ( + + {file.generatedToc && ( + + )} + +); + +const container = connector(ReviewTocButton); +export { container as ReviewTocButton }; diff --git a/app/react/ToggledFeatures/tocGeneration/TocGeneratedLabel.tsx b/app/react/ToggledFeatures/tocGeneration/TocGeneratedLabel.tsx new file mode 100644 index 0000000000..5ef7e63b17 --- /dev/null +++ b/app/react/ToggledFeatures/tocGeneration/TocGeneratedLabel.tsx @@ -0,0 +1,14 @@ +import React from 'react'; +import { FeatureToggle } from 'app/components/Elements/FeatureToggle'; +import { FileType } from 'shared/types/fileType'; + +export interface TocGeneratedLabelProps { + file: FileType; + children: JSX.Element | string; +} + +export const TocGeneratedLabel = ({ file, children }: TocGeneratedLabelProps) => ( + + {file.generatedToc && {children}} + +); diff --git a/app/react/ToggledFeatures/tocGeneration/actions.ts b/app/react/ToggledFeatures/tocGeneration/actions.ts new file mode 100644 index 0000000000..8ab8f5b5fc --- /dev/null +++ b/app/react/ToggledFeatures/tocGeneration/actions.ts @@ -0,0 +1,36 @@ +import { actions } from 'app/BasicReducer/reducer'; +import { actions as formActions } from 'react-redux-form'; +import { RequestParams } from 'app/utils/RequestParams'; +import api from 'app/utils/api'; +import { notificationActions } from 'app/Notifications'; +import { IStore } from 'app/istore'; +import { Dispatch } from 'redux'; +import { ensure } from 'shared/tsUtils'; +import { FileType } from 'shared/types/fileType'; + +const tocGenerationActions = { + reviewToc(fileId: string) { + return async (dispatch: Dispatch, getState: () => IStore) => { + const currentDoc = getState().documentViewer.doc.toJS(); + dispatch(formActions.reset('documentViewer.sidepanel.metadata')); + + const updatedFile = (await api.post('files/tocReviewed', new RequestParams({ fileId }))).json; + const doc = { + ...currentDoc, + defaultDoc: updatedFile, + documents: ensure(currentDoc.documents).map(d => { + if (d._id === updatedFile._id) { + return updatedFile; + } + return d; + }), + }; + + dispatch(notificationActions.notify('Document updated', 'success')); + dispatch(formActions.reset('documentViewer.sidepanel.metadata')); + dispatch(actions.set('viewer/doc', doc)); + }; + }, +}; + +export { tocGenerationActions }; diff --git a/app/react/ToggledFeatures/tocGeneration/index.ts b/app/react/ToggledFeatures/tocGeneration/index.ts new file mode 100644 index 0000000000..7785be5636 --- /dev/null +++ b/app/react/ToggledFeatures/tocGeneration/index.ts @@ -0,0 +1,4 @@ +export { TocGeneratedLabel } from './TocGeneratedLabel'; +export { ReviewTocButton } from './ReviewTocButton'; +export { FilterTocGeneration } from './FilterTocGeneration'; +export { tocGenerationUtils } from './utils'; diff --git a/app/react/ToggledFeatures/tocGeneration/specs/FilterTocGeneration.spec.tsx b/app/react/ToggledFeatures/tocGeneration/specs/FilterTocGeneration.spec.tsx new file mode 100644 index 0000000000..a2532c54ba --- /dev/null +++ b/app/react/ToggledFeatures/tocGeneration/specs/FilterTocGeneration.spec.tsx @@ -0,0 +1,68 @@ +import React from 'react'; +import { shallow, ShallowWrapper } from 'enzyme'; +import SelectFilter from 'app/Library/components/SelectFilter'; +import { FilterTocGeneration } from '../FilterTocGeneration'; +import { Aggregations } from 'shared/types/Aggregations'; + +describe('ReviewTocButton', () => { + let component: ShallowWrapper; + const aggregations = { + all: { + generatedToc: { + buckets: [ + { key: 'false', filtered: { doc_count: 2 } }, + { key: 'true', filtered: { doc_count: 5 } }, + ], + }, + }, + }; + + const render = (aggs: Aggregations = aggregations) => { + component = shallow( {}} aggregations={aggs} />); + }; + + it('should render nothing if file generatedToc is false', () => { + render(); + const options = component.find(SelectFilter).prop('options'); + expect(options).toEqual([ + expect.objectContaining({ value: true, results: 5 }), + expect.objectContaining({ value: false, results: 2 }), + ]); + }); + + describe('when there is bucket missing', () => { + it('should not fail (render blank state)', () => { + render({ + all: { + generatedToc: { + buckets: [{ key: 'false', filtered: { doc_count: 2 } }], + }, + }, + }); + const options = component.find(SelectFilter).prop('options'); + expect(options).toEqual([ + expect.objectContaining({ value: true, results: 0 }), + expect.objectContaining({ value: false, results: 2 }), + ]); + }); + }); + + describe('when aggregations are not defined/complete', () => { + it('should not fail (render blank state)', () => { + //@ts-ignore + render({}); + let options = component.find(SelectFilter).prop('options'); + expect(options).toEqual([ + expect.objectContaining({ value: true, results: 0 }), + expect.objectContaining({ value: false, results: 0 }), + ]); + + render({ all: {} }); + options = component.find(SelectFilter).prop('options'); + expect(options).toEqual([ + expect.objectContaining({ value: true, results: 0 }), + expect.objectContaining({ value: false, results: 0 }), + ]); + }); + }); +}); diff --git a/app/react/ToggledFeatures/tocGeneration/specs/ReviewTocButton.spec.tsx b/app/react/ToggledFeatures/tocGeneration/specs/ReviewTocButton.spec.tsx new file mode 100644 index 0000000000..43b032cc8d --- /dev/null +++ b/app/react/ToggledFeatures/tocGeneration/specs/ReviewTocButton.spec.tsx @@ -0,0 +1,37 @@ +import React from 'react'; +import { shallow, ShallowWrapper } from 'enzyme'; +import configureStore, { MockStore, MockStoreCreator } from 'redux-mock-store'; +import { Provider } from 'react-redux'; +import { ClientFile } from 'app/istore'; +import { ReviewTocButton } from '../ReviewTocButton'; + +describe('ReviewTocButton', () => { + let component: ShallowWrapper; + + const mockStoreCreator: MockStoreCreator = configureStore([]); + const render = (file: Partial) => { + const store: MockStore = mockStoreCreator({}); + component = shallow( + + + test + + + ) + .dive() + .dive(); + }; + + it('should render nothing if file generatedToc is false', () => { + render({ generatedToc: false }); + expect(component.find('button').length).toEqual(0); + + render({}); + expect(component.find('button').length).toEqual(0); + }); + + it('should render when generatedToc is true', () => { + render({ generatedToc: true }); + expect(component.find('button').length).toEqual(1); + }); +}); diff --git a/app/react/ToggledFeatures/tocGeneration/specs/actions.spec.ts b/app/react/ToggledFeatures/tocGeneration/specs/actions.spec.ts new file mode 100644 index 0000000000..8950ca6b73 --- /dev/null +++ b/app/react/ToggledFeatures/tocGeneration/specs/actions.spec.ts @@ -0,0 +1,99 @@ +import api from 'app/utils/api'; +import backend from 'fetch-mock'; +import * as notificationsTypes from 'app/Notifications/actions/actionTypes'; +import { actions as relationshipActions } from 'app/Relationships'; +import { APIURL } from 'app/config'; +import configureMockStore from 'redux-mock-store'; +import thunk from 'redux-thunk'; +import Immutable from 'immutable'; +import { mockID } from 'shared/uniqueID.js'; +import { ClientEntitySchema, IStore } from 'app/istore'; +import { tocGenerationActions } from '../actions'; + +const middlewares = [thunk]; +const mockStore = configureMockStore(middlewares); + +describe('reviewToc', () => { + it('should store the document with the response of reviewToc', done => { + mockID(); + const fileId = 'fileId'; + + backend.post(`${APIURL}files/tocReviewed`, { + body: JSON.stringify({ _id: fileId, generatedToc: false }), + }); + + spyOn(relationshipActions, 'reloadRelationships').and.returnValue({ + type: 'reloadRelationships', + }); + + const doc: ClientEntitySchema = { + name: 'doc', + _id: 'id', + sharedId: 'sharedId', + defaultDoc: { + _id: fileId, + generatedToc: true, + }, + documents: [ + { + _id: fileId, + generatedToc: true, + }, + ], + }; + + const updatedEntity = { + name: 'doc', + _id: 'id', + sharedId: 'sharedId', + defaultDoc: { + _id: fileId, + generatedToc: false, + }, + documents: [ + { + _id: fileId, + generatedToc: false, + }, + ], + }; + + const expectedActions = [ + { type: 'rrf/reset', model: 'documentViewer.sidepanel.metadata' }, + { + type: notificationsTypes.NOTIFY, + notification: { message: 'Document updated', type: 'success', id: 'unique_id' }, + }, + { type: 'rrf/reset', model: 'documentViewer.sidepanel.metadata' }, + { type: 'viewer/doc/SET', value: updatedEntity }, + ]; + + const store = mockStore({ + documentViewer: { + doc: Immutable.fromJS(doc), + references: Immutable.fromJS([]), + targetDocReferences: Immutable.fromJS([]), + targetDoc: Immutable.fromJS(doc), + uiState: Immutable.fromJS({}), + }, + }); + + spyOn(api, 'post').and.callThrough(); + store + //fot this to be properly typed, redux, redux-thunk need to be updated (and probably others), + //producing hundreds of type errors + //@ts-ignore + .dispatch(tocGenerationActions.reviewToc(fileId)) + .then(() => { + expect(api.post).toHaveBeenCalledWith('files/tocReviewed', { + data: { + fileId: 'fileId', + }, + headers: {}, + }); + expect(store.getActions()).toEqual(expectedActions); + }) + .then(done) + .catch(done.fail); + }); +}); diff --git a/app/react/ToggledFeatures/tocGeneration/specs/utils.spec.ts b/app/react/ToggledFeatures/tocGeneration/specs/utils.spec.ts new file mode 100644 index 0000000000..bfdcd09eb5 --- /dev/null +++ b/app/react/ToggledFeatures/tocGeneration/specs/utils.spec.ts @@ -0,0 +1,19 @@ +import Immutable from 'immutable'; +import { processQuery } from 'app/Library/helpers/requestState'; + +describe('Library/Uploads processQuery()', () => { + it('should add aggregateGeneratedToc if feature activated', () => { + const params = { q: '(order:desc,sort:creationDate)' }; + const globalResources = { + settings: { + collection: Immutable.fromJS({ features: { tocGeneration: {} } }), + }, + }; + const query = processQuery(params, globalResources, 'library'); + expect(query).toEqual({ + order: 'desc', + sort: 'creationDate', + aggregateGeneratedToc: true, + }); + }); +}); diff --git a/app/react/ToggledFeatures/tocGeneration/utils.ts b/app/react/ToggledFeatures/tocGeneration/utils.ts new file mode 100644 index 0000000000..42cef93ce9 --- /dev/null +++ b/app/react/ToggledFeatures/tocGeneration/utils.ts @@ -0,0 +1,11 @@ +import { Settings } from 'shared/types/settingsType'; +import { SearchParams } from 'shared/types/searchParams.d.ts'; + +export const tocGenerationUtils = { + aggregations(params: SearchParams, settings: Settings) { + return { + ...params, + ...(settings?.features?.tocGeneration ? { aggregateGeneratedToc: true } : {}), + }; + }, +}; diff --git a/app/react/Uploads/specs/UploadsRoute.spec.js b/app/react/Uploads/specs/UploadsRoute.spec.js index 18889e06cb..644c98994d 100644 --- a/app/react/Uploads/specs/UploadsRoute.spec.js +++ b/app/react/Uploads/specs/UploadsRoute.spec.js @@ -26,6 +26,7 @@ describe('UploadsRoute', () => { ]; const globalResources = { templates: Immutable(templates), + settings: { collection: Immutable({ features: {} }) }, thesauris: Immutable([]), relationTypes: Immutable([]), }; diff --git a/app/react/istore.d.ts b/app/react/istore.d.ts index 39366d8aba..955b58df2b 100644 --- a/app/react/istore.d.ts +++ b/app/react/istore.d.ts @@ -8,6 +8,7 @@ import { TemplateSchema } from 'shared/types/templateType'; import { EntitySchema } from 'shared/types/entityType'; import { UserGroupSchema } from 'shared/types/userGroupType'; import { ConnectionSchema } from 'shared/types/connectionType'; +import { FileType } from 'shared/types/fileType'; export interface TasksState { SyncState?: TaskStatus; @@ -83,8 +84,12 @@ interface ClientTemplateSchema extends TemplateSchema { _id: string; } +export interface ClientFile extends FileType { + _id: string; +} + export interface ClientEntitySchema extends EntitySchema { - documents?: []; + documents?: ClientFile[]; } export interface IStore { @@ -110,8 +115,8 @@ export interface IStore { documentViewer: { references: IImmutable; targetDocReferences: IImmutable; - doc: IImmutable; - targetDoc: IImmutable; + doc: IImmutable; + targetDoc: IImmutable; uiState: IImmutable<{ activeReference: string; }>; diff --git a/app/server.js b/app/server.js index 1cc383b1bb..1befa11243 100644 --- a/app/server.js +++ b/app/server.js @@ -119,7 +119,7 @@ DB.connect(config.DBHOST, dbAuth).then(async () => { const { evidencesVault, features } = await settings.get(); if (evidencesVault && evidencesVault.token && evidencesVault.template) { - console.info('==> 📥 evidences vault config detected, started sync ....'); + console.info('==> 📥 evidences vault config detected, started sync ....'); repeater.start( () => vaultSync.sync(evidencesVault.token, evidencesVault.template), 10000 diff --git a/app/shared/types/Aggregations.d.ts b/app/shared/types/Aggregations.d.ts new file mode 100644 index 0000000000..e1c059a581 --- /dev/null +++ b/app/shared/types/Aggregations.d.ts @@ -0,0 +1,13 @@ +export interface Aggregations { + all: { + [key: string]: { + buckets: Array<{ + key: string; + filtered: { + // eslint-disable-next-line camelcase + doc_count: number; + }; + }>; + }; + }; +} diff --git a/app/shared/types/connectionType.d.ts b/app/shared/types/connectionType.d.ts index 0bcd15b543..1a026f5731 100644 --- a/app/shared/types/connectionType.d.ts +++ b/app/shared/types/connectionType.d.ts @@ -19,6 +19,7 @@ export interface ConnectionSchema { title?: string; template?: ObjectIdSchema; published?: boolean; + generatedToc?: boolean; icon?: { _id?: string | null; label?: string; @@ -42,6 +43,7 @@ export interface ConnectionSchema { | null | string | number + | boolean | { label?: string | null; url?: string | null; @@ -75,6 +77,7 @@ export interface ConnectionSchema { | null | string | number + | boolean | { label?: string | null; url?: string | null; diff --git a/app/shared/types/searchParams.d.ts b/app/shared/types/searchParams.d.ts new file mode 100644 index 0000000000..53c1826b95 --- /dev/null +++ b/app/shared/types/searchParams.d.ts @@ -0,0 +1,33 @@ +/* eslint-disable */ +/**AUTO-GENERATED. RUN yarn emit-types to update.*/ + +export interface SearchParams { + query?: { + aggregateGeneratedToc?: boolean; + filters?: { + [k: string]: unknown | undefined; + }; + customFilters?: { + generatedToc?: { + values?: [] | [string]; + }; + }; + types?: [] | [string]; + _types?: [] | [string]; + fields?: [] | [string]; + allAggregations?: boolean; + aggregations?: string; + order?: 'asc' | 'desc'; + sort?: string; + limit?: number; + from?: number; + searchTerm?: string | number; + includeUnpublished?: boolean; + userSelectedSorting?: boolean; + treatAs?: string; + unpublished?: boolean; + select?: [] | [string]; + geolocation?: boolean; + }; + [k: string]: unknown | undefined; +} diff --git a/app/api/search/searchSchema.ts b/app/shared/types/searchParams.ts similarity index 85% rename from app/api/search/searchSchema.ts rename to app/shared/types/searchParams.ts index abae216f28..38be2b30da 100644 --- a/app/api/search/searchSchema.ts +++ b/app/shared/types/searchParams.ts @@ -1,14 +1,19 @@ -export const searchSchema = { +export const emitSchemaTypes = true; +export const searchParamsSchema = { + title: 'searchParams', properties: { query: { + additionalProperties: false, properties: { aggregateGeneratedToc: { type: 'boolean' }, filters: { type: 'object' }, customFilters: { + additionalProperties: false, type: 'object', properties: { generatedToc: { type: 'object', + additionalProperties: false, properties: { values: { type: 'array', items: [{ type: 'boolean' }] }, }, diff --git a/emitSchemaTypes.js b/emitSchemaTypes.js index 9f0c2df6fe..4b626ea540 100644 --- a/emitSchemaTypes.js +++ b/emitSchemaTypes.js @@ -63,28 +63,30 @@ const writeTypeFile = (file, commonImport, snippets) => { const emitSchemaTypes = async file => { try { - if (!file.match(/Schema/) || file.match(/spec/)) { + if (file.match(/spec/)) { return; } - const schemas = require(`./${file}`); + if (file.match(/shared\/types/) || file.match(/Schema/)) { + const schemas = require(`./${file}`); - if (!schemas.emitSchemaTypes) { - return; - } + if (!schemas.emitSchemaTypes) { + return; + } - const snippets = await Promise.all( - Object.entries(schemas).map(([name, schema]) => { - if (!name.match(/Schema$/)) { - return ''; - } - return compile(schema, schema.title || firstUp(name), opts); - }) - ); + const snippets = await Promise.all( + Object.entries(schemas).map(([name, schema]) => { + if (!name.match(/Schema$/)) { + return ''; + } + return compile(schema, schema.title || firstUp(name), opts); + }) + ); - const contents = fs.readFileSync(file).toString(); + const contents = fs.readFileSync(file).toString(); - writeTypeFile(file, typeImports(contents.match(typeImportFindRegex)), snippets); + writeTypeFile(file, typeImports(contents.match(typeImportFindRegex)), snippets); + } } catch (err) { console.error(`Failed emitting types from ${file}: ${err}.`); } From d1cb5d3823b294d56fdd3b407d8bd3bc4090e8d5 Mon Sep 17 00:00:00 2001 From: Kevin Nderitu Date: Fri, 26 Feb 2021 14:44:22 +0300 Subject: [PATCH 14/31] added basic styles on autogenerated toc --- .../Documents/components/DocumentSidePanel.js | 9 ++--- app/react/Documents/components/ShowToc.js | 12 ++++--- .../Documents/components/scss/showToc.scss | 34 +++++++++++++++++++ app/react/Documents/components/scss/toc.scss | 18 ++++++++++ 4 files changed, 64 insertions(+), 9 deletions(-) create mode 100644 app/react/Documents/components/scss/showToc.scss create mode 100644 app/react/Documents/components/scss/toc.scss diff --git a/app/react/Documents/components/DocumentSidePanel.js b/app/react/Documents/components/DocumentSidePanel.js index 9d9cf7d75e..e4aed13429 100644 --- a/app/react/Documents/components/DocumentSidePanel.js +++ b/app/react/Documents/components/DocumentSidePanel.js @@ -1,9 +1,11 @@ +/* eslint-disable max-lines */ import { Tabs, TabLink, TabContent } from 'react-tabs-redux'; import { browserHistory } from 'react-router'; import { connect } from 'react-redux'; import Immutable from 'immutable'; import PropTypes from 'prop-types'; import React, { Component } from 'react'; +import './scss/toc.scss'; import { MetadataFormButtons, ShowMetadata } from 'app/Metadata'; import { NeedAuthorization } from 'app/Auth'; @@ -16,7 +18,6 @@ import ShowIf from 'app/App/ShowIf'; import SidePanel from 'app/Layout/SidePanel'; import DocumentSemanticSearchResults from 'app/SemanticSearch/components/DocumentResults'; import { CopyFromEntity } from 'app/Metadata/components/CopyFromEntity'; -import { FeatureToggle } from 'app/components/Elements/FeatureToggle'; import { TocGeneratedLabel, ReviewTocButton } from 'app/ToggledFeatures/tocGeneration'; import { Icon } from 'UI'; @@ -348,10 +349,10 @@ export class DocumentSidePanel extends Component {

Table of contents + + auto-created ⓘ +

- - auto-created ⓘ -
{tocElement.get('label')} + + {tocElement.getIn(['selectionRectangles', 0]).get('page')} + @@ -65,7 +70,4 @@ function mapDispatchToProps() { return { scrollToToc }; } -export default connect( - null, - mapDispatchToProps -)(ShowToc); +export default connect(null, mapDispatchToProps)(ShowToc); diff --git a/app/react/Documents/components/scss/showToc.scss b/app/react/Documents/components/scss/showToc.scss new file mode 100644 index 0000000000..702d875114 --- /dev/null +++ b/app/react/Documents/components/scss/showToc.scss @@ -0,0 +1,34 @@ +@import "../../../App/scss/config/_colors.scss"; + +.toc { + padding-left: 5px !important; + padding-right: 5px !important; + + ul.toc-view { + $indents: 5; + + @for $i from 0 through $indents { + li.toc-indent-#{$i} { + border-radius: 2px; + height: 28px; + + &:hover { + background-color: $c-info-light; + } + a:hover { + text-decoration: none; + }; + a { + padding-left: 4px; + } + } + } + } +} + +.page-number { + position: relative; + float: right; + font-weight: normal; + padding-right: 4px; +} \ No newline at end of file diff --git a/app/react/Documents/components/scss/toc.scss b/app/react/Documents/components/scss/toc.scss new file mode 100644 index 0000000000..e02c88b19b --- /dev/null +++ b/app/react/Documents/components/scss/toc.scss @@ -0,0 +1,18 @@ +@import "../../../App/scss/config/_colors.scss"; + +.tocHeader { + & { + padding-left: 5px; + } + + h1 { + span.label-generatedToc { + margin-left: 10px; + padding: 1px 6px 1px 6px; + font-size: 8px; + border: 1px solid $c-info-dark; + border-radius: 20px; + background-color: $c-info-light; + } + } +} \ No newline at end of file From 56503004bb8b6180fec820765eb2d7f8896de0b6 Mon Sep 17 00:00:00 2001 From: Kevin Nderitu Date: Mon, 1 Mar 2021 16:33:34 +0300 Subject: [PATCH 15/31] Updated autogenerated toc --- .../Documents/components/scss/showToc.scss | 20 ++++++++++++++- app/react/Documents/components/scss/toc.scss | 25 +++++++++++++------ 2 files changed, 37 insertions(+), 8 deletions(-) diff --git a/app/react/Documents/components/scss/showToc.scss b/app/react/Documents/components/scss/showToc.scss index 702d875114..2572cdb7b8 100644 --- a/app/react/Documents/components/scss/showToc.scss +++ b/app/react/Documents/components/scss/showToc.scss @@ -13,13 +13,31 @@ height: 28px; &:hover { - background-color: $c-info-light; + background-color: #E2E7F4; +; } a:hover { text-decoration: none; }; a { padding-left: 4px; + + @if $i == 0 { + font-weight: 500; + font-size: 16px; + line-height: 18.75px; + color: #444; + } @else if $i == 1 { + font-weight: 400; + font-size: 14px; + line-height: 16.41px; + color: #333; + } @else if $i == 2 { + font-weight: 400; + font-size: 14px; + line-height: 16.41px; + color: #555; + } } } } diff --git a/app/react/Documents/components/scss/toc.scss b/app/react/Documents/components/scss/toc.scss index e02c88b19b..f30a474a67 100644 --- a/app/react/Documents/components/scss/toc.scss +++ b/app/react/Documents/components/scss/toc.scss @@ -2,17 +2,28 @@ .tocHeader { & { - padding-left: 5px; + padding-left: 5px !important; } h1 { + :first-child { + height: 23px; + font-weight: 500; + font-size: 20px; + line-height: 23.44px; + } + span.label-generatedToc { - margin-left: 10px; - padding: 1px 6px 1px 6px; - font-size: 8px; - border: 1px solid $c-info-dark; - border-radius: 20px; - background-color: $c-info-light; + .translation { + margin-left: 10px; + padding: 2px 8px 2px 8px; + font-size: 10px; + border: 1px solid rgba(0, 0, 0, 0.15); + border-radius: 100px; + background-color: #CBE0FF; + width: 100px; + height: 18px; + } } } } \ No newline at end of file From 8868f577f7a4b16b667219314c52c48e64a07804 Mon Sep 17 00:00:00 2001 From: Daneryl Date: Tue, 2 Mar 2021 12:15:54 +0100 Subject: [PATCH 16/31] fix lint errors --- app/api/files/routes.ts | 1 - app/api/search/documentQueryBuilder.js | 2 +- app/api/search/search.js | 9 +++++++-- app/api/search/specs/fixtures_elastic.js | 10 ++++++++-- app/api/search/specs/searchSchema.spec.ts | 6 +++++- app/react/Attachments/components/File.tsx | 5 +---- app/react/Library/actions/libraryActions.js | 1 - .../actions/specs/filterActions.spec.js | 5 ++++- .../actions/specs/libraryActions.spec.js | 10 ++++++++-- app/react/Library/components/FiltersForm.js | 5 +---- .../helpers/specs/libraryFilters.spec.js | 20 +++++++++++++++---- .../tocGeneration/specs/actions.spec.ts | 2 +- 12 files changed, 52 insertions(+), 24 deletions(-) diff --git a/app/api/files/routes.ts b/app/api/files/routes.ts index 9e3170aea2..577519b1cf 100644 --- a/app/api/files/routes.ts +++ b/app/api/files/routes.ts @@ -10,7 +10,6 @@ import activitylogMiddleware from 'api/activitylog/activitylogMiddleware'; import { CSVLoader } from 'api/csv'; import { files } from './files'; import { validation, createError, handleError } from '../utils'; -import entities from 'api/entities'; export default (app: Application) => { app.post( diff --git a/app/api/search/documentQueryBuilder.js b/app/api/search/documentQueryBuilder.js index ee0631acc3..42eb896b7b 100644 --- a/app/api/search/documentQueryBuilder.js +++ b/app/api/search/documentQueryBuilder.js @@ -1,7 +1,7 @@ /* eslint-disable camelcase, max-lines */ import { preloadOptionsSearch } from 'shared/config'; -import filterToMatch, { textFilter, multiselectFilter } from './metadataMatchers'; +import filterToMatch, { multiselectFilter } from './metadataMatchers'; import { propertyToAggregation, generatedTocAggregations } from './metadataAggregations'; export default function() { diff --git a/app/api/search/search.js b/app/api/search/search.js index b3d80184d3..067474bdea 100644 --- a/app/api/search/search.js +++ b/app/api/search/search.js @@ -244,7 +244,12 @@ const _denormalizeAggregations = async (aggregations, templates, dictionaries, l const properties = propertiesHelper.allUniqueProperties(templates); return Object.keys(aggregations).reduce(async (denormaLizedAgregationsPromise, key) => { const denormaLizedAgregations = await denormaLizedAgregationsPromise; - if (!aggregations[key].buckets || key === '_types' || aggregations[key].type === 'nested' || key === 'generatedToc') { + if ( + !aggregations[key].buckets || + key === '_types' || + aggregations[key].type === 'nested' || + key === 'generatedToc' + ) { return Object.assign(denormaLizedAgregations, { [key]: aggregations[key] }); } @@ -683,7 +688,7 @@ const search = { return snippetsFromSearchHit(response.body.hits.hits[0]); }, - async indexEntities(query, select = '', limit, batchCallback = () => { }) { + async indexEntities(query, select = '', limit, batchCallback = () => {}) { return indexEntities({ query, select, diff --git a/app/api/search/specs/fixtures_elastic.js b/app/api/search/specs/fixtures_elastic.js index 3981a52437..198e3ed3f1 100644 --- a/app/api/search/specs/fixtures_elastic.js +++ b/app/api/search/specs/fixtures_elastic.js @@ -180,7 +180,10 @@ export const fixtures = { field2: [{ value: 'bane' }], select1: [{ value: 'EgyptID', label: 'Egypt' }], rich_text: [{ value: 'rich' }], - multiselect1: [{ value: 'EgyptID', label: 'Egypt' }, { value: 'SpainID', label: 'Spain' }], + multiselect1: [ + { value: 'EgyptID', label: 'Egypt' }, + { value: 'SpainID', label: 'Spain' }, + ], groupedDictionary: [{ value: 'GermanyID' }], nestedField_nested: [{ value: { nested1: ['1', '2', '3'] } }], city_geolocation: [{ value: { lat: 1, lon: 2 } }], @@ -438,7 +441,10 @@ export const fixtures = { { label: 'Europe', id: 'EuropeID', - values: [{ label: 'Germany', id: 'GermanyID' }, { label: 'France', id: 'franceID' }], + values: [ + { label: 'Germany', id: 'GermanyID' }, + { label: 'France', id: 'franceID' }, + ], }, ], }, diff --git a/app/api/search/specs/searchSchema.spec.ts b/app/api/search/specs/searchSchema.spec.ts index 8a23458c1b..ffaecc57bf 100644 --- a/app/api/search/specs/searchSchema.spec.ts +++ b/app/api/search/specs/searchSchema.spec.ts @@ -43,7 +43,11 @@ describe('search schema', () => { async function testInvalidProperty(invalidProperty: any) { const invalidSearch = { query: { ...validQuery, ...invalidProperty } }; - await validation.validateRequest(searchParamsSchema)(invalidSearch, null, expectInvalidSchema); + await validation.validateRequest(searchParamsSchema)( + invalidSearch, + null, + expectInvalidSchema + ); } it('should be invalid if allAgregations is not a boolean value', async () => { diff --git a/app/react/Attachments/components/File.tsx b/app/react/Attachments/components/File.tsx index e18dd4bc0a..33d9cd7101 100644 --- a/app/react/Attachments/components/File.tsx +++ b/app/react/Attachments/components/File.tsx @@ -210,7 +210,4 @@ export class File extends Component { const mapDispatchToProps = (dispatch: Dispatch<{}>, props: FileProps) => bindActionCreators({ updateFile, deleteFile }, wrapDispatch(dispatch, props.storeKey)); -export const ConnectedFile = connect( - null, - mapDispatchToProps -)(File); +export const ConnectedFile = connect(null, mapDispatchToProps)(File); diff --git a/app/react/Library/actions/libraryActions.js b/app/react/Library/actions/libraryActions.js index e63a141d86..c87266d1bc 100644 --- a/app/react/Library/actions/libraryActions.js +++ b/app/react/Library/actions/libraryActions.js @@ -12,7 +12,6 @@ import { toUrlParams } from 'shared/JSONRequest'; import { RequestParams } from 'app/utils/RequestParams'; import { store } from 'app/store'; import searchAPI from 'app/Search/SearchAPI'; -import { tocGenerationUtils } from 'app/ToggledFeatures/tocGeneration'; import { selectedDocumentsChanged, maybeSaveQuickLabels } from './quickLabelActions'; export function enterLibrary() { diff --git a/app/react/Library/actions/specs/filterActions.spec.js b/app/react/Library/actions/specs/filterActions.spec.js index a1a187e5be..f8e1fb23b3 100644 --- a/app/react/Library/actions/specs/filterActions.spec.js +++ b/app/react/Library/actions/specs/filterActions.spec.js @@ -24,7 +24,10 @@ describe('filterActions', () => { let filtersState; beforeEach(() => { - libraryFilters = [{ name: 'author', filter: true }, { name: 'country', filter: true }]; + libraryFilters = [ + { name: 'author', filter: true }, + { name: 'country', filter: true }, + ]; search = { searchTerm: '', filters: { author: 'RR Martin', country: '' } }; filtersState = { documentTypes, diff --git a/app/react/Library/actions/specs/libraryActions.spec.js b/app/react/Library/actions/specs/libraryActions.spec.js index d1d94de2cf..4b3853e962 100644 --- a/app/react/Library/actions/specs/libraryActions.spec.js +++ b/app/react/Library/actions/specs/libraryActions.spec.js @@ -181,7 +181,10 @@ describe('libraryActions', () => { { name: 'relationshipfilter', type: 'relationshipfilter', - filters: [{ name: 'status', type: 'select' }, { name: 'empty', type: 'date' }], + filters: [ + { name: 'status', type: 'select' }, + { name: 'empty', type: 'date' }, + ], }, ], documentTypes: ['decision'], @@ -387,7 +390,10 @@ describe('libraryActions', () => { }, { type: types.UPDATE_DOCUMENTS, - docs: [{ sharedId: '1', metadataResponse }, { sharedId: '2', metadataResponse }], + docs: [ + { sharedId: '1', metadataResponse }, + { sharedId: '2', metadataResponse }, + ], }, ]; const store = mockStore({}); diff --git a/app/react/Library/components/FiltersForm.js b/app/react/Library/components/FiltersForm.js index a06985851a..9d08345a1f 100644 --- a/app/react/Library/components/FiltersForm.js +++ b/app/react/Library/components/FiltersForm.js @@ -128,7 +128,4 @@ function mapDispatchToProps(dispatch, props) { return bindActionCreators({ searchDocuments }, wrapDispatch(dispatch, props.storeKey)); } -export default connect( - mapStateToProps, - mapDispatchToProps -)(FiltersForm); +export default connect(mapStateToProps, mapDispatchToProps)(FiltersForm); diff --git a/app/react/Library/helpers/specs/libraryFilters.spec.js b/app/react/Library/helpers/specs/libraryFilters.spec.js index 72225d6450..b911e63c13 100644 --- a/app/react/Library/helpers/specs/libraryFilters.spec.js +++ b/app/react/Library/helpers/specs/libraryFilters.spec.js @@ -33,17 +33,26 @@ describe('library helper', () => { const thesauris = [ { _id: 'abc1', - values: [{ id: 1, value: 'value1' }, { id: 2, value: 'value2' }], + values: [ + { id: 1, value: 'value1' }, + { id: 2, value: 'value2' }, + ], }, { _id: 'thesauri2', type: 'template', - values: [{ id: 3, value: 'value3' }, { id: 4, value: 'value4' }], + values: [ + { id: 3, value: 'value3' }, + { id: 4, value: 'value4' }, + ], }, { _id: 'thesauri3', type: 'template', - values: [{ id: 5, value: 'value5' }, { id: 6, value: 'value6' }], + values: [ + { id: 5, value: 'value5' }, + { id: 6, value: 'value6' }, + ], }, ]; @@ -141,7 +150,10 @@ describe('library helper', () => { filter: true, type: 'select', content: 'abc1', - options: [{ id: 1, value: 'value1' }, { id: 2, value: 'value2' }], + options: [ + { id: 1, value: 'value1' }, + { id: 2, value: 'value2' }, + ], }, { name: 'date', filter: true, type: 'text' }, ]; diff --git a/app/react/ToggledFeatures/tocGeneration/specs/actions.spec.ts b/app/react/ToggledFeatures/tocGeneration/specs/actions.spec.ts index 8950ca6b73..96e8905fb2 100644 --- a/app/react/ToggledFeatures/tocGeneration/specs/actions.spec.ts +++ b/app/react/ToggledFeatures/tocGeneration/specs/actions.spec.ts @@ -7,7 +7,7 @@ import configureMockStore from 'redux-mock-store'; import thunk from 'redux-thunk'; import Immutable from 'immutable'; import { mockID } from 'shared/uniqueID.js'; -import { ClientEntitySchema, IStore } from 'app/istore'; +import { ClientEntitySchema } from 'app/istore'; import { tocGenerationActions } from '../actions'; const middlewares = [thunk]; From f632eadc6cd51f7b2be3253c049877d1a8b9f0f7 Mon Sep 17 00:00:00 2001 From: Daneryl Date: Tue, 2 Mar 2021 13:05:22 +0100 Subject: [PATCH 17/31] fix blank state panel --- app/react/Library/components/ViewMetadataPanel.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/react/Library/components/ViewMetadataPanel.js b/app/react/Library/components/ViewMetadataPanel.js index f3487a6278..b0469bdb12 100644 --- a/app/react/Library/components/ViewMetadataPanel.js +++ b/app/react/Library/components/ViewMetadataPanel.js @@ -26,7 +26,7 @@ const mapStateToProps = (state, props) => { const doc = library.ui.get('selectedDocuments').first() || Immutable.fromJS({ documents: [] }); const defaultLanguage = state.settings.collection.get('languages').find(l => l.get('defautl')); const file = entityDefaultDocument( - doc.get('documents').toJS(), + doc.get('documents') ? doc.get('documents').toJS() : [{}], doc.get('language'), defaultLanguage ); From eb09f2f2ee82fafad20cc1baaf6be5f9270a9c3f Mon Sep 17 00:00:00 2001 From: Daneryl Date: Tue, 2 Mar 2021 17:34:20 +0100 Subject: [PATCH 18/31] fix e2e --- app/react/Templates/utils/getFieldLabel.js | 2 +- app/react/Templates/utils/specs/getFieldLabel.spec.js | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/app/react/Templates/utils/getFieldLabel.js b/app/react/Templates/utils/getFieldLabel.js index 8807314df6..b8828ca03a 100644 --- a/app/react/Templates/utils/getFieldLabel.js +++ b/app/react/Templates/utils/getFieldLabel.js @@ -13,7 +13,7 @@ function getMetadataFieldLabel(field, template) { export default function getFieldLabel(field, template) { const _template = template && template.toJS ? template.toJS() : template; - if (field === 'title') { + if (field === 'title' && _template) { return getTitleLabel(_template); } if (field.startsWith('metadata.') && _template) { diff --git a/app/react/Templates/utils/specs/getFieldLabel.spec.js b/app/react/Templates/utils/specs/getFieldLabel.spec.js index e248a85a6a..b79d3f8a6c 100644 --- a/app/react/Templates/utils/specs/getFieldLabel.spec.js +++ b/app/react/Templates/utils/specs/getFieldLabel.spec.js @@ -40,6 +40,13 @@ describe('getFieldLabel', () => { field = 'title'; expect(runGetLabel()).toEqual(t(template._id, 'Name')); }); + describe('when template is not defined', () => { + it('should return the input field', () => { + template = undefined; + field = 'title'; + expect(runGetLabel()).toEqual('title'); + }); + }); }); describe('when field is not in template', () => { From eba2caf177be5cc909791b733b8b7ad75f288d4d Mon Sep 17 00:00:00 2001 From: Daneryl Date: Wed, 3 Mar 2021 12:29:58 +0100 Subject: [PATCH 19/31] prevent crash on toc service unavailable --- app/api/search/search.js | 3 -- .../toc_generation/specs/tocService.spec.ts | 48 +++++++++++++------ app/api/toc_generation/tocService.ts | 42 ++++++++-------- 3 files changed, 56 insertions(+), 37 deletions(-) diff --git a/app/api/search/search.js b/app/api/search/search.js index 067474bdea..f47b95bef1 100644 --- a/app/api/search/search.js +++ b/app/api/search/search.js @@ -195,9 +195,6 @@ const indexedDictionaryValues = dictionary => }, {}); const _getAggregationDictionary = async (aggregation, language, property, dictionaries) => { - if (!property) { - return [{values: []}, {}]; - } if (property.type === 'relationship') { const entitiesSharedId = aggregation.buckets.map(bucket => bucket.key); diff --git a/app/api/toc_generation/specs/tocService.spec.ts b/app/api/toc_generation/specs/tocService.spec.ts index d77ec52000..328a87edc7 100644 --- a/app/api/toc_generation/specs/tocService.spec.ts +++ b/app/api/toc_generation/specs/tocService.spec.ts @@ -2,43 +2,57 @@ import { testingDB } from 'api/utils/testing_db'; import request from 'shared/JSONRequest'; import { files } from 'api/files'; import { elasticTesting } from 'api/utils/elastic_testing'; +import errorLog from 'api/log/errorLog'; import { fixtures } from './fixtures'; import { tocService } from '../tocService'; describe('tocService', () => { const service = tocService('url'); - beforeAll(async () => { - spyOn(request, 'uploadFile').and.callFake(async (url, filename) => { - expect(url).toBe('url'); - if (filename === 'pdf1.pdf') { - return Promise.resolve([{ label: 'section1 pdf1' }]); - } - if (filename === 'pdf3.pdf') { - return Promise.resolve([{ label: 'section1 pdf3' }]); - } - throw new Error(`this file is not supposed to be sent for toc generation ${filename}`); - }); - }); afterAll(async () => { await testingDB.disconnect(); }); + let requestMock: jest.SpyInstance; + describe('processNext', () => { - beforeAll(async () => { + beforeEach(async () => { + spyOn(errorLog, 'error'); + requestMock = jest.spyOn(request, 'uploadFile'); + requestMock.mockImplementation(async (url, filename) => { + expect(url).toBe('url'); + if (filename === 'pdf1.pdf') { + return Promise.resolve([{ label: 'section1 pdf1' }]); + } + if (filename === 'pdf3.pdf') { + return Promise.resolve([{ label: 'section1 pdf3' }]); + } + throw new Error(`this file is not supposed to be sent for toc generation ${filename}`); + }); + const elasticIndex = 'toc.service.index'; await testingDB.clearAllAndLoad(fixtures, elasticIndex); await elasticTesting.resetIndex(); - await service.processNext(); - await service.processNext(); await elasticTesting.refresh(); }); + it('should not fail when request fails', async () => { + requestMock.mockImplementation(async () => { + throw new Error('request error'); + }); + await expect(service.processNext()).resolves.not.toThrow(); + }); + it('should not fail when there is no more to process', async () => { + await service.processNext(); + await service.processNext(); await expect(service.processNext()).resolves.not.toThrow(); }); it('should send the next pdfFile and save toc generated', async () => { + await service.processNext(); + await service.processNext(); + let [fileProcessed] = await files.get({ filename: 'pdf1.pdf' }); expect(fileProcessed.toc).toEqual([{ label: 'section1 pdf1' }]); expect(fileProcessed.generatedToc).toEqual(true); @@ -49,6 +63,10 @@ describe('tocService', () => { }); it('should reindex the affected entities', async () => { + await service.processNext(); + await service.processNext(); + await elasticTesting.refresh(); + const entitiesIndexed = await elasticTesting.getIndexedEntities(); expect(entitiesIndexed).toEqual([ diff --git a/app/api/toc_generation/tocService.ts b/app/api/toc_generation/tocService.ts index 69df4c7c59..e24de725ba 100644 --- a/app/api/toc_generation/tocService.ts +++ b/app/api/toc_generation/tocService.ts @@ -1,4 +1,6 @@ import { files, uploadsPath } from 'api/files'; +import { prettifyError } from 'api/utils/handleError'; +import errorLog from 'api/log/errorLog'; import request from 'shared/JSONRequest'; import entities from 'api/entities'; @@ -16,26 +18,28 @@ const tocService = (serviceUrl: string) => ({ limit: 1, } ); - if (!nextFile) { - return null; - } - - const toc = await request.uploadFile( - serviceUrl, - nextFile.filename, - uploadsPath(nextFile.filename) - ); - await files.save({ ...nextFile, toc, generatedToc: true }); - const [entity] = await entities.get({ sharedId: nextFile.entity }, {}); - return entities.save( - { - ...entity, - generatedToc: true, - }, - { user: {}, language: nextFile.language }, - false - ); + try { + if (nextFile) { + const toc = await request.uploadFile( + serviceUrl, + nextFile.filename, + uploadsPath(nextFile.filename) + ); + await files.save({ ...nextFile, toc, generatedToc: true }); + const [entity] = await entities.get({ sharedId: nextFile.entity }, {}); + await entities.save( + { + ...entity, + generatedToc: true, + }, + { user: {}, language: nextFile.language }, + false + ); + } + } catch (e) { + errorLog.error(prettifyError(e).prettyMessage); + } }, }); From 5928bb0413b9cebd6f262f14a54cdb7a43730260 Mon Sep 17 00:00:00 2001 From: Kevin Nderitu Date: Thu, 4 Mar 2021 14:48:45 +0300 Subject: [PATCH 20/31] Removed unnecessary scss import --- app/react/Documents/components/scss/showToc.scss | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/react/Documents/components/scss/showToc.scss b/app/react/Documents/components/scss/showToc.scss index 2572cdb7b8..60149b1997 100644 --- a/app/react/Documents/components/scss/showToc.scss +++ b/app/react/Documents/components/scss/showToc.scss @@ -1,5 +1,3 @@ -@import "../../../App/scss/config/_colors.scss"; - .toc { padding-left: 5px !important; padding-right: 5px !important; From d8ef75b6915d897dca3faa86c5dea29ddcb0fa17 Mon Sep 17 00:00:00 2001 From: Daneryl Date: Wed, 3 Mar 2021 15:51:33 +0100 Subject: [PATCH 21/31] fixed/refactor styles --- app/react/App/scss/modules/_search.scss | 11 ++++ app/react/App/scss/modules/_uploads.scss | 19 +++---- app/react/Attachments/components/File.tsx | 52 ++++++++++--------- .../components/specs/File.spec.tsx | 2 +- .../DocumentForm/components/FormGroup.js | 7 ++- .../Documents/components/DocumentSidePanel.js | 11 ++-- app/react/Documents/components/scss/toc.scss | 41 ++++++--------- .../__snapshots__/ThesauriForm.spec.js.snap | 4 +- .../ThesauriFormGroup.spec.js.snap | 4 +- .../tocGeneration/FilterTocGeneration.tsx | 2 +- .../tocGeneration/TocGeneratedLabel.tsx | 2 +- nightmare/helpers/selectors.js | 2 +- 12 files changed, 84 insertions(+), 73 deletions(-) diff --git a/app/react/App/scss/modules/_search.scss b/app/react/App/scss/modules/_search.scss index 59b2275316..f77c054e62 100644 --- a/app/react/App/scss/modules/_search.scss +++ b/app/react/App/scss/modules/_search.scss @@ -233,12 +233,23 @@ display: none; } +.admin-filter { + span, label { + color: #2B56C1; + } + + input { + border: #2B56C1; + } +} + .search__filter { list-style-type: none; margin: 0; padding: 15px; color: $c-grey-dark; + &:not(:last-child), .form-group:not(:last-child) & { border-bottom: 1px solid $c-grey-light; diff --git a/app/react/App/scss/modules/_uploads.scss b/app/react/App/scss/modules/_uploads.scss index f865b298ce..aa4ff732b4 100644 --- a/app/react/App/scss/modules/_uploads.scss +++ b/app/react/App/scss/modules/_uploads.scss @@ -87,9 +87,14 @@ list-style: none; } + .upload-button { + margin-left: 15px; + } + .btn { border-radius: 2px; - margin-left: 15px; + margin-right: 15px; + margin-top: 15px; } } @@ -97,23 +102,11 @@ .file-form { padding: 15px; border-bottom: 1px solid #f4f4f4; - .btn { - float: right; - } &-originalname { margin-bottom: 15px; } - &-language { - background: #fff6da; - border: 1px solid rgba(0, 0, 0, 0.15); - box-sizing: border-box; - border-radius: 4px; - display: inline-block; - padding: 3px; - } - &-failed { box-sizing: border-box; border-radius: 4px; diff --git a/app/react/Attachments/components/File.tsx b/app/react/Attachments/components/File.tsx index 33d9cd7101..5be517c351 100644 --- a/app/react/Attachments/components/File.tsx +++ b/app/react/Attachments/components/File.tsx @@ -101,32 +101,36 @@ export class File extends Component { const { language, filename = '' } = this.props.file; return (
-
- {language ? transformLanguage(language) || '' : ''} +
+ + {language ? transformLanguage(language) || '' : ''} + + + ML TOC +
- - ML TOC - - - -   - Download - - - - - - View - + Download + + + + + + View + +
); } diff --git a/app/react/Attachments/components/specs/File.spec.tsx b/app/react/Attachments/components/specs/File.spec.tsx index 95ca643fa4..49370757e8 100644 --- a/app/react/Attachments/components/specs/File.spec.tsx +++ b/app/react/Attachments/components/specs/File.spec.tsx @@ -42,7 +42,7 @@ describe('file', () => { expect(title).toBe('Human_name_1.pdf'); const language = component - .find('.file-language') + .find('.badge') .find(Translate) .props().children; expect(language).toBe('english'); diff --git a/app/react/DocumentForm/components/FormGroup.js b/app/react/DocumentForm/components/FormGroup.js index 74ce5089f0..df56f9b9aa 100644 --- a/app/react/DocumentForm/components/FormGroup.js +++ b/app/react/DocumentForm/components/FormGroup.js @@ -2,7 +2,7 @@ import PropTypes from 'prop-types'; import React from 'react'; const FormGroup = props => { - let className = 'form-group'; + let className = `${props.className} form-group`; if ((!props.pristine || props.submitFailed) && props.valid === false) { className += ' has-error'; } @@ -12,7 +12,12 @@ const FormGroup = props => { const childrenType = PropTypes.oneOfType([PropTypes.object, PropTypes.array]); +FormGroup.defaultProps = { + className: '', +}; + FormGroup.propTypes = { + className: PropTypes.string, pristine: PropTypes.bool, valid: PropTypes.bool, submitFailed: PropTypes.bool, diff --git a/app/react/Documents/components/DocumentSidePanel.js b/app/react/Documents/components/DocumentSidePanel.js index e4aed13429..96f248889b 100644 --- a/app/react/Documents/components/DocumentSidePanel.js +++ b/app/react/Documents/components/DocumentSidePanel.js @@ -324,7 +324,7 @@ export class DocumentSidePanel extends Component {
)} diff --git a/app/react/UI/Icon/library.js b/app/react/UI/Icon/library.js index 17dfade388..e1f01cbcdb 100644 --- a/app/react/UI/Icon/library.js +++ b/app/react/UI/Icon/library.js @@ -89,6 +89,7 @@ import { faUsers } from '@fortawesome/free-solid-svg-icons/faUsers'; import { faUserTimes } from '@fortawesome/free-solid-svg-icons/faUserTimes'; import { faHandPaper } from '@fortawesome/free-solid-svg-icons/faHandPaper'; import { faExternalLinkAlt } from '@fortawesome/free-solid-svg-icons/faExternalLinkAlt'; +import { faTasks } from '@fortawesome/free-solid-svg-icons/faTasks'; import { saveAndNext } from './save-and-next'; import { exportCsv } from './export-csv'; import { copyFrom } from './copy-from'; @@ -186,6 +187,7 @@ const icons = { faHandPaper, faExternalLinkAlt, saveAndNext, + faTasks, exportCsv, copyFrom, funnelFilter, From 97efec72ed43885f42bda758bb07fbdd2f0ef495 Mon Sep 17 00:00:00 2001 From: Daneryl Date: Mon, 8 Mar 2021 15:38:58 +0100 Subject: [PATCH 31/31] filteredAggregations labels now using t function --- .../ToggledFeatures/tocGeneration/FilterTocGeneration.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/react/ToggledFeatures/tocGeneration/FilterTocGeneration.tsx b/app/react/ToggledFeatures/tocGeneration/FilterTocGeneration.tsx index 594f682949..3753a45758 100644 --- a/app/react/ToggledFeatures/tocGeneration/FilterTocGeneration.tsx +++ b/app/react/ToggledFeatures/tocGeneration/FilterTocGeneration.tsx @@ -20,12 +20,12 @@ const filteredAggregation = (aggregations: Aggregations, key: string) => { const options = (aggregations: Aggregations = { all: {} }) => [ { - label: 'Automatically generated', + label: t('System', 'Automatically generated'), value: true, results: filteredAggregation(aggregations, 'true'), }, { - label: 'Reviewed', + label: t('System', 'Reviewed'), value: false, results: filteredAggregation(aggregations, 'false'), },