diff --git a/app/api/csv/specs/fixtures.js b/app/api/csv/specs/fixtures.js index 74e5c2117f..c630fffa32 100644 --- a/app/api/csv/specs/fixtures.js +++ b/app/api/csv/specs/fixtures.js @@ -19,37 +19,44 @@ export default { name: 'base template', properties: [ { + _id: db.id(), type: propertyTypes.text, label: 'text label', name: templateUtils.safeName('text label'), }, { + _id: db.id(), type: propertyTypes.numeric, label: 'numeric label', name: templateUtils.safeName('numeric label'), }, { + _id: db.id(), type: propertyTypes.select, label: 'select label', name: templateUtils.safeName('select label'), content: thesauri1Id, }, { + _id: db.id(), type: 'non_defined_type', label: 'not defined type', name: templateUtils.safeName('not defined type'), }, { + _id: db.id(), type: propertyTypes.text, label: 'not configured on csv', name: templateUtils.safeName('not configured on csv'), }, { + _id: db.id(), type: propertyTypes.geolocation, label: 'geolocation', name: templateUtils.safeName('geolocation_geolocation'), }, { + _id: db.id(), type: propertyTypes.generatedid, label: 'Auto ID', name: templateUtils.safeName('auto id'), diff --git a/app/api/entities/denormalize.ts b/app/api/entities/denormalize.ts new file mode 100644 index 0000000000..4e1dcce4e1 --- /dev/null +++ b/app/api/entities/denormalize.ts @@ -0,0 +1,127 @@ +import { EntitySchema } from 'shared/types/entityType'; +import { PropertySchema } from 'shared/types/commonTypes'; +import templates from 'api/templates/templates'; +import { search } from 'api/search'; +import { TemplateSchema } from 'shared/types/templateType'; +import { WithId } from 'api/odm'; + +import model from './entitiesModel'; + +interface Changes { + label: string; + icon?: EntitySchema['icon']; +} + +interface Params { + id: string; + language?: string; + template?: string; +} + +export const updateDenormalization = async ( + { id, language, template }: Params, + changes: Changes, + properties: PropertySchema[] +) => + Promise.all( + properties.map(async property => + model.updateMany( + { + ...(template ? { template } : {}), + ...(language ? { language } : {}), + [`metadata.${property.name}.value`]: id, + }, + { + $set: Object.keys(changes).reduce( + (set, prop) => ({ + ...set, + [`metadata.${property.name}.$[valueObject].${prop}`]: changes[prop], + }), + {} + ), + }, + { arrayFilters: [{ 'valueObject.value': id }] } + ) + ) + ); + +export const updateTransitiveDenormalization = async ( + { id, language }: Params, + changes: Changes, + properties: PropertySchema[] +) => + Promise.all( + properties.map(async property => + model.updateMany( + { language, [`metadata.${property.name}.inheritedValue.value`]: id }, + { + ...(changes.icon + ? { [`metadata.${property.name}.$[].inheritedValue.$[valueObject].icon`]: changes.icon } + : {}), + [`metadata.${property.name}.$[].inheritedValue.$[valueObject].label`]: changes.label, + }, + { arrayFilters: [{ 'valueObject.value': id }] } + ) + ) + ); + +export const denormalizeRelated = async ( + entity: WithId, + template: WithId +) => { + if (!entity.title || !entity.language || !entity.sharedId) { + throw new Error('denormalization requires an entity with title, sharedId and language'); + } + + const transitiveProperties = await templates.propsThatNeedTransitiveDenormalization( + template._id.toString() + ); + const properties = await templates.propsThatNeedDenormalization(template._id.toString()); + + await updateTransitiveDenormalization( + { id: entity.sharedId, language: entity.language }, + { label: entity.title, icon: entity.icon }, + transitiveProperties + ); + + await Promise.all( + properties.map(async prop => { + const inheritProperty = (template.properties || []).find( + p => prop.inheritProperty === p._id?.toString() + ); + return updateDenormalization( + { + // @ts-ignore we have a sharedId guard, why ts does not like this ? bug ? + id: entity.sharedId, + language: entity.language, + template: prop.template, + }, + { + ...(inheritProperty ? { inheritedValue: entity.metadata?.[inheritProperty.name] } : {}), + label: entity.title, + icon: entity.icon, + }, + [prop] + ); + }) + ); + + if (properties.length || transitiveProperties.length) { + await search.indexEntities({ + $and: [ + { language: entity.language }, + { + $or: [ + ...properties.map(property => ({ + [`metadata.${property.name}.value`]: entity.sharedId, + })), + ...transitiveProperties.map(property => ({ + [`metadata.${property.name}.inheritedValue.value`]: entity.sharedId, + })), + ], + }, + ], + }); + } + ////Crappy draft code ends +}; diff --git a/app/api/entities/entities.js b/app/api/entities/entities.js index 748c8a5aeb..47d09e7d52 100644 --- a/app/api/entities/entities.js +++ b/app/api/entities/entities.js @@ -21,6 +21,11 @@ import { validateEntity } from 'shared/types/entitySchema'; import { deleteFiles, deleteUploadedFiles } from '../files/filesystem'; import model from './entitiesModel'; import settings from '../settings'; +import { + denormalizeRelated, + updateTransitiveDenormalization, + updateDenormalization, +} from './denormalize'; /** Repopulate metadata object .label from thesauri and relationships. */ async function denormalizeMetadata(metadata, entity, template, dictionariesByKey) { @@ -140,22 +145,14 @@ async function updateEntity(entity, _template, unrestricted = false) { return Promise.all( docLanguages.map(async d => { if (d._id.toString() === entity._id.toString()) { - if ( - (entity.title && currentDoc.title !== entity.title) || - (entity.icon && !currentDoc.icon) || - (entity.icon && currentDoc.icon && currentDoc.icon._id !== entity.icon._id) - ) { - await this.renameRelatedEntityInMetadata({ ...currentDoc, ...entity }); - } - const toSave = { ...entity }; - delete toSave.published; delete toSave.permissions; if (entity.metadata) { toSave.metadata = await denormalizeMetadata(entity.metadata, entity, template); } + if (entity.suggestedMetadata) { toSave.suggestedMetadata = await denormalizeMetadata( entity.suggestedMetadata, @@ -163,6 +160,10 @@ async function updateEntity(entity, _template, unrestricted = false) { template ); } + if (template._id) { + const fullEntity = { ...currentDoc, ...toSave }; + await denormalizeRelated(fullEntity, template); + } return saveFunc(toSave); } @@ -189,6 +190,11 @@ async function updateEntity(entity, _template, unrestricted = false) { if (typeof entity.generatedToc !== 'undefined') { d.generatedToc = entity.generatedToc; } + + if (template._id) { + await denormalizeRelated(d, template); + } + return saveFunc(d); }) ); @@ -382,7 +388,7 @@ export default { )[0]; if (updateRelationships) { await relationships.saveEntityBasedReferences(entity, language); - await this.updateDenormalizedMetadataInRelatedEntities(entity); + // await this.updateDenormalizedMetadataInRelatedEntities(entity); } if (index) { await search.indexEntities({ sharedId }, '+fullText'); @@ -391,12 +397,6 @@ export default { return entity; }, - async updateDenormalizedMetadataInRelatedEntities(entity) { - const related = await relationships.getByDocument(entity.sharedId, entity.language); - const sharedIds = related.map(r => r.entityData.sharedId); - await this.updateMetdataFromRelationships(sharedIds, entity.language); - }, - async denormalize(_doc, { user, language }) { await validateEntity(_doc); const doc = _doc; @@ -755,68 +755,47 @@ export default { ]); }, - /** Propagate the change of a thesaurus or related entity label to all entity metadata. */ - async renameInMetadata(valueId, changes, propertyContent, { types, restrictLanguage = null }) { - const properties = (await templates.get({ 'properties.content': propertyContent })) + /** Propagate the change of a thesaurus label to all entity metadata. */ + async renameThesaurusInMetadata(valueId, newLabel, thesaurusId, language) { + const properties = ( + await templates.get({ + 'properties.content': thesaurusId, + }) + ) .reduce((m, t) => m.concat(t.properties), []) - .filter(p => types.includes(p.type)) - .filter( - p => propertyContent && p.content && propertyContent.toString() === p.content.toString() - ); + .filter(p => p.content && thesaurusId === p.content.toString()); - if (!properties.length) { - return Promise.resolve(); - } + await updateDenormalization({ id: valueId, language }, { label: newLabel }, properties); - await Promise.all( - properties.map(property => - model.updateMany( - { language: restrictLanguage, [`metadata.${property.name}.value`]: valueId }, - { - $set: Object.keys(changes).reduce( - (set, prop) => ({ - ...set, - [`metadata.${property.name}.$[valueObject].${prop}`]: changes[prop], - }), - {} - ), - }, - { arrayFilters: [{ 'valueObject.value': valueId }] } - ) - ) + const transitiveProps = await templates.propsThatNeedTransitiveDenormalization( + thesaurusId.toString() ); - return search.indexEntities({ - $and: [ - { - language: restrictLanguage, - }, - { - $or: properties.map(property => ({ [`metadata.${property.name}.value`]: valueId })), - }, - ], - }); - }, - - /** Propagate the change of a thesaurus label to all entity metadata. */ - async renameThesaurusInMetadata(valueId, newLabel, thesaurusId, language) { - await this.renameInMetadata(valueId, { label: newLabel }, thesaurusId, { - types: [propertyTypes.select, propertyTypes.multiselect], - restrictLanguage: language, - }); - }, - - /** Propagate the title change of a related entity to all entity metadata. */ - async renameRelatedEntityInMetadata(relatedEntity) { - await this.renameInMetadata( - relatedEntity.sharedId, - { label: relatedEntity.title, icon: relatedEntity.icon }, - relatedEntity.template, - { - types: [propertyTypes.select, propertyTypes.multiselect, propertyTypes.relationship], - restrictLanguage: relatedEntity.language, - } + await updateTransitiveDenormalization( + { id: valueId, language }, + { label: newLabel }, + transitiveProps ); + + if (properties.length || transitiveProps.length) { + await search.indexEntities({ + $and: [ + { + language, + }, + { + $or: [ + ...properties.map(property => ({ + [`metadata.${property.name}.value`]: valueId, + })), + ...transitiveProps.map(property => ({ + [`metadata.${property.name}.inheritedValue.value`]: valueId, + })), + ], + }, + ], + }); + } }, async createThumbnail(entity) { diff --git a/app/api/entities/specs/denormalization.spec.ts b/app/api/entities/specs/denormalization.spec.ts new file mode 100644 index 0000000000..ba94043824 --- /dev/null +++ b/app/api/entities/specs/denormalization.spec.ts @@ -0,0 +1,565 @@ +/* eslint-disable max-statements */ +/* eslint-disable max-lines */ +import db, { DBFixture } from 'api/utils/testing_db'; +import entities from 'api/entities'; + +import { EntitySchema } from 'shared/types/entityType'; +import thesauris from 'api/thesauri'; +import { elasticTesting } from 'api/utils/elastic_testing'; +import { getFixturesFactory } from '../../utils/fixturesFactory'; + +const load = async (data: DBFixture, index?: string) => + db.setupFixturesAndContext( + { + ...data, + settings: [{ _id: db.id(), languages: [{ key: 'en', default: true }, { key: 'es' }] }], + translations: [ + { locale: 'en', contexts: [] }, + { locale: 'es', contexts: [] }, + ], + }, + index + ); + +describe('Denormalize relationships', () => { + const factory = getFixturesFactory(); + + const modifyEntity = async (id: string, entityData: EntitySchema, language: string = 'en') => { + await entities.save( + { _id: factory.id(`${id}-${language}`), sharedId: id, ...entityData, language }, + { language, user: {} }, + true + ); + }; + + afterAll(async () => db.disconnect()); + + describe('title and basic property (text)', () => { + it('should update denormalized title and icon', async () => { + const fixtures: DBFixture = { + templates: [ + factory.template('templateA', [ + factory.relationshipProp('relationship', 'templateB', 'text'), + ]), + factory.template('templateB', [factory.property('text')]), + ], + entities: [ + factory.entity('A1', 'templateA', { + relationship: [factory.metadataValue('B1'), factory.metadataValue('B2')], + }), + factory.entity('B1', 'templateB', {}, { icon: { _id: 'icon_id' } }), + factory.entity('B2', 'templateB'), + ], + }; + + await load(fixtures); + await modifyEntity('B1', { title: 'new Title' }); + await modifyEntity('B2', { title: 'new Title 2' }); + + const relatedEntity = await entities.getById('A1', 'en'); + expect(relatedEntity?.metadata).toMatchObject({ + relationship: [{ label: 'new Title', icon: { _id: 'icon_id' } }, { label: 'new Title 2' }], + }); + }); + + it('should update title, icon and text property on related entities denormalized properties', async () => { + const fixtures: DBFixture = { + templates: [ + factory.template('templateA', [ + factory.inherit('relationship', 'templateB', 'text'), + factory.inherit('relationship2', 'templateC', 'another_text'), + ]), + factory.template('templateB', [factory.property('text')]), + factory.template('templateC', [factory.property('another_text')]), + ], + entities: [ + factory.entity('A1', 'templateA', { + relationship: [factory.metadataValue('B1'), factory.metadataValue('B2')], + relationship2: [factory.metadataValue('C1')], + }), + factory.entity( + 'B1', + 'templateB', + {}, + { + icon: { _id: 'icon_id', label: 'icon_label', type: 'icon_type' }, + } + ), + factory.entity('B2', 'templateB'), + factory.entity('C1', 'templateC'), + ], + }; + + await load(fixtures); + + await modifyEntity('B1', { + title: 'new Title', + metadata: { text: [{ value: 'text 1 changed' }] }, + }); + + await modifyEntity('B2', { + title: 'new Title 2', + metadata: { text: [{ value: 'text 2 changed' }] }, + }); + + await modifyEntity('C1', { + title: 'new Title C1', + metadata: { another_text: [{ value: 'another text changed' }] }, + }); + + const relatedEntity = await entities.getById('A1', 'en'); + expect(relatedEntity?.metadata).toMatchObject({ + relationship: [ + { + label: 'new Title', + icon: { _id: 'icon_id', label: 'icon_label', type: 'icon_type' }, + inheritedValue: [{ value: 'text 1 changed' }], + }, + { + label: 'new Title 2', + inheritedValue: [{ value: 'text 2 changed' }], + }, + ], + relationship2: [ + { + label: 'new Title C1', + inheritedValue: [{ value: 'another text changed' }], + }, + ], + }); + }); + + it('should update title and text property denormalized on related entities from 2 different templates', async () => { + const fixtures: DBFixture = { + templates: [ + factory.template('templateA', [ + factory.property('text'), + factory.property('another_text'), + ]), + factory.template('templateB', [factory.inherit('relationship', 'templateA', 'text')]), + factory.template('templateC', [ + factory.inherit('relationship', 'templateA', 'another_text'), + ]), + ], + entities: [ + factory.entity('A1', 'templateA'), + factory.entity('B1', 'templateB', { relationship: [factory.metadataValue('A1')] }), + factory.entity('B2', 'templateB', { relationship: [factory.metadataValue('A1')] }), + factory.entity('C1', 'templateC', { relationship: [factory.metadataValue('A1')] }), + ], + }; + + await load(fixtures); + + await modifyEntity('A1', { + title: 'new A1', + metadata: { + text: [{ value: 'text changed' }], + another_text: [{ value: 'another_text changed' }], + }, + }); + + const [relatedB1, relatedB2, relatedC] = [ + await entities.getById('B1', 'en'), + await entities.getById('B2', 'en'), + await entities.getById('C1', 'en'), + ]; + + expect(relatedB1?.metadata?.relationship).toMatchObject([ + { label: 'new A1', inheritedValue: [{ value: 'text changed' }] }, + ]); + + expect(relatedB2?.metadata?.relationship).toMatchObject([ + { label: 'new A1', inheritedValue: [{ value: 'text changed' }] }, + ]); + + expect(relatedC?.metadata?.relationship).toMatchObject([ + { label: 'new A1', inheritedValue: [{ value: 'another_text changed' }] }, + ]); + }); + + it('should update title and 2 different text properties denormalized on related entities', async () => { + const fixtures: DBFixture = { + templates: [ + factory.template('templateA', [factory.property('text1'), factory.property('text2')]), + factory.template('templateB', [factory.inherit('relationship_b', 'templateA', 'text1')]), + factory.template('templateC', [factory.inherit('relationship_c', 'templateA', 'text2')]), + ], + entities: [ + factory.entity('A1', 'templateA'), + factory.entity('B1', 'templateB', { relationship_b: [factory.metadataValue('A1')] }), + factory.entity('C1', 'templateC', { relationship_c: [factory.metadataValue('A1')] }), + ], + }; + + await load(fixtures); + + await modifyEntity('A1', { + title: 'new A1', + metadata: { text1: [{ value: 'text 1 changed' }], text2: [{ value: 'text 2 changed' }] }, + }); + + const [relatedB, relatedC] = [ + await entities.getById('B1', 'en'), + await entities.getById('C1', 'en'), + ]; + + expect(relatedB?.metadata?.relationship_b).toMatchObject([ + { label: 'new A1', inheritedValue: [{ value: 'text 1 changed' }] }, + ]); + + expect(relatedC?.metadata?.relationship_c).toMatchObject([ + { label: 'new A1', inheritedValue: [{ value: 'text 2 changed' }] }, + ]); + }); + }); + + describe('when the relationship property has no content', () => { + it('should denormalize and index the title on related entities', async () => { + const fixtures: DBFixture = { + templates: [ + factory.template('templateA', [ + factory.relationshipProp('relationship', '', { content: '' }), + ]), + factory.template('templateB'), + factory.template('templateC'), + ], + entities: [ + factory.entity('A1', 'templateA', { + relationship: [factory.metadataValue('B1'), factory.metadataValue('C1')], + }), + factory.entity('B1', 'templateB'), + factory.entity('C1', 'templateC'), + ], + }; + + await load(fixtures, 'index_denormalization'); + + await modifyEntity('A1', { + metadata: { relationship: [factory.metadataValue('B1'), factory.metadataValue('C1')] }, + }); + + await modifyEntity('B1', { title: 'new B1' }); + await modifyEntity('C1', { title: 'new C1' }); + + const relatedEntity = await entities.getById('A1', 'en'); + + expect(relatedEntity?.metadata?.relationship).toMatchObject([ + { + label: 'new B1', + }, + { + label: 'new C1', + }, + ]); + + await elasticTesting.refresh(); + const results = await elasticTesting.getIndexedEntities(); + + const [A1] = results.filter(r => r.sharedId === 'A1'); + + expect(A1?.metadata?.relationship).toMatchObject([ + { + label: 'new B1', + }, + { + label: 'new C1', + }, + ]); + }); + }); + + describe('inherited select/multiselect (thesauri)', () => { + beforeEach(async () => { + const fixtures: DBFixture = { + templates: [ + factory.template('templateA', [ + factory.inherit('relationship', 'templateB', 'multiselect'), + ]), + factory.template('templateB', [ + factory.property('multiselect', 'multiselect', { + content: factory.id('thesauri').toString(), + }), + factory.property('property_without_content'), + ]), + ], + dictionaries: [factory.thesauri('thesauri', ['T1', 'T2', 'T3'])], + entities: [ + factory.entity('A1', 'templateA', { + relationship: [factory.metadataValue('B1'), factory.metadataValue('B2')], + }), + factory.entity('B1', 'templateB', { + multiselect: [factory.metadataValue('T1')], + }), + factory.entity('B2', 'templateB'), + ], + }; + await load(fixtures, 'index_denormalize'); + }); + + it('should update denormalized properties when thesauri selected changes', async () => { + await modifyEntity('B1', { + metadata: { multiselect: [{ value: 'T2' }, { value: 'T3' }] }, + }); + + await modifyEntity('B2', { + metadata: { multiselect: [{ value: 'T1' }] }, + }); + + const relatedEntity = await entities.getById('A1', 'en'); + expect(relatedEntity?.metadata?.relationship).toMatchObject([ + { + inheritedValue: [ + { value: 'T2', label: 'T2' }, + { value: 'T3', label: 'T3' }, + ], + }, + { + inheritedValue: [{ value: 'T1', label: 'T1' }], + }, + ]); + }); + + it('should update and index denormalized properties when thesauri label changes', async () => { + await modifyEntity('B1', { + metadata: { multiselect: [{ value: 'T2' }, { value: 'T3' }] }, + }); + await modifyEntity('B2', { + metadata: { multiselect: [{ value: 'T1' }] }, + }); + + await modifyEntity('A1', { + metadata: { + relationship: [factory.metadataValue('B1'), factory.metadataValue('B2')], + }, + }); + + await thesauris.save(factory.thesauri('thesauri', [['T1', 'new 1'], 'T2', ['T3', 'new 3']])); + + await elasticTesting.refresh(); + const results = await elasticTesting.getIndexedEntities(); + + const A1 = results.find(r => r.sharedId === 'A1'); + + expect(A1?.metadata?.relationship).toMatchObject([ + { + inheritedValue: [ + { value: 'T2', label: 'T2' }, + { value: 'T3', label: 'new 3' }, + ], + }, + { + inheritedValue: [{ value: 'T1', label: 'new 1' }], + }, + ]); + }); + }); + + describe('inherited relationship', () => { + beforeEach(async () => { + const fixtures: DBFixture = { + templates: [ + factory.template('templateA', [ + factory.inherit('relationship', 'templateB', 'relationshipB'), + ]), + factory.template('templateB', [factory.relationshipProp('relationshipB', 'templateC')]), + factory.template('templateC'), + factory.template('templateD', [ + factory.inherit('relationshipD', 'templateA', 'relationship'), + ]), + ], + entities: [ + factory.entity('A1', 'templateA', { relationship: [{ value: 'B1' }, { value: 'B2' }] }), + factory.entity('A2', 'templateA', { relationship: [{ value: 'B1' }, { value: 'B2' }] }), + factory.entity('B1', 'templateB'), + factory.entity('B2', 'templateB'), + factory.entity('C1', 'templateC'), + factory.entity('C2', 'templateC'), + ], + }; + await load(fixtures); + await modifyEntity('B1', { metadata: { relationshipB: [{ value: 'C1' }] } }); + await modifyEntity('B2', { metadata: { relationshipB: [{ value: 'C2' }, { value: 'C1' }] } }); + await modifyEntity('A1', { metadata: { relationship: [{ value: 'B1' }, { value: 'B2' }] } }); + await modifyEntity('A2', { metadata: { relationship: [{ value: 'B1' }, { value: 'B2' }] } }); + }); + + it('should update denormalized properties when relationship selected changes', async () => { + const relatedEntity = await entities.getById('A1', 'en'); + expect(relatedEntity?.metadata?.relationship).toMatchObject([ + { inheritedValue: [{ value: 'C1', label: 'C1' }] }, + { + inheritedValue: [ + { value: 'C2', label: 'C2' }, + { value: 'C1', label: 'C1' }, + ], + }, + ]); + }); + + it('should update denormalized properties when relationship inherited label changes', async () => { + await modifyEntity('C1', { title: 'new C1' }); + await modifyEntity('C2', { title: 'new C2' }); + + const relatedEntity = await entities.getById('A1', 'en'); + + expect(relatedEntity?.metadata?.relationship).toMatchObject([ + { inheritedValue: [{ value: 'C1', label: 'new C1' }] }, + { + inheritedValue: [ + { value: 'C2', label: 'new C2' }, + { value: 'C1', label: 'new C1' }, + ], + }, + ]); + }); + }); + + describe('languages and indexation', () => { + beforeEach(async () => { + await load( + { + templates: [ + factory.template('templateA', [ + factory.inherit('relationshipA', 'templateB', 'relationshipB'), + ]), + factory.template('templateB', [factory.inherit('relationshipB', 'templateC', 'text')]), + factory.template('templateC', [factory.property('text')]), + ], + entities: [ + factory.entity('A1', 'templateA', { relationshipA: [factory.metadataValue('B1')] }), + factory.entity( + 'A1', + 'templateA', + { relationshipA: [factory.metadataValue('B1')] }, + { language: 'es' } + ), + factory.entity('B1', 'templateB', { relationshipB: [factory.metadataValue('C1')] }), + factory.entity( + 'B1', + 'templateB', + { relationshipB: [factory.metadataValue('C1')] }, + { language: 'es' } + ), + factory.entity('C1', 'templateC'), + factory.entity('C1', 'templateC', {}, { language: 'es' }), + ], + }, + 'index_denormalization' + ); + + /// generate inherited values ! + + await modifyEntity('B1', { relationshipB: [factory.metadataValue('C1')] }, 'en'); + await modifyEntity('B1', { relationshipB: [factory.metadataValue('C1')] }, 'es'); + + await modifyEntity('A1', { relationshipA: [factory.metadataValue('B1')] }, 'en'); + await modifyEntity('A1', { relationshipA: [factory.metadataValue('B1')] }, 'es'); + + await modifyEntity('C1', { metadata: { text: [{ value: 'text' }] } }); + await modifyEntity('C1', { metadata: { text: [{ value: 'texto' }] } }, 'es'); + + /// generate inherited values ! + }); + + it('should index the correct entities on a simple relationship', async () => { + await modifyEntity( + 'C1', + { title: 'new Es title', metadata: { text: [{ value: 'nuevo texto para ES' }] } }, + 'es' + ); + + await elasticTesting.refresh(); + const results = await elasticTesting.getIndexedEntities(); + + const B1en = results.find(r => r.sharedId === 'B1' && r.language === 'en'); + const B1es = results.find(r => r.sharedId === 'B1' && r.language === 'es'); + + expect(B1en?.metadata?.relationshipB).toMatchObject([ + { value: 'C1', label: 'C1', inheritedValue: [{ value: 'text' }] }, + ]); + expect(B1es?.metadata?.relationshipB).toMatchObject([ + { value: 'C1', label: 'new Es title', inheritedValue: [{ value: 'nuevo texto para ES' }] }, + ]); + }); + + it('should index the correct entities on a transitive relationship', async () => { + await modifyEntity('C1', { title: 'new Es title' }, 'es'); + + await elasticTesting.refresh(); + const results = await elasticTesting.getIndexedEntities(); + + const A1en = results.find(r => r.sharedId === 'A1' && r.language === 'en'); + const A1es = results.find(r => r.sharedId === 'A1' && r.language === 'es'); + + expect(A1en?.metadata?.relationshipA).toMatchObject([ + { value: 'B1', inheritedValue: [{ label: 'C1' }] }, + ]); + + expect(A1es?.metadata?.relationshipA).toMatchObject([ + { value: 'B1', inheritedValue: [{ label: 'new Es title' }] }, + ]); + }); + }); + + describe('when changing a multiselect in one language', () => { + beforeEach(async () => { + await load( + { + templates: [ + factory.template('templateA', [ + factory.inherit('relationshipA', 'templateB', 'multiselect'), + ]), + factory.template('templateB', [ + factory.property('multiselect', 'multiselect', { + content: factory.id('thesauri').toString(), + }), + ]), + ], + dictionaries: [factory.thesauri('thesauri', ['T1', 'T2', 'T3'])], + entities: [ + factory.entity('A1', 'templateA', { relationshipA: [factory.metadataValue('B1')] }), + factory.entity( + 'A1', + 'templateA', + { relationshipA: [factory.metadataValue('B1')] }, + { language: 'es' } + ), + factory.entity('B1', 'templateB', { + multiselect: [factory.metadataValue('T1')], + }), + factory.entity( + 'B1', + 'templateB', + { + multiselect: [factory.metadataValue('T1')], + }, + { language: 'es' } + ), + ], + }, + 'index_denormalization' + ); + }); + + it('should denormalize the VALUE for all the languages', async () => { + await modifyEntity('B1', { + metadata: { multiselect: [{ value: 'T1' }, { value: 'T2' }] }, + }); + + await elasticTesting.refresh(); + const results = await elasticTesting.getIndexedEntities(); + + const A1en = results.find(r => r.sharedId === 'A1' && r.language === 'en'); + const A1es = results.find(r => r.sharedId === 'A1' && r.language === 'es'); + + expect(A1en?.metadata?.relationshipA).toMatchObject([ + { value: 'B1', inheritedValue: [{ value: 'T1' }, { value: 'T2' }] }, + ]); + + expect(A1es?.metadata?.relationshipA).toMatchObject([ + { value: 'B1', inheritedValue: [{ value: 'T1' }, { value: 'T2' }] }, + ]); + }); + }); +}); diff --git a/app/api/entities/specs/entities.spec.js b/app/api/entities/specs/entities.spec.js index 95722d8a4d..24f38f925d 100644 --- a/app/api/entities/specs/entities.spec.js +++ b/app/api/entities/specs/entities.spec.js @@ -15,7 +15,6 @@ import { UserRole } from 'shared/types/userSchema'; import entities from '../entities.js'; import fixtures, { batmanFinishesId, - shared2, templateId, templateChangingNames, syncPropertiesEntityId, @@ -252,98 +251,6 @@ describe('entities', () => { }); }); - describe('when icon changes', () => { - it('should update icon on entities with the entity as relationship', async () => { - const doc = { - _id: shared2, - sharedId: 'shared2', - icon: { - _id: 'changedIcon', - }, - }; - - await entities.save(doc, { language: 'en' }); - let relatedEntity = await entities.getById('shared', 'en'); - expect(relatedEntity.metadata.enemies[0].icon._id).toBe('changedIcon'); - - relatedEntity = await entities.getById('other', 'en'); - expect(relatedEntity.metadata.enemies[1].icon._id).toBe('changedIcon'); - expect(relatedEntity.metadata.enemies[0].icon).toBe(null); - expect(relatedEntity.metadata.enemies[2].icon).toBe(null); - - const updatedDoc = { - _id: shared2, - sharedId: 'shared2', - icon: { - _id: 'changedIconAgain', - }, - }; - - await entities.save(updatedDoc, { language: 'en' }); - relatedEntity = await entities.getById('shared', 'en'); - expect(relatedEntity.metadata.enemies[0].icon._id).toBe('changedIconAgain'); - }); - }); - - describe('when title changes', () => { - it('should update title on entities with the entity as relationship', async () => { - const doc = { - _id: shared2, - sharedId: 'shared2', - title: 'changedTitle', - }; - - await entities.save(doc, { language: 'en' }); - let relatedEntity = await entities.getById('shared', 'en'); - expect(relatedEntity.metadata.enemies[0].label).toBe('changedTitle'); - - relatedEntity = await entities.getById('other', 'en'); - expect(relatedEntity.metadata.enemies[1].label).toBe('changedTitle'); - expect(relatedEntity.metadata.enemies[0].label).toBe('shouldNotChange'); - expect(relatedEntity.metadata.enemies[2].label).toBe('shouldNotChange1'); - }); - - it('should not change related labels on other languages', async () => { - const doc = { - _id: shared2, - sharedId: 'shared2', - title: 'changedTitle', - }; - - await entities.save(doc, { language: 'en' }); - - const relatedEntity = await entities.getById('other', 'es'); - - expect(relatedEntity.metadata.enemies[0].label).toBe('translated1'); - expect(relatedEntity.metadata.enemies[1].label).toBe('translated2'); - }); - - it('should index entities changed after propagating label change', async () => { - const doc = { - _id: shared2, - sharedId: 'shared2', - title: 'changedTitle', - }; - - search.indexEntities.and.callThrough(); - - await entities.save(doc, { language: 'en' }); - - const documentsToIndex = search.bulkIndex.calls.argsFor(0)[0]; - - expect(documentsToIndex).toEqual([ - expect.objectContaining({ - sharedId: 'shared', - language: 'en', - }), - expect.objectContaining({ - sharedId: 'other', - language: 'en', - }), - ]); - }); - }); - describe('when published/template/generatedToc property changes', () => { it('should replicate the change for all the languages and ignore the published field', done => { const doc = { diff --git a/app/api/entities/specs/fixtures.js b/app/api/entities/specs/fixtures.js index 59c350765a..8776ba0562 100644 --- a/app/api/entities/specs/fixtures.js +++ b/app/api/entities/specs/fixtures.js @@ -390,22 +390,24 @@ export default { _id: templateId, name: 'template_test', properties: [ - { type: 'text', name: 'text' }, + { _id: db.id(), type: 'text', name: 'text' }, { _id: inheritedProperty, type: 'text', name: 'property1' }, - { type: 'text', name: 'property2' }, - { type: 'text', name: 'description' }, - { type: 'select', name: 'select', content: dictionary }, - { type: 'multiselect', name: 'multiselect', content: dictionary }, - { type: 'date', name: 'date' }, - { type: 'multidate', name: 'multidate' }, - { type: 'multidaterange', name: 'multidaterange' }, - { type: 'daterange', name: 'daterange' }, + { _id: db.id(), type: 'text', name: 'property2' }, + { _id: db.id(), type: 'text', name: 'description' }, + { _id: db.id(), type: 'select', name: 'select', content: dictionary }, + { _id: db.id(), type: 'multiselect', name: 'multiselect', content: dictionary }, + { _id: db.id(), type: 'date', name: 'date' }, + { _id: db.id(), type: 'multidate', name: 'multidate' }, + { _id: db.id(), type: 'multidaterange', name: 'multidaterange' }, + { _id: db.id(), type: 'daterange', name: 'daterange' }, { + _id: db.id(), type: 'relationship', name: 'friends', relationType: relationType1, }, { + _id: db.id(), type: 'relationship', name: 'enemies', relationType: relationType4, @@ -413,8 +415,8 @@ export default { inherit: true, inheritProperty: inheritedProperty, }, - { type: 'nested', name: 'field_nested' }, - { type: 'numeric', name: 'numeric' }, + { _id: db.id(), type: 'nested', name: 'field_nested' }, + { _id: db.id(), type: 'numeric', name: 'numeric' }, ], }, { @@ -422,6 +424,7 @@ export default { name: 'templateWithOnlyMultiSelectSelect', properties: [ { + _id: db.id(), type: 'relationship', name: 'multiselect', content: templateWithEntityAsThesauri.toString(), @@ -432,23 +435,33 @@ export default { _id: templateWithOnlySelect, name: 'templateWithOnlySelect', properties: [ - { type: 'relationship', name: 'select', content: templateChangingNames.toString() }, + { + _id: db.id(), + type: 'relationship', + name: 'select', + content: templateChangingNames.toString(), + }, ], }, { _id: templateWithEntityAsThesauri, name: 'template_with_thesauri_as_template', properties: [ - { type: 'relationship', name: 'select', content: templateId.toString() }, - { type: 'relationship', name: 'multiselect', content: templateId.toString() }, + { _id: db.id(), type: 'relationship', name: 'select', content: templateId.toString() }, + { _id: db.id(), type: 'relationship', name: 'multiselect', content: templateId.toString() }, ], }, { _id: templateWithEntityAsThesauri2, name: 'template_with_thesauri_as_template', properties: [ - { type: 'relationship', name: 'select2', content: templateId.toString() }, - { type: 'relationship', name: 'multiselect2', content: templateId.toString() }, + { _id: db.id(), type: 'relationship', name: 'select2', content: templateId.toString() }, + { + _id: db.id(), + type: 'relationship', + name: 'multiselect2', + content: templateId.toString(), + }, ], }, { @@ -456,23 +469,23 @@ export default { name: 'template_changing_names', default: true, properties: [ - { id: '1', type: 'text', name: 'property1' }, - { id: '2', type: 'text', name: 'property2' }, - { id: '3', type: 'text', name: 'property3' }, + { _id: db.id(), id: '1', type: 'text', name: 'property1' }, + { _id: db.id(), id: '2', type: 'text', name: 'property2' }, + { _id: db.id(), id: '3', type: 'text', name: 'property3' }, ], }, ], connections: [ { _id: referenceId, entity: 'shared', template: null, hub: hub1, entityData: {} }, - { entity: 'shared2', template: relationType1, hub: hub1, entityData: {} }, - { entity: 'shared', template: null, hub: hub2, entityData: {} }, - { entity: 'source2', template: relationType2, hub: hub2, entityData: {} }, - { entity: 'another', template: relationType3, hub: hub3, entityData: {} }, - { entity: 'document', template: relationType3, hub: hub3, entityData: {} }, - { entity: 'shared', template: relationType2, hub: hub4, entityData: {} }, - { entity: 'shared1', template: relationType2, hub: hub4, entityData: {} }, - { entity: 'shared1', template: relationType2, hub: hub5, entityData: {} }, - { entity: 'shared', template: relationType2, hub: hub5, entityData: {} }, + { _id: db.id(), entity: 'shared2', template: relationType1, hub: hub1, entityData: {} }, + { _id: db.id(), entity: 'shared', template: null, hub: hub2, entityData: {} }, + { _id: db.id(), entity: 'source2', template: relationType2, hub: hub2, entityData: {} }, + { _id: db.id(), entity: 'another', template: relationType3, hub: hub3, entityData: {} }, + { _id: db.id(), entity: 'document', template: relationType3, hub: hub3, entityData: {} }, + { _id: db.id(), entity: 'shared', template: relationType2, hub: hub4, entityData: {} }, + { _id: db.id(), entity: 'shared1', template: relationType2, hub: hub4, entityData: {} }, + { _id: db.id(), entity: 'shared1', template: relationType2, hub: hub5, entityData: {} }, + { _id: db.id(), entity: 'shared', template: relationType2, hub: hub5, entityData: {} }, ], dictionaries: [ { diff --git a/app/api/evidences_vault/specs/fixtures.js b/app/api/evidences_vault/specs/fixtures.js index 67cbe3ef84..16ada31dc9 100644 --- a/app/api/evidences_vault/specs/fixtures.js +++ b/app/api/evidences_vault/specs/fixtures.js @@ -12,26 +12,31 @@ export default { name: 'template', properties: [ { + _id: db.id(), type: propertyTypes.media, label: 'video', name: templateUtils.safeName('video'), }, { + _id: db.id(), type: propertyTypes.link, label: 'original url', name: templateUtils.safeName('original url'), }, { + _id: db.id(), type: propertyTypes.image, label: 'screenshot', name: templateUtils.safeName('screenshot'), }, { + _id: db.id(), type: propertyTypes.date, label: 'time of request', name: templateUtils.safeName('time of request'), }, { + _id: db.id(), type: propertyTypes.markdown, label: 'data', name: templateUtils.safeName('data'), diff --git a/app/api/templates/templates.ts b/app/api/templates/templates.ts index bc2e5724e3..3d226bae2c 100644 --- a/app/api/templates/templates.ts +++ b/app/api/templates/templates.ts @@ -67,6 +67,40 @@ const updateTranslation = async (currentTemplate: TemplateSchema, template: Temp }; export default { + async propsThatNeedDenormalization(templateId: string) { + return ( + await model.get({ + $or: [{ 'properties.content': templateId }, { 'properties.content': '' }], + }) + ).reduce<{ [k: string]: string | undefined }[]>( + (m, t) => + m.concat( + t.properties + ?.filter(p => templateId === p.content?.toString() || p.content === '') + .map(p => ({ + name: p.name, + inheritProperty: p.inheritProperty, + template: t._id, + })) || [] + ), + [] + ); + }, + + async propsThatNeedTransitiveDenormalization(contentId: string) { + const properties = (await model.get({ 'properties.content': contentId })) + .reduce((m, t) => m.concat(t.properties || []), []) + .filter(p => contentId === p.content?.toString()); + + return ( + await model.get({ + 'properties.inheritProperty': { + $in: properties.map(p => p._id?.toString()).filter(v => v), + }, + }) + ).reduce((m, t) => m.concat(t.properties || []), []); + }, + async save(template: TemplateSchema, language: string, reindex = true) { /* eslint-disable no-param-reassign */ template.properties = template.properties || []; diff --git a/app/api/utils/fixturesFactory.ts b/app/api/utils/fixturesFactory.ts new file mode 100644 index 0000000000..998c05d3b1 --- /dev/null +++ b/app/api/utils/fixturesFactory.ts @@ -0,0 +1,87 @@ +import { ObjectId } from 'mongodb'; +import db from 'api/utils/testing_db'; +import { EntitySchema } from 'shared/types/entityType'; +import { PropertySchema, MetadataSchema } from 'shared/types/commonTypes'; + +export function getIdMapper() { + const map = new Map(); + + return function setAndGet(key: string) { + if (!map.has(key)) map.set(key, db.id() as ObjectId); + + return map.get(key)!; + }; +} + +export function getFixturesFactory() { + const idMapper = getIdMapper(); + + return Object.freeze({ + id: idMapper, + + template: (name: string, properties: PropertySchema[] = []) => ({ + _id: idMapper(name), + properties, + }), + + entity: ( + id: string, + template?: string, + metadata: MetadataSchema = {}, + props: EntitySchema = { language: 'en' } + ): EntitySchema => { + const language = props.language || 'en'; + return { + _id: idMapper(`${id}-${language}`), + sharedId: id, + title: `${id}`, + ...(template ? { template: idMapper(template) } : {}), + metadata, + language, + ...props, + }; + }, + + inherit(name: string, content: string, property: string, props = {}): PropertySchema { + return this.relationshipProp(name, content, { + // inherit: { property: idMapper(property).toString() }, + inheritProperty: idMapper(property).toString(), + inherit: true, + ...props, + }); + }, + + relationshipProp(name: string, content: string, props = {}): PropertySchema { + return this.property(name, 'relationship', { + relationType: idMapper('rel1').toString(), + content: idMapper(content).toString(), + ...props, + }); + }, + + property: ( + name: string, + type: PropertySchema['type'] = 'text', + props = {} + ): PropertySchema => ({ + _id: idMapper(name), + id: name, + label: name, + name, + type, + ...props, + }), + + metadataValue: (value: string) => ({ value, label: value }), + + thesauri: (name: string, values: Array) => ({ + name, + _id: idMapper(name), + values: values.map(value => + typeof value === 'string' + ? { _id: idMapper(value), id: value, label: value } + : { _id: idMapper(value[0]), id: value[0], label: value[1] } + ), + }), + }); +} diff --git a/app/api/utils/specs/fixturesFactory.spec.ts b/app/api/utils/specs/fixturesFactory.spec.ts new file mode 100644 index 0000000000..c7ec3061ef --- /dev/null +++ b/app/api/utils/specs/fixturesFactory.spec.ts @@ -0,0 +1,35 @@ +import { getIdMapper, getFixturesFactory } from '../fixturesFactory'; + +describe('getIdMapper', () => { + it('should create a new id', () => { + const ids = getIdMapper(); + + expect(ids('key')).toBeDefined(); + }); + + it('should create a different ids', () => { + const ids = getIdMapper(); + + expect(ids('key1')).not.toEqual(ids('key2')); + }); + + it('should cache ids', () => { + const ids = getIdMapper(); + + expect(ids('key')).toEqual(ids('key')); + }); +}); + +describe('getFixturesFactory', () => { + it('should return different instances', () => { + expect(getFixturesFactory()).not.toBe(getFixturesFactory); + }); + + it('should map the ids correctly encapsulated in the instance', () => { + const factory1 = getFixturesFactory(); + const factory2 = getFixturesFactory(); + + expect(factory1.id('some')).toBe(factory1.id('some')); + expect(factory1.id('some')).not.toBe(factory2.id('some')); + }); +});