diff --git a/app/api/csv/arrangeThesauri.ts b/app/api/csv/arrangeThesauri.ts new file mode 100644 index 0000000000..9dcf1504a1 --- /dev/null +++ b/app/api/csv/arrangeThesauri.ts @@ -0,0 +1,142 @@ +import { ImportFile } from 'api/csv/importFile'; +import { WithId } from 'api/odm'; +import thesauri from 'api/thesauri'; +import { PropertySchema } from 'shared/types/commonTypes'; +import { TemplateSchema } from 'shared/types/templateType'; +import { ThesaurusSchema } from 'shared/types/thesaurusType'; + +import csv, { CSVRow } from './csv'; +import { splitMultiselectLabels } from './typeParsers/multiselect'; +import { normalizeThesaurusLabel } from './typeParsers/select'; + +const filterJSObject = (input: { [k: string]: any }, keys: string[]): { [k: string]: any } => { + const result: { [k: string]: any } = {}; + keys.forEach(k => { + if (input.hasOwnProperty(k)) { + result[k] = input[k]; + } + }); + return result; +}; + +class ArrangeThesauriError extends Error { + source: Error; + row: CSVRow; + index: number; + + constructor(source: Error, row: CSVRow, index: number) { + super(source.message); + this.source = source; + this.row = row; + this.index = index; + } +} + +const createNameToIdMap = ( + thesauriRelatedProperties: PropertySchema[] | undefined, + languages?: string[] +): { [k: string]: string } => { + const nameToThesauriId: { [k: string]: string } = {}; + + thesauriRelatedProperties?.forEach(p => { + if (p.content && p.type) { + const thesarusID = p.content.toString(); + nameToThesauriId[p.name] = thesarusID; + languages?.forEach(suffix => { + nameToThesauriId[`${p.name}__${suffix}`] = thesarusID; + }); + } + }); + + return nameToThesauriId; +}; + +type ThesauriValueData = { + thesauriIdToExistingValues: Map>; + thesauriIdToNewValues: Map>; + thesauriIdToNormalizedNewValues: Map>; +}; + +const setupIdValueMaps = (allRelatedThesauri: WithId[]): ThesauriValueData => { + const thesauriIdToExistingValues = new Map(); + const thesauriIdToNewValues = new Map(); + const thesauriIdToNormalizedNewValues = new Map(); + + allRelatedThesauri.forEach(t => { + const id = t._id.toString(); + thesauriIdToExistingValues.set( + id, + new Set(t.values?.map(v => normalizeThesaurusLabel(v.label))) + ); + thesauriIdToNewValues.set(id, new Set()); + thesauriIdToNormalizedNewValues.set(id, new Set()); + }); + + return { thesauriIdToExistingValues, thesauriIdToNewValues, thesauriIdToNormalizedNewValues }; +}; + +const syncSaveThesauri = async ( + allRelatedThesauri: WithId[], + thesauriIdToNewValues: Map> +) => { + const thesauriWithNewValues = allRelatedThesauri.filter( + t => (thesauriIdToNewValues.get(t._id.toString()) || new Set()).size > 0 + ); + for (let i = 0; i < thesauriWithNewValues.length; i += 1) { + const thesaurus = allRelatedThesauri[i]; + const newValues = Array.from( + thesauriIdToNewValues.get(thesaurus._id.toString()) || [] + ).map(tval => ({ label: tval })); + const thesaurusValues = thesaurus.values || []; + // eslint-disable-next-line no-await-in-loop + await thesauri.save({ + ...thesaurus, + values: thesaurusValues.concat(newValues), + }); + } +}; + +const arrangeThesauri = async ( + file: ImportFile, + template: TemplateSchema, + languages?: string[], + stopOnError: boolean = true +) => { + const thesauriRelatedProperties = template.properties?.filter(p => + ['select', 'multiselect'].includes(p.type) + ); + + const nameToThesauriId = createNameToIdMap(thesauriRelatedProperties, languages); + + const allRelatedThesauri = await thesauri.get({ + $in: Array.from( + new Set(thesauriRelatedProperties?.map(p => p.content?.toString()).filter(t => t)) + ), + }); + + const thesauriValueData = setupIdValueMaps(allRelatedThesauri); + + await csv(await file.readStream(), stopOnError) + .onRow(async (row: CSVRow) => { + Object.entries(filterJSObject(nameToThesauriId, Object.keys(row))).forEach(([name, id]) => { + const labels = splitMultiselectLabels(row[name]); + Object.entries(labels).forEach(([normalizedLabel, originalLabel]) => { + if ( + !thesauriValueData.thesauriIdToExistingValues.get(id)?.has(normalizedLabel) && + !thesauriValueData.thesauriIdToNormalizedNewValues.get(id)?.has(normalizedLabel) + ) { + thesauriValueData.thesauriIdToNewValues.get(id)?.add(originalLabel); + thesauriValueData.thesauriIdToNormalizedNewValues.get(id)?.add(normalizedLabel); + } + }); + }); + }) + .onError(async (e: Error, row: CSVRow, index: number) => { + throw new ArrangeThesauriError(e, row, index); + }) + .read(); + + await syncSaveThesauri(allRelatedThesauri, thesauriValueData.thesauriIdToNewValues); +}; + +export { arrangeThesauri, ArrangeThesauriError }; diff --git a/app/api/csv/csvLoader.ts b/app/api/csv/csvLoader.ts index 048a3f89bf..b05d01d932 100644 --- a/app/api/csv/csvLoader.ts +++ b/app/api/csv/csvLoader.ts @@ -10,6 +10,7 @@ import { ThesaurusSchema } from 'shared/types/thesaurusType'; import { ensure } from 'shared/tsUtils'; import { ObjectId } from 'mongodb'; +import { arrangeThesauri } from './arrangeThesauri'; import csv, { CSVRow } from './csv'; import importFile from './importFile'; import { importEntity, translateEntity } from './importEntity'; @@ -55,6 +56,7 @@ export class CSVLoader extends EventEmitter { (await settings.get()).languages ).map((l: LanguageSchema) => l.key); const { newNameGeneration = false } = await settings.get(); + await arrangeThesauri(file, template, availableLanguages); await csv(await file.readStream(), this.stopOnError) .onRow(async (row: CSVRow) => { @@ -64,7 +66,6 @@ export class CSVLoader extends EventEmitter { options.language, newNameGeneration ); - if (rawEntity) { const entity = await importEntity(rawEntity, template, file, options); await translateEntity(entity, rawTranslations, template, file); diff --git a/app/api/csv/importEntity.ts b/app/api/csv/importEntity.ts index 62fdad0077..3a058c3ca8 100644 --- a/app/api/csv/importEntity.ts +++ b/app/api/csv/importEntity.ts @@ -5,13 +5,13 @@ import { processDocument } from 'api/files/processDocument'; import { RawEntity } from 'api/csv/entityRow'; import { TemplateSchema } from 'shared/types/templateType'; import { MetadataSchema, PropertySchema } from 'shared/types/commonTypes'; +import { propertyTypes } from 'shared/propertyTypes'; import { ImportFile } from 'api/csv/importFile'; import { EntitySchema } from 'shared/types/entityType'; import { ensure } from 'shared/tsUtils'; - import { attachmentsPath, files } from 'api/files'; -import { propertyTypes } from 'shared/propertyTypes'; import { generateID } from 'shared/IDGenerator'; + import typeParsers from './typeParsers'; const parse = async (toImportEntity: RawEntity, prop: PropertySchema) => diff --git a/app/api/csv/importFile.ts b/app/api/csv/importFile.ts index 5ab2db8a81..2ee03a7ffd 100644 --- a/app/api/csv/importFile.ts +++ b/app/api/csv/importFile.ts @@ -1,6 +1,5 @@ import fs from 'fs'; import path from 'path'; -import { Readable } from 'stream'; import { generateFileName, fileFromReadStream, uploadsPath } from 'api/files/filesystem'; import { createError } from 'api/utils'; @@ -17,16 +16,13 @@ const extractFromZip = async (zipPath: string, fileName: string) => { }; export class ImportFile { - filePath: string | Readable; + filePath: string; - constructor(filePath: string | Readable) { + constructor(filePath: string) { this.filePath = filePath; } async readStream(fileName = 'import.csv') { - if (this.filePath instanceof Readable) { - return this.filePath; - } if (path.extname(this.filePath) === '.zip') { return extractFromZip(this.filePath, fileName); } @@ -46,6 +42,6 @@ export class ImportFile { } } -const importFile = (filePath: string | Readable) => new ImportFile(filePath); +const importFile = (filePath: string) => new ImportFile(filePath); export default importFile; diff --git a/app/api/csv/specs/__snapshots__/importFile.spec.ts.snap b/app/api/csv/specs/__snapshots__/importFile.spec.ts.snap index 748622c07e..5b4864d04b 100644 --- a/app/api/csv/specs/__snapshots__/importFile.spec.ts.snap +++ b/app/api/csv/specs/__snapshots__/importFile.spec.ts.snap @@ -1,11 +1,11 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP exports[`importFile readStream should return a readable stream for the csv file 1`] = ` -"Title , text label , numeric label, non configured, select label, not defined type, geolocation_geolocation,auto id, additional tag(s) +"Title , text label , numeric label, non configured, select_label, not defined type, geolocation_geolocation,auto id, additional tag(s), multi_select_label -title1, text value 1, 1977, ______________, thesauri1 , notType1 , 1|1,,tag1 -title2, text value 2, 2019, ______________, thesauri2 , notType2 , ,,tag2 -title3, text value 3, 2020, ______________, thesauri2 , notType3 , 0|0,,tag3 +title1, text value 1, 1977, ______________, thesauri1 , notType1 , 1|1,,tag1, multivalue1 +title2, text value 2, 2019, ______________, thesauri2 , notType2 , ,,tag2, multivalue2 +title3, text value 3, 2020, ______________, thesauri2 , notType3 , 0|0,,tag3, multivalue1|multivalue3 " `; diff --git a/app/api/csv/specs/arrangeThesauriTest.csv b/app/api/csv/specs/arrangeThesauriTest.csv new file mode 100644 index 0000000000..20efec600f --- /dev/null +++ b/app/api/csv/specs/arrangeThesauriTest.csv @@ -0,0 +1,17 @@ +title,unrelated_property,select_property__en, select_property__es,multiselect_property__en, multiselect_property__es +select_1,unrelated_text,B,Bes,A,Aes +select_2,unrelated_text,C,Ces,A,Aes +select_3,unrelated_text,b,bes,A,Aes +select_4,unrelated_text,B,Bes,A,Aes +select_5,unrelated_text,d,des,A,Aes +select_6,unrelated_text,D,Des,A,Aes +select_7,unrelated_text, b,bes,A,Aes +select_8,unrelated_text, , ,A,Aes +select_8,unrelated_text, , ,A,Aes +multiselect_1,unrelated_text,A,Aes,B,Bes +multiselect_2,unrelated_text,A,Aes,c,ces +multiselect_3,unrelated_text,A,Aes,A|b,Aes|bes +multiselect_4,unrelated_text,A,Aes,a|B|C,aes|Bes|Ces +multiselect_5,unrelated_text,A,Aes, a| b | , aes| bes | +multiselect_6,unrelated_text,A,Aes, | | , | | +multiselect_7, unrelated_text,A,Aes,A|B|C|D| |E| e| g ,Aes|Bes|Ces|Des| |Ees| ees| ges diff --git a/app/api/csv/specs/csvLoader.spec.js b/app/api/csv/specs/csvLoader.spec.js index b94f08ca01..10243947ca 100644 --- a/app/api/csv/specs/csvLoader.spec.js +++ b/app/api/csv/specs/csvLoader.spec.js @@ -1,14 +1,15 @@ /* eslint-disable max-lines */ +import path from 'path'; + import db from 'api/utils/testing_db'; import entities from 'api/entities'; -import path from 'path'; import translations from 'api/i18n'; import { search } from 'api/search'; import { CSVLoader } from 'api/csv'; import { templateWithGeneratedTitle } from 'api/csv/specs/csvLoaderFixtures'; import fixtures, { template1Id } from './csvLoaderFixtures'; -import { stream } from './helpers'; +import { mockCsvFileReadStream } from './helpers'; import typeParsers from '../typeParsers'; describe('csvLoader', () => { @@ -18,7 +19,6 @@ describe('csvLoader', () => { beforeAll(async () => { await db.clearAllAndLoad(fixtures); spyOn(search, 'indexEntities').and.returnValue(Promise.resolve()); - spyOn(translations, 'updateContext').and.returnValue(Promise.resolve()); }); afterAll(async () => db.disconnect()); @@ -33,6 +33,7 @@ describe('csvLoader', () => { describe('load translations', () => { let csv; + let readStreamMock; beforeEach(async () => { await db.clearAllAndLoad(fixtures); @@ -44,8 +45,13 @@ describe('csvLoader', () => { original 3, value 3, valor 3, valeur 3, 3 ,`; }); + afterEach(() => { + readStreamMock.mockRestore(); + }); + it('should set all translations from csv', async () => { - await loader.loadTranslations(stream(csv), 'System'); + readStreamMock = mockCsvFileReadStream(csv); + await loader.loadTranslations('mockedFileFromString', 'System'); const [english, spanish, french] = await translations.get(); expect(english.contexts[0].values).toEqual({ 'original 1': 'value 1', @@ -65,8 +71,9 @@ describe('csvLoader', () => { }); it('should not update a language that exists in the system but not in csv', async () => { + readStreamMock = mockCsvFileReadStream(csv); await translations.addLanguage('aa'); - await loader.loadTranslations(stream(csv), 'System'); + await loader.loadTranslations('mockedFileFromString', 'System'); const [afar] = await translations.get({ locale: 'aa' }); expect(afar.contexts[0].values).toEqual({ 'original 1': 'original 1', @@ -78,8 +85,8 @@ describe('csvLoader', () => { it('should not remove translations that are not in the csv', async () => { const localCsv = `Key, English, original 1, value 1`; - - await loader.loadTranslations(stream(localCsv), 'System'); + readStreamMock = mockCsvFileReadStream(localCsv); + await loader.loadTranslations('mockedFileFromString', 'System'); const [english] = await translations.get(); expect(english.contexts[0].values).toEqual({ @@ -91,8 +98,8 @@ describe('csvLoader', () => { it('should not import empty language translations', async () => { const localCsv = `Key, English, Spanish original 1,, sp value 1`; - - await loader.loadTranslations(stream(localCsv), 'System'); + readStreamMock = mockCsvFileReadStream(localCsv); + await loader.loadTranslations('mockedFileFromString', 'System'); const [english, spanish] = await translations.get(); expect(english.contexts[0].values).toEqual({ @@ -116,7 +123,6 @@ describe('csvLoader', () => { loader.on('entityLoaded', entity => { events.push(entity.title); }); - try { await loader.load(csvFile, template1Id, { language: 'en' }); } catch (e) { @@ -153,6 +159,7 @@ describe('csvLoader', () => { 'geolocation_geolocation', 'auto_id', 'additional_tag(s)', + 'multi_select_label', ]); }); @@ -288,7 +295,7 @@ describe('csvLoader', () => { }); describe('when sharedId is provided', () => { - it('should update the entitiy', async () => { + it('should update the entity', async () => { const entity = await entities.save( { title: 'entity4444', template: template1Id }, { user: {}, language: 'en' } @@ -296,28 +303,34 @@ describe('csvLoader', () => { const csv = `id , title , ${entity.sharedId}, new title, , title2 ,`; - + const readStreamMock = mockCsvFileReadStream(csv); const testingLoader = new CSVLoader(); - await testingLoader.load(stream(csv), template1Id, { language: 'en' }); + await testingLoader.load('mockedFileFromString', template1Id, { language: 'en' }); const [expected] = await entities.get({ sharedId: entity.sharedId, language: 'en', }); expect(expected.title).toBe('new title'); + readStreamMock.mockRestore(); }); }); describe('when the title is not provided', () => { + let readStreamMock; + afterEach(() => { + readStreamMock.mockRestore(); + }); describe('title not marked with generated Id option', () => { it('should throw a validation error', async () => { const csv = `title , numeric label , 10 title2, 10`; + readStreamMock = mockCsvFileReadStream(csv); const testingLoader = new CSVLoader(); try { - await testingLoader.load(stream(csv), template1Id, { language: 'en' }); + await testingLoader.load('mockedFileFromString', template1Id, { language: 'en' }); } catch (e) { expect(e.message).toEqual('validation failed'); expect(e.errors[0].dataPath).toEqual('.title'); @@ -329,9 +342,12 @@ describe('csvLoader', () => { const csv = `title , numeric label , 10 title2, 10`; + readStreamMock = mockCsvFileReadStream(csv); const testingLoader = new CSVLoader(); - await testingLoader.load(stream(csv), templateWithGeneratedTitle, { language: 'en' }); + await testingLoader.load('mockedFileFromString', templateWithGeneratedTitle, { + language: 'en', + }); const result = await entities.get({ 'metadata.numeric_label.value': 10, language: 'en', @@ -343,8 +359,11 @@ describe('csvLoader', () => { const csv = `numeric label 20 22`; + readStreamMock = mockCsvFileReadStream(csv); const testingLoader = new CSVLoader(); - await testingLoader.load(stream(csv), templateWithGeneratedTitle, { language: 'en' }); + await testingLoader.load('mockedFileFromString', templateWithGeneratedTitle, { + language: 'en', + }); const result = await entities.get({ 'metadata.numeric_label.value': { $in: [20, 22] }, language: 'en', diff --git a/app/api/csv/specs/csvLoaderFixtures.ts b/app/api/csv/specs/csvLoaderFixtures.ts index ff02a39835..95ee8f4796 100644 --- a/app/api/csv/specs/csvLoaderFixtures.ts +++ b/app/api/csv/specs/csvLoaderFixtures.ts @@ -3,10 +3,38 @@ import { propertyTypes } from 'shared/propertyTypes'; import { templateUtils } from 'api/templates'; const template1Id = db.id(); +const multiSelectThesaurusId = db.id(); const thesauri1Id = db.id(); const templateToRelateId = db.id(); const templateWithGeneratedTitle = db.id(); +const commonTranslationContexts = [ + { + id: 'System', + label: 'System', + values: [ + { key: 'original 1', value: 'original 1' }, + { key: 'original 2', value: 'original 2' }, + { key: 'original 3', value: 'original 3' }, + ], + }, + { + id: thesauri1Id.toString(), + label: 'thesauri1', + values: [{ key: 'thesauri1', value: 'thesauri1' }], + type: 'Dictionary', + }, + { + id: multiSelectThesaurusId.toString(), + label: 'multi_select_thesaurus', + values: [ + { key: 'multi_select_thesaurus', value: 'multi_select_thesaurus' }, + { key: 'multivalue1', value: 'multivalue1' }, + ], + type: 'Dictionary', + }, +]; + export default { templates: [ { @@ -32,7 +60,7 @@ export default { type: propertyTypes.select, label: 'select label', name: templateUtils.safeName('select label'), - content: thesauri1Id, + content: thesauri1Id.toString(), }, { type: 'non_defined_type', @@ -60,6 +88,12 @@ export default { label: 'additional tag(s)', name: templateUtils.safeName('additional tag(s)', true), }, + { + type: propertyTypes.multiselect, + label: 'multi_select_label', + name: templateUtils.safeName('multi_select_label'), + content: multiSelectThesaurusId.toString(), + }, ], }, { @@ -87,6 +121,16 @@ export default { }, ], }, + { + _id: multiSelectThesaurusId, + name: 'multi_select_thesaurus', + values: [ + { + label: 'multivalue1', + id: db.id().toString(), + }, + ], + }, ], settings: [ @@ -106,49 +150,19 @@ export default { { _id: db.id(), locale: 'en', - contexts: [ - { - id: 'System', - label: 'System', - values: [ - { key: 'original 1', value: 'original 1' }, - { key: 'original 2', value: 'original 2' }, - { key: 'original 3', value: 'original 3' }, - ], - }, - ], + contexts: commonTranslationContexts, }, { _id: db.id(), locale: 'es', - contexts: [ - { - id: 'System', - label: 'System', - values: [ - { key: 'original 1', value: 'original 1' }, - { key: 'original 2', value: 'original 2' }, - { key: 'original 3', value: 'original 3' }, - ], - }, - ], + contexts: commonTranslationContexts, }, { _id: db.id(), locale: 'fr', - contexts: [ - { - id: 'System', - label: 'System', - values: [ - { key: 'original 1', value: 'original 1' }, - { key: 'original 2', value: 'original 2' }, - { key: 'original 3', value: 'original 3' }, - ], - }, - ], + contexts: commonTranslationContexts, }, ], }; -export { template1Id, templateWithGeneratedTitle }; +export { template1Id, templateWithGeneratedTitle, thesauri1Id }; diff --git a/app/api/csv/specs/csvLoaderSelects.spec.js b/app/api/csv/specs/csvLoaderSelects.spec.js new file mode 100644 index 0000000000..f7629fb247 --- /dev/null +++ b/app/api/csv/specs/csvLoaderSelects.spec.js @@ -0,0 +1,200 @@ +import path from 'path'; + +import translations from 'api/i18n/translations'; +import thesauri from 'api/thesauri'; +import { getFixturesFactory } from 'api/utils/fixturesFactory'; +import db from 'api/utils/testing_db'; + +import { CSVLoader } from '../csvLoader'; + +const fixtureFactory = getFixturesFactory(); + +const commonTranslationContexts = (id1, id2) => [ + { + id: 'System', + label: 'System', + values: [ + { key: 'original 1', value: 'original 1' }, + { key: 'original 2', value: 'original 2' }, + { key: 'original 3', value: 'original 3' }, + ], + }, + { + id: id1.toString(), + label: 'select_thesaurus', + values: [ + { key: 'select_thesaurus', value: 'select_thesaurus' }, + { key: 'A', value: 'A' }, + ], + type: 'Dictionary', + }, + { + id: id2.toString(), + label: 'multiselect_thesaurus', + values: [ + { key: 'multiselect_thesaurus', value: 'multiselect_thesaurus' }, + { key: 'A', value: 'A' }, + { key: 'B', value: 'B' }, + ], + type: 'Dictionary', + }, +]; + +const fixtures = { + dictionaries: [ + fixtureFactory.thesauri('select_thesaurus', ['A']), + fixtureFactory.thesauri('multiselect_thesaurus', ['A', 'B']), + ], + templates: [ + fixtureFactory.template('template', [ + fixtureFactory.property('unrelated_property', 'text'), + fixtureFactory.property('select_property', 'select', { + content: fixtureFactory.id('select_thesaurus'), + }), + fixtureFactory.property('multiselect_property', 'multiselect', { + content: fixtureFactory.id('multiselect_thesaurus'), + }), + ]), + fixtureFactory.template('no_selects_template', [ + fixtureFactory.property('unrelated_property', 'text'), + ]), + ], + entities: [ + fixtureFactory.entity('existing_entity_id', 'template', { + unrelated_property: fixtureFactory.metadataValue('unrelated_value'), + select_property: fixtureFactory.metadataValue('A'), + multiselect_property: [fixtureFactory.metadataValue('A'), fixtureFactory.metadataValue('B')], + }), + ], + settings: [ + { + _id: db.id(), + site_name: 'Uwazi', + languages: [ + { key: 'en', label: 'English', default: true }, + { key: 'es', label: 'Spanish' }, + ], + }, + ], + translations: [ + { + _id: db.id(), + locale: 'en', + contexts: commonTranslationContexts( + fixtureFactory.id('select_thesaurus'), + fixtureFactory.id('multiselect_thesaurus') + ), + }, + { + _id: db.id(), + locale: 'es', + contexts: commonTranslationContexts( + fixtureFactory.id('select_thesaurus'), + fixtureFactory.id('multiselect_thesaurus') + ), + }, + ], +}; + +const loader = new CSVLoader(); + +describe('loader', () => { + let fileSpy; + let selectThesaurus; + let selectLabels; + let selectLabelsSet; + let multiselectThesaurus; + let multiselectLabels; + let multiselectLabelsSet; + + beforeAll(async () => { + await db.clearAllAndLoad(fixtures); + await loader.load( + path.join(__dirname, '/arrangeThesauriTest.csv'), + fixtureFactory.id('template') + ); + selectThesaurus = await thesauri.getById(fixtureFactory.id('select_thesaurus')); + selectLabels = selectThesaurus.values.map(tv => tv.label); + selectLabelsSet = new Set(selectLabels); + multiselectThesaurus = await thesauri.getById(fixtureFactory.id('multiselect_thesaurus')); + multiselectLabels = multiselectThesaurus.values.map(tv => tv.label); + multiselectLabelsSet = new Set(multiselectLabels); + }); + afterAll(async () => { + db.disconnect(); + fileSpy.mockRestore(); + }); + + it('should create values in thesauri', async () => { + expect(selectLabels).toEqual(['A', 'B', 'Bes', 'C', 'Ces', 'd', 'des', 'Aes']); + expect(multiselectLabels).toEqual([ + 'A', + 'B', + 'Aes', + 'Bes', + 'c', + 'ces', + 'D', + 'E', + 'g', + 'Des', + 'Ees', + 'ges', + ]); + }); + + it('should not repeat case sensitive values', async () => { + ['a', 'aes', 'b', 'bes', 'c', 'ces', 'D', 'Des'].forEach(letter => + expect(selectLabelsSet.has(letter)).toBe(false) + ); + ['a', 'aes', 'b', 'bes', 'C', 'Ces', 'd', 'des', 'e', 'ees', 'G', 'Ges'].forEach(letter => + expect(multiselectLabelsSet.has(letter)).toBe(false) + ); + }); + + it('should not add values with trimmable white space or blank values', async () => { + selectLabels.forEach(label => { + expect(label).toBe(label.trim()); + }); + multiselectLabels.forEach(label => { + expect(label).toBe(label.trim()); + }); + }); + + it('should not create repeated values', async () => { + expect(selectLabels.length).toBe(selectLabelsSet.size); + expect(multiselectLabels.length).toBe(multiselectLabelsSet.size); + }); + + it('should check that the thesauri saving saves all contexts properly', async () => { + const trs = await translations.get(); + trs.forEach(tr => { + expect(tr.contexts.find(c => c.label === 'select_thesaurus').values).toMatchObject({ + A: 'A', + Aes: 'Aes', + B: 'B', + Bes: 'Bes', + C: 'C', + Ces: 'Ces', + d: 'd', + des: 'des', + select_thesaurus: 'select_thesaurus', + }); + expect(tr.contexts.find(c => c.label === 'multiselect_thesaurus').values).toMatchObject({ + A: 'A', + Aes: 'Aes', + B: 'B', + Bes: 'Bes', + D: 'D', + Des: 'Des', + E: 'E', + Ees: 'Ees', + c: 'c', + ces: 'ces', + g: 'g', + ges: 'ges', + multiselect_thesaurus: 'multiselect_thesaurus', + }); + }); + }); +}); diff --git a/app/api/csv/specs/csvLoaderThesauri.spec.js b/app/api/csv/specs/csvLoaderThesauri.spec.js index 4442ed7e08..b7ceaed8b0 100644 --- a/app/api/csv/specs/csvLoaderThesauri.spec.js +++ b/app/api/csv/specs/csvLoaderThesauri.spec.js @@ -5,7 +5,7 @@ import settings from 'api/settings'; import { CSVLoader } from '../csvLoader'; import fixtures from './fixtures'; -import { stream } from './helpers'; +import { mockCsvFileReadStream } from './helpers'; describe('csvLoader thesauri', () => { const loader = new CSVLoader(); @@ -37,7 +37,9 @@ describe('csvLoader thesauri', () => { value 3, valor 3, valeur 3, 3 ,`; thesauriId = _id; - result = await loader.loadThesauri(stream(csv), _id, { language: 'en' }); + const mockedFile = mockCsvFileReadStream(csv); + result = await loader.loadThesauri('mockedFileFromString', _id, { language: 'en' }); + mockedFile.mockRestore(); }); const getTranslation = async lang => diff --git a/app/api/csv/specs/fixtures.js b/app/api/csv/specs/fixtures.js index c630fffa32..10fcf04adc 100644 --- a/app/api/csv/specs/fixtures.js +++ b/app/api/csv/specs/fixtures.js @@ -70,6 +70,18 @@ export default { _id: thesauri1Id, name: 'thesauri1', values: [ + { + label: 'value1', + id: db.id().toString(), + }, + { + label: 'value2', + id: db.id().toString(), + }, + { + label: 'Value3', + id: db.id().toString(), + }, { label: ' value4 ', id: db.id().toString(), diff --git a/app/api/csv/specs/helpers.js b/app/api/csv/specs/helpers.js index 8d55a7a108..b4ed9ac2c6 100644 --- a/app/api/csv/specs/helpers.js +++ b/app/api/csv/specs/helpers.js @@ -18,12 +18,25 @@ const createTestingZip = (filesToZip, fileName, directory = __dirname) => .on('error', reject); }); -const stream = string => - new Readable({ - read() { - this.push(string); - this.push(null); - }, - }); +class ReadableString extends Readable { + constructor(input) { + super(); + this.input = input; + } + + freshCopy() { + return new ReadableString(this.input); + } + + _read() { + this.push(this.input); + this.push(null); + } +} + +const stream = string => new ReadableString(string); + +const mockCsvFileReadStream = str => + jest.spyOn(fs, 'createReadStream').mockImplementation(() => stream(str)); -export { stream, createTestingZip }; +export { stream, createTestingZip, mockCsvFileReadStream }; diff --git a/app/api/csv/specs/test.csv b/app/api/csv/specs/test.csv index 227513f50f..25506ae1ea 100644 --- a/app/api/csv/specs/test.csv +++ b/app/api/csv/specs/test.csv @@ -1,5 +1,5 @@ -Title , text label , numeric label, non configured, select label, not defined type, geolocation_geolocation,auto id, additional tag(s) +Title , text label , numeric label, non configured, select_label, not defined type, geolocation_geolocation,auto id, additional tag(s), multi_select_label -title1, text value 1, 1977, ______________, thesauri1 , notType1 , 1|1,,tag1 -title2, text value 2, 2019, ______________, thesauri2 , notType2 , ,,tag2 -title3, text value 3, 2020, ______________, thesauri2 , notType3 , 0|0,,tag3 +title1, text value 1, 1977, ______________, thesauri1 , notType1 , 1|1,,tag1, multivalue1 +title2, text value 2, 2019, ______________, thesauri2 , notType2 , ,,tag2, multivalue2 +title3, text value 3, 2020, ______________, thesauri2 , notType3 , 0|0,,tag3, multivalue1|multivalue3 diff --git a/app/api/csv/typeParsers/multiselect.ts b/app/api/csv/typeParsers/multiselect.ts index db38fb6cd2..9e208cf428 100644 --- a/app/api/csv/typeParsers/multiselect.ts +++ b/app/api/csv/typeParsers/multiselect.ts @@ -1,10 +1,35 @@ import thesauri from 'api/thesauri'; -import { unique, emptyString } from 'api/utils/filters'; import { RawEntity } from 'api/csv/entityRow'; import { ThesaurusSchema } from 'shared/types/thesaurusType'; import { MetadataObjectSchema, PropertySchema } from 'shared/types/commonTypes'; import { ensure } from 'shared/tsUtils'; +import { normalizeThesaurusLabel } from './select'; + +function labelNotNull(label: string | null): label is string { + return label !== null; +} + +export function splitMultiselectLabels(labelString: string): { [k: string]: string } { + const labels = labelString + .split('|') + .map(l => l.trim()) + .filter(l => l.length > 0); + const result: { [k: string]: string } = {}; + labels.forEach(label => { + const normalizedLabel = normalizeThesaurusLabel(label); + if (labelNotNull(normalizedLabel) && !result.hasOwnProperty(normalizedLabel)) { + result[normalizedLabel] = label; + } + }); + return result; +} + +export function normalizeMultiselectLabels(labelArray: string[]): string[] { + const normalizedLabels = labelArray.map(l => normalizeThesaurusLabel(l)).filter(labelNotNull); + return Array.from(new Set(normalizedLabels)); +} + const multiselect = async ( entityToImport: RawEntity, property: PropertySchema @@ -12,31 +37,14 @@ const multiselect = async ( const currentThesauri = (await thesauri.getById(property.content)) || ({} as ThesaurusSchema); const thesauriValues = currentThesauri.values || []; - const values = entityToImport[ensure(property.name)] - .split('|') - .map(v => v.trim()) - .filter(emptyString) - .filter(unique); - - const newValues = values.filter( - v => !thesauriValues.find(cv => cv.label.trim().toLowerCase() === v.toLowerCase()) - ); - - const lowerCaseValues = values.map(v => v.toLowerCase()); - if (!newValues.length) { - return thesauriValues - .filter(value => lowerCaseValues.includes(value.label.trim().toLowerCase())) - .map(value => ({ value: ensure(value.id), label: value.label })); - } - - const updated = await thesauri.save({ - ...currentThesauri, - values: thesauriValues.concat(newValues.map(label => ({ label }))), - }); + const values = splitMultiselectLabels(entityToImport[ensure(property.name)]); + + const result = Object.keys(values) + .map(key => thesauriValues.find(tv => normalizeThesaurusLabel(tv.label) === key)) + .map(tv => tv) + .map(tv => ({ value: ensure(tv?.id), label: ensure(tv?.label) })); - return (updated.values || []) - .filter(value => lowerCaseValues.includes(value.label.toLowerCase())) - .map(value => ({ value: ensure(value.id), label: value.label })); + return result; }; export default multiselect; diff --git a/app/api/csv/typeParsers/select.ts b/app/api/csv/typeParsers/select.ts index 9f22043dfb..73c6a28f4e 100644 --- a/app/api/csv/typeParsers/select.ts +++ b/app/api/csv/typeParsers/select.ts @@ -1,9 +1,14 @@ import thesauri from 'api/thesauri'; import { RawEntity } from 'api/csv/entityRow'; -import { ThesaurusValueSchema, ThesaurusSchema } from 'shared/types/thesaurusType'; +import { ThesaurusSchema } from 'shared/types/thesaurusType'; import { MetadataObjectSchema, PropertySchema } from 'shared/types/commonTypes'; import { ensure } from 'shared/tsUtils'; +export function normalizeThesaurusLabel(label: string): string | null { + const trimmed = label.trim().toLowerCase(); + return trimmed.length > 0 ? trimmed : null; +} + const select = async ( entityToImport: RawEntity, property: PropertySchema @@ -11,30 +16,18 @@ const select = async ( const currentThesauri = (await thesauri.getById(property.content)) || ({} as ThesaurusSchema); const thesauriValues = currentThesauri.values || []; - if (entityToImport[ensure(property.name)].trim() === '') { + const propValue = entityToImport[ensure(property.name)]; + const normalizedPropValue = normalizeThesaurusLabel(propValue); + if (!normalizedPropValue) { return null; } - const thesauriMatching = (v: ThesaurusValueSchema) => - v.label.trim().toLowerCase() === - entityToImport[ensure(property.name)].trim().toLowerCase(); - - let value = thesauriValues.find(thesauriMatching); - - if (!value) { - const updated = await thesauri.save({ - ...currentThesauri, - values: thesauriValues.concat([ - { - label: entityToImport[ensure(property.name)], - }, - ]), - }); - value = (updated.values || []).find(thesauriMatching); - } + const thesarusValue = thesauriValues.find( + tv => normalizeThesaurusLabel(tv.label) === normalizedPropValue + ); - if (value?.id) { - return [{ value: value.id, label: value.label }]; + if (thesarusValue?.id) { + return [{ value: thesarusValue.id, label: thesarusValue.label }]; } return null; diff --git a/app/api/csv/typeParsers/specs/multiselect.spec.js b/app/api/csv/typeParsers/specs/multiselect.spec.js index da055fa4d9..dfe21122e4 100644 --- a/app/api/csv/typeParsers/specs/multiselect.spec.js +++ b/app/api/csv/typeParsers/specs/multiselect.spec.js @@ -42,30 +42,21 @@ describe('multiselect', () => { thesauri1 = await thesauri.getById(thesauri1Id); }); - it('should create thesauri values and return an array of ids', async () => { - expect(thesauri1.values[0].label).toBe(' value4 '); - expect(thesauri1.values[1].label).toBe('Value1'); - expect(thesauri1.values[2].label).toBe('value3'); - expect(thesauri1.values[3].label).toBe('value2'); - - expect(value1).toEqual([{ value: thesauri1.values[0].id, label: ' value4 ' }]); + it('should find values in thesauri and return an array of ids and labels', async () => { + expect(value1).toEqual([{ value: thesauri1.values[3].id, label: ' value4 ' }]); expect(value2).toEqual([ - { value: thesauri1.values[1].id, label: 'Value1' }, - { value: thesauri1.values[2].id, label: 'value3' }, + { value: thesauri1.values[0].id, label: 'value1' }, + { value: thesauri1.values[2].id, label: 'Value3' }, ]); expect(value3).toEqual([ - { value: thesauri1.values[1].id, label: 'Value1' }, - { value: thesauri1.values[2].id, label: 'value3' }, - { value: thesauri1.values[3].id, label: 'value2' }, + { value: thesauri1.values[0].id, label: 'value1' }, + { value: thesauri1.values[1].id, label: 'value2' }, + { value: thesauri1.values[2].id, label: 'Value3' }, ]); expect(value4).toEqual([ - { value: thesauri1.values[0].id, label: ' value4 ' }, - { value: thesauri1.values[1].id, label: 'Value1' }, - { value: thesauri1.values[3].id, label: 'value2' }, + { value: thesauri1.values[0].id, label: 'value1' }, + { value: thesauri1.values[1].id, label: 'value2' }, + { value: thesauri1.values[3].id, label: ' value4 ' }, ]); }); - - it('should not create blank values, or repeat values', async () => { - expect(thesauri1.values.length).toBe(4); - }); }); diff --git a/app/api/csv/typeParsers/specs/select.spec.js b/app/api/csv/typeParsers/specs/select.spec.js index 3158721233..a7e573e26f 100644 --- a/app/api/csv/typeParsers/specs/select.spec.js +++ b/app/api/csv/typeParsers/specs/select.spec.js @@ -10,57 +10,23 @@ describe('select', () => { beforeEach(async () => db.clearAllAndLoad(fixtures)); afterAll(async () => db.disconnect()); - it('should create thesauri value and return the id', async () => { + it('should find thesauri value and return the id and value', async () => { const templateProp = { name: 'select_prop', content: thesauri1Id }; - const value1 = await typeParsers.select({ select_prop: 'value' }, templateProp); - const value2 = await typeParsers.select({ select_prop: 'value2' }, templateProp); + const value1 = await typeParsers.select({ select_prop: 'value1' }, templateProp); + const value2 = await typeParsers.select({ select_prop: 'vAlUe2' }, templateProp); const thesauri1 = await thesauri.getById(thesauri1Id); - expect(thesauri1.values[1].label).toBe('value'); - expect(thesauri1.values[2].label).toBe('value2'); - expect(value1).toEqual([{ value: thesauri1.values[1].id, label: 'value' }]); - expect(value2).toEqual([{ value: thesauri1.values[2].id, label: 'value2' }]); + expect(value1).toEqual([{ value: thesauri1.values[0].id, label: 'value1' }]); + expect(value2).toEqual([{ value: thesauri1.values[1].id, label: 'value2' }]); }); - it('should not repeat case sensitive values', async () => { - const templateProp = { name: 'select_prop', content: thesauri1Id }; - - await typeParsers.select({ select_prop: 'Value' }, templateProp); - await typeParsers.select({ select_prop: 'value ' }, templateProp); - - await typeParsers.select({ select_prop: 'vAlue2' }, templateProp); - await typeParsers.select({ select_prop: 'vAlue2' }, templateProp); - - await typeParsers.select({ select_prop: 'value4' }, templateProp); - await typeParsers.select({ select_prop: 'ValUe4' }, templateProp); - - const thesauri1 = await thesauri.getById(thesauri1Id); - - expect(thesauri1.values.map(v => v.label)).toEqual([' value4 ', 'Value', 'vAlue2']); - }); - - it('should not create repeated values', async () => { - const templateProp = { name: 'select_prop', content: thesauri1Id }; - - await typeParsers.select({ select_prop: 'value4' }, templateProp); - await typeParsers.select({ select_prop: 'value ' }, templateProp); - await typeParsers.select({ select_prop: 'value' }, templateProp); - await typeParsers.select({ select_prop: ' value ' }, templateProp); - await typeParsers.select({ select_prop: 'value4' }, templateProp); - const thesauri1 = await thesauri.getById(thesauri1Id); - - expect(thesauri1.values.length).toBe(2); - }); - - it('should not create blank values', async () => { + it('should return null on blank values', async () => { const templateProp = { name: 'select_prop', content: thesauri1Id }; const rawEntity = { select_prop: ' ' }; const value = await typeParsers.select(rawEntity, templateProp); - const thesauri1 = await thesauri.getById(thesauri1Id); - expect(thesauri1.values.length).toBe(1); expect(value).toBe(null); }); }); diff --git a/app/api/files/jsRoutes.js b/app/api/files/jsRoutes.js index a4fb4e5eef..ab848363a5 100644 --- a/app/api/files/jsRoutes.js +++ b/app/api/files/jsRoutes.js @@ -126,7 +126,7 @@ const routes = app => { return '/api/public'; }, proxyReqOptDecorator(proxyReqOpts) { - const { tenant, ...headers } = proxyReqOpts.headers; + const { tenant, cookie, ...headers } = proxyReqOpts.headers; return { ...proxyReqOpts, headers: { ...headers }, diff --git a/app/api/files/routes.ts b/app/api/files/routes.ts index c070e962c7..8cb04cf99c 100644 --- a/app/api/files/routes.ts +++ b/app/api/files/routes.ts @@ -213,6 +213,7 @@ export default (app: Application) => { }); req.emitToSessionSocket('IMPORT_CSV_START'); + loader .load(req.file.path, req.body.template, { language: req.language, user: req.user }) .then(() => { diff --git a/app/api/files/specs/jsRoutes.spec.js b/app/api/files/specs/jsRoutes.spec.js index d3a48420ca..18e1138359 100644 --- a/app/api/files/specs/jsRoutes.spec.js +++ b/app/api/files/specs/jsRoutes.spec.js @@ -162,21 +162,25 @@ describe('upload routes', () => { await remoteServer.close(); }); - it('should return the captcha and store it', done => { + it('should remove the tenant and cookie from headers', done => { remoteApp = express(); remoteApp.post('/api/public', (_req, res) => { res.json(_req.headers); }); - remoteServer = remoteApp.listen(54321, async () => { const response = await request(app) .post('/api/remotepublic') .send({ title: 'Title' }) + .set( + 'cookie', + 'locale=en; SL_G_WPT_TO=en; connect.sid=s%3AnK04AiZIYyWOjO_p.kFF17AeJhqKr207n95pV8' + ) .set('tenant', 'tenant') .expect(200); const headersOnRemote = JSON.parse(response.text); - expect(headersOnRemote.tenant).not.toBeDefined(); + expect(headersOnRemote.tenant).toBeUndefined(); + expect(headersOnRemote.cookie).toBeUndefined(); done(); }); }); diff --git a/app/api/migrations/migrations/53-delete-orphaned-connections/index.js b/app/api/migrations/migrations/53-delete-orphaned-connections/index.js new file mode 100644 index 0000000000..f84113d35a --- /dev/null +++ b/app/api/migrations/migrations/53-delete-orphaned-connections/index.js @@ -0,0 +1,34 @@ +/* eslint-disable no-await-in-loop */ +export default { + delta: 53, + + name: 'delete-orphaned-connections', + + description: 'Removes all orphaned connections', + + async up(db) { + // For every connection, check if the entity connected exists + // If the entity does not exist, delete the connection + // Check whether there is only one connections in the hub contained in the connection + // If there is only one connection, delete that connection + const cursor = await db.collection('connections').find({}); + + while (await cursor.hasNext()) { + const connection = await cursor.next(); + const { hub: hubId, entity: sharedId } = connection; + + // Checking if entity exists + const entity = await db.collection('entities').findOne({ sharedId }); + if (!entity) { + await db.collection('connections').deleteOne({ _id: connection._id }); + } + + const connectionsInHub = await db.collection('connections').find({ hub: hubId }); + const count = await connectionsInHub.count(); + if (count === 1) { + const { _id } = await connectionsInHub.next(); + await db.collection('connections').deleteOne({ _id }); + } + } + }, +}; diff --git a/app/api/migrations/migrations/53-delete-orphaned-connections/specs/53-delete-orphaned-connections.spec.js b/app/api/migrations/migrations/53-delete-orphaned-connections/specs/53-delete-orphaned-connections.spec.js new file mode 100644 index 0000000000..304ef76522 --- /dev/null +++ b/app/api/migrations/migrations/53-delete-orphaned-connections/specs/53-delete-orphaned-connections.spec.js @@ -0,0 +1,38 @@ +import testingDB from 'api/utils/testing_db'; +import migration from '../index.js'; +import fixtures from './fixtures.js'; + +describe('migration delete-orphaned-connections', () => { + beforeEach(async () => { + spyOn(process.stdout, 'write'); + await testingDB.clearAllAndLoad(fixtures); + }); + + afterAll(async () => { + await testingDB.tearDown(); + }); + + it('should have a delta number', () => { + expect(migration.delta).toBe(53); + }); + + it('should delete all connections which do not have an existing entity', async () => { + await migration.up(testingDB.mongodb); + const connections = await testingDB.mongodb + .collection('connections') + .find() + .toArray(); + expect(connections.length).toBe(3); + }); + + it('should delete all connections who are alone in a hub', async () => { + await migration.up(testingDB.mongodb); + const connections = await testingDB.mongodb + .collection('connections') + .find() + .toArray(); + expect(connections).toEqual( + expect.arrayContaining([expect.not.objectContaining({ hub: 'hub3' })]) + ); + }); +}); diff --git a/app/api/migrations/migrations/53-delete-orphaned-connections/specs/fixtures.js b/app/api/migrations/migrations/53-delete-orphaned-connections/specs/fixtures.js new file mode 100644 index 0000000000..8ac4ecd5b8 --- /dev/null +++ b/app/api/migrations/migrations/53-delete-orphaned-connections/specs/fixtures.js @@ -0,0 +1,54 @@ +export default { + entities: [ + { + sharedId: 'sharedid1', + title: 'test_doc', + }, + { + sharedId: 'sharedid2', + title: 'test_doc_2', + }, + { + sharedId: 'sharedid1', + title: 'test_doc', + }, + { + sharedId: 'sharedid3', + title: 'test_doc_2', + }, + { + sharedId: 'sharedid1', + title: 'test_doc', + }, + { + sharedId: 'sharedid3', + title: 'test_doc_2', + }, + ], + connections: [ + { + entity: 'sharedid3', + hub: 'hub1', + }, + { + entity: 'sharedid1', + hub: 'hub1', + }, + { + entity: 'sharedid2', + hub: 'hub1', + }, + { + entity: 'sharedid3', + hub: 'hub2', + }, + { + entity: 'sharedid1', + hub: 'hub3', + }, + { + entity: 'shareid4', + hub: 'hub3', + }, + ], +}; diff --git a/app/react/Library/components/LibraryModeToggleButtons.js b/app/react/Library/components/LibraryModeToggleButtons.js index a505f147d6..1b2c1277a1 100644 --- a/app/react/Library/components/LibraryModeToggleButtons.js +++ b/app/react/Library/components/LibraryModeToggleButtons.js @@ -143,7 +143,9 @@ export function mapStateToProps(state, props) { templates.find(_t => _t.get('properties').find(p => p.get('type') === 'geolocation')) ); - const numberOfMarkers = numberOfMarkersSelector({ state, storeKey: props.storeKey }); + const numberOfMarkers = showGeolocation + ? numberOfMarkersSelector({ state, storeKey: props.storeKey }) + : 0; return { searchUrl: encodedSearch(state[props.storeKey]), diff --git a/app/react/Library/helpers/requestState.js b/app/react/Library/helpers/requestState.js index 2697f30a4b..e65eaf987f 100644 --- a/app/react/Library/helpers/requestState.js +++ b/app/react/Library/helpers/requestState.js @@ -55,9 +55,21 @@ export default function requestState(request, globalResources, calculateTableCol const documentsRequest = request.set( tocGenerationUtils.aggregations(docsQuery, globalResources.settings.collection.toJS()) ); - const markersRequest = request.set({ ...docsQuery, geolocation: true }); - return Promise.all([api.search(documentsRequest), api.search(markersRequest)]).then( + const templatesWithGeolocation = globalResources.templates.find(template => + template.get('properties').find(property => property.get('type') === 'geolocation') + ); + + const markersRequest = templatesWithGeolocation + ? api.search( + request.set({ + ...docsQuery, + geolocation: true, + }) + ) + : { rows: [] }; + + return Promise.all([api.search(documentsRequest), markersRequest]).then( ([documents, markers]) => { const templates = globalResources.templates.toJS(); const filterState = libraryHelpers.URLQueryToState( diff --git a/app/react/Library/helpers/specs/resquestState.spec.js b/app/react/Library/helpers/specs/resquestState.spec.js index f64df8e357..fbe6bc1211 100644 --- a/app/react/Library/helpers/specs/resquestState.spec.js +++ b/app/react/Library/helpers/specs/resquestState.spec.js @@ -6,37 +6,41 @@ import rison from 'rison-node'; import requestState, { processQuery } from '../requestState'; describe('static requestState()', () => { - const aggregations = { buckets: [] }; - const templates = [ - { - name: 'Decision', - _id: 'abc1', - properties: [ - { name: 'p', filter: true, type: 'text', prioritySorting: true }, - { name: 'country', filter: false, type: 'select', content: 'countries' }, - ], - }, - { name: 'Ruling', _id: 'abc2', properties: [] }, - ]; - const relationTypes = [ - { name: 'Victim', _id: 'abc3', properties: [{ name: 'p', filter: true, type: 'text' }] }, - ]; - - const thesauris = [{ name: 'countries', _id: '1', values: [] }]; - const documents = { - rows: [{ title: 'Something to publish' }, { title: 'My best recipes' }], - totalRows: 2, - aggregations, - }; - const globalResources = { - templates: Immutable.fromJS(templates), - settings: { collection: Immutable.fromJS({ features: {} }) }, - thesauris: Immutable.fromJS(thesauris), - relationTypes: Immutable.fromJS(relationTypes), - user: Immutable.fromJS({}), - }; - + let globalResources; + let templates; beforeEach(() => { + const aggregations = { buckets: [] }; + templates = [ + { + name: 'Decision', + _id: 'abc1', + properties: [ + { name: 'p', filter: true, type: 'text', prioritySorting: true }, + { name: 'country', filter: false, type: 'select', content: 'countries' }, + { name: 'location', filter: false, type: 'geolocation' }, + ], + }, + { name: 'Ruling', _id: 'abc2', properties: [] }, + ]; + const relationTypes = [ + { name: 'Victim', _id: 'abc3', properties: [{ name: 'p', filter: true, type: 'text' }] }, + ]; + + const thesauris = [{ name: 'countries', _id: '1', values: [] }]; + const documents = { + rows: [{ title: 'Something to publish' }, { title: 'My best recipes' }], + totalRows: 2, + aggregations, + }; + + globalResources = { + templates: Immutable.fromJS(templates), + settings: { collection: Immutable.fromJS({ features: {} }) }, + thesauris: Immutable.fromJS(thesauris), + relationTypes: Immutable.fromJS(relationTypes), + user: Immutable.fromJS({}), + }; + spyOn(searchAPI, 'search').and.returnValue(Promise.resolve(documents)); }); @@ -80,8 +84,39 @@ describe('static requestState()', () => { const actions = await requestState(request, globalResources); expect(searchAPI.search).toHaveBeenCalledWith(expectedSearch); + expect(actions).toMatchSnapshot(); }); + + it('should work when there are no geolocation type properties', async () => { + templates = [ + { + name: 'Appeal', + _id: 'abc2', + properties: [ + { name: 'p', filter: true, type: 'text', prioritySorting: true }, + { name: 'description', filter: false, type: 'markdown' }, + ], + }, + ]; + globalResources.templates = Immutable.fromJS(templates); + const query = { q: rison.encode({ filters: { something: 1 }, types: [] }) }; + const expectedSearch = { + data: { + sort: prioritySortingCriteria.get({ templates: Immutable.fromJS(templates) }).sort, + order: prioritySortingCriteria.get({ templates: Immutable.fromJS(templates) }).order, + filters: { something: 1 }, + types: [], + view: undefined, + }, + headers: {}, + }; + const request = new RequestParams(query); + + await requestState(request, globalResources); + + expect(searchAPI.search).toHaveBeenCalledWith(expectedSearch); + }); }); describe('processQuery()', () => { diff --git a/app/react/Metadata/components/MetadataFormFields.js b/app/react/Metadata/components/MetadataFormFields.js index 7a4cf89e46..ded21a2d2b 100644 --- a/app/react/Metadata/components/MetadataFormFields.js +++ b/app/react/Metadata/components/MetadataFormFields.js @@ -356,8 +356,8 @@ export const mapStateToProps = (state, ownProps) => { attachments = selectedDocuments.size ? selectedDocuments.get(0).get('attachments') : undefined; } - if (storeKey === 'documentView') { - const entity = state.documentView.doc; + if (storeKey === 'documentViewer') { + const entity = state[storeKey].doc; attachments = entity.get('attachments'); } diff --git a/e2e/helpers/createEntity.ts b/e2e/helpers/createEntity.ts new file mode 100644 index 0000000000..ff9f0bc20f --- /dev/null +++ b/e2e/helpers/createEntity.ts @@ -0,0 +1,46 @@ +import { host } from '../config'; +import { ElementHandle } from 'puppeteer'; + +interface FilesOptions { + pdf?: string; + supportingFile?: string; +} + +const uploadPDFToEntity = async (pdfName: string) => { + await expect(page).toUploadFile('#upload-button-input', pdfName); +}; + +const uploadSupportingFileToEntity = async (fileName: string): Promise => { + await expect(page).toClick('button[type="button"].upload-button'); + const [fileChooser] = await Promise.all([ + page.waitForFileChooser(), + page.click('div.attachments-modal__dropzone > button'), + ]); + await fileChooser.accept([fileName]); +}; + +export const createEntity = async (templateName: string, files: FilesOptions) => { + await page.goto(`${host}`); + await expect(page).toClick('button', { text: 'Create entity' }); + await expect(page).toFill('textarea[name="library.sidepanel.metadata.title"]', templateName); + let options: ElementHandle[] = []; + options = await page.$$('select.form-control > option'); + + // @ts-ignore + options.forEach(async (option: ElementHandle): void => { + const value = await option.evaluate(optionEl => ({ + text: optionEl.textContent, + value: optionEl.getAttribute('value') as string, + })); + if (value.text === templateName) await page.select('select.form-control', value.value); + }); + await expect(page).toMatchElement('button[form="metadataForm"]', { text: 'Save' }); + await expect(page).toClick('button[form="metadataForm"]', { text: 'Save' }); + await expect(page).toClick('span', { text: 'Entity created' }); + + if (files) { + if (files.pdf) await uploadPDFToEntity(files.pdf); + if (files.supportingFile) await uploadSupportingFileToEntity(files.supportingFile); + await expect(page).toClick('span', { text: 'Attachment uploaded' }); + } +}; diff --git a/e2e/helpers/createTemplate.ts b/e2e/helpers/createTemplate.ts new file mode 100644 index 0000000000..5b3616ed38 --- /dev/null +++ b/e2e/helpers/createTemplate.ts @@ -0,0 +1,15 @@ +import { host } from '../config'; + +export const createTemplate = async (name: string) => { + await page.goto(`${host}/en/settings/account`); + await expect(page).toClick('span', { text: 'Templates' }); + await expect(page).toClick('span', { text: 'Add template' }); + await expect(page).toFill('input[name="template.data.name"]', name); + if (name === 'With image') { + const imageElement = await page.$$('ul.property-options-list > .list-group-item'); + const button = await imageElement[11].$$('button'); + await button[0].click(); + } + await expect(page).toClick('button[type="submit"]'); + await expect(page).toClick('span', { text: 'Saved successfully.' }); +}; diff --git a/e2e/suites/convert-entity-template.test.ts b/e2e/suites/convert-entity-template.test.ts new file mode 100644 index 0000000000..f7768f5c56 --- /dev/null +++ b/e2e/suites/convert-entity-template.test.ts @@ -0,0 +1,42 @@ +import { createEntity } from '../helpers/createEntity'; +import { createTemplate } from '../helpers/createTemplate'; +import { adminLogin, logout } from '../helpers/login'; +import proxyMock from '../helpers/proxyMock'; +import insertFixtures from '../helpers/insertFixtures'; +import disableTransitions from '../helpers/disableTransitions'; + +const setupPreFlights = async (): Promise => { + await insertFixtures(); + await proxyMock(); + await adminLogin(); +}; + +const setupTest = async () => { + await createTemplate('With image'); + await createEntity('With image', { + pdf: `${__dirname}/test_files/valid.pdf`, + supportingFile: `${__dirname}/test_files/batman.jpg`, + }); +}; + +describe('Image is rendered when editing an entity in document view', () => { + beforeAll(async () => { + await setupPreFlights(); + await setupTest(); + await disableTransitions(); + }); + + it('Should select image for image property from supporting files', async () => { + await expect(page).toClick('a[type="button"]'); + + await expect(page).toClick('.metadata-sidepanel button.edit-metadata', { + text: 'Edit', + }); + await expect(page).toClick('span', { text: 'Select supporting file' }); + await expect(page).toMatchElement('div.media-grid-card-header > h5', { text: 'batman.jpg' }); + }); + + afterAll(async () => { + await logout(); + }); +}); diff --git a/e2e/suites/test_files/batman.jpg b/e2e/suites/test_files/batman.jpg new file mode 100644 index 0000000000..d4ad77f4e7 Binary files /dev/null and b/e2e/suites/test_files/batman.jpg differ diff --git a/package.json b/package.json index 91a667638d..9b76544fc6 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "uwazi", - "version": "1.39.0", + "version": "1.40.0", "description": "Uwazi is a free, open-source solution for organising, analysing and publishing your documents.", "keywords": [ "react" diff --git a/yarn.lock b/yarn.lock index 57b6fc1ea8..c7f7a12951 100644 --- a/yarn.lock +++ b/yarn.lock @@ -14499,8 +14499,9 @@ tmp@^0.2.1: rimraf "^3.0.0" tmpl@1.0.x: - version "1.0.4" - resolved "https://registry.yarnpkg.com/tmpl/-/tmpl-1.0.4.tgz#23640dd7b42d00433911140820e5cf440e521dd1" + version "1.0.5" + resolved "https://registry.yarnpkg.com/tmpl/-/tmpl-1.0.5.tgz#8683e0b902bb9c20c4f726e3c0b69f36518c07cc" + integrity sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw== to-arraybuffer@^1.0.0: version "1.0.1"