From c16cb36825f3a17e4582e7a64d469ae5c3053a1c Mon Sep 17 00:00:00 2001 From: Kevin Date: Wed, 15 Sep 2021 21:59:26 +0300 Subject: [PATCH 01/52] Added storeKey document viewer --- app/react/Metadata/components/MetadataFormFields.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/react/Metadata/components/MetadataFormFields.js b/app/react/Metadata/components/MetadataFormFields.js index 7a4cf89e46..451f69e1d5 100644 --- a/app/react/Metadata/components/MetadataFormFields.js +++ b/app/react/Metadata/components/MetadataFormFields.js @@ -356,8 +356,8 @@ export const mapStateToProps = (state, ownProps) => { attachments = selectedDocuments.size ? selectedDocuments.get(0).get('attachments') : undefined; } - if (storeKey === 'documentView') { - const entity = state.documentView.doc; + if (storeKey === 'documentView' || storeKey === 'documentViewer') { + const entity = state[storeKey].doc; attachments = entity.get('attachments'); } From 21c2f85f976e1d024d99430a4e9039d3b0430f0e Mon Sep 17 00:00:00 2001 From: Kevin Date: Thu, 16 Sep 2021 13:19:51 +0300 Subject: [PATCH 02/52] Init migration --- .../51-delete-orphaned-connections/index.js | 12 ++++++++++ .../51-delete-orphaned-connections.spec.js | 22 +++++++++++++++++++ .../specs/fixtures.js | 5 +++++ 3 files changed, 39 insertions(+) create mode 100644 app/api/migrations/migrations/51-delete-orphaned-connections/index.js create mode 100644 app/api/migrations/migrations/51-delete-orphaned-connections/specs/51-delete-orphaned-connections.spec.js create mode 100644 app/api/migrations/migrations/51-delete-orphaned-connections/specs/fixtures.js diff --git a/app/api/migrations/migrations/51-delete-orphaned-connections/index.js b/app/api/migrations/migrations/51-delete-orphaned-connections/index.js new file mode 100644 index 0000000000..aa997c742e --- /dev/null +++ b/app/api/migrations/migrations/51-delete-orphaned-connections/index.js @@ -0,0 +1,12 @@ +export default { + delta: 51, + + name: 'delete-orphaned-connections', + + description: 'Removes all orphaned connections', + + async up() { + process.stdout.write(`${this.name}...\r\n`); + return Promise.reject(new Error('error! change this, recently created migration')); + }, +}; diff --git a/app/api/migrations/migrations/51-delete-orphaned-connections/specs/51-delete-orphaned-connections.spec.js b/app/api/migrations/migrations/51-delete-orphaned-connections/specs/51-delete-orphaned-connections.spec.js new file mode 100644 index 0000000000..32eed3d42d --- /dev/null +++ b/app/api/migrations/migrations/51-delete-orphaned-connections/specs/51-delete-orphaned-connections.spec.js @@ -0,0 +1,22 @@ +import testingDB from 'api/utils/testing_db'; +import migration from '../index.js'; +import fixtures from './fixtures.js'; + +describe('migration delete-orphaned-connections', () => { + beforeEach(async () => { + spyOn(process.stdout, 'write'); + await testingDB.clearAllAndLoad(fixtures); + }); + + afterAll(async () => { + await testingDB.tearDown(); + }); + + it('should have a delta number', () => { + expect(migration.delta).toBe(51); + }); + + it('should fail', async () => { + await migration.up(); + }); +}); diff --git a/app/api/migrations/migrations/51-delete-orphaned-connections/specs/fixtures.js b/app/api/migrations/migrations/51-delete-orphaned-connections/specs/fixtures.js new file mode 100644 index 0000000000..de6913240d --- /dev/null +++ b/app/api/migrations/migrations/51-delete-orphaned-connections/specs/fixtures.js @@ -0,0 +1,5 @@ +import db from 'api/utils/testing_db'; + +export default { + entities: [{ title: 'test_doc' }] +}; From 4fbd14c20f6e5381fc33641642f31960cc3a0394 Mon Sep 17 00:00:00 2001 From: Kevin Date: Fri, 17 Sep 2021 10:59:37 +0300 Subject: [PATCH 03/52] Remove documentView storeKey check --- app/react/Metadata/components/MetadataFormFields.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/react/Metadata/components/MetadataFormFields.js b/app/react/Metadata/components/MetadataFormFields.js index 451f69e1d5..ded21a2d2b 100644 --- a/app/react/Metadata/components/MetadataFormFields.js +++ b/app/react/Metadata/components/MetadataFormFields.js @@ -356,7 +356,7 @@ export const mapStateToProps = (state, ownProps) => { attachments = selectedDocuments.size ? selectedDocuments.get(0).get('attachments') : undefined; } - if (storeKey === 'documentView' || storeKey === 'documentViewer') { + if (storeKey === 'documentViewer') { const entity = state[storeKey].doc; attachments = entity.get('attachments'); } From 9b8b61f3f1e36dd741e140e498a167cfcdd245b3 Mon Sep 17 00:00:00 2001 From: Kevin Date: Fri, 17 Sep 2021 11:34:49 +0300 Subject: [PATCH 04/52] Aded basic implementation with primitive tests --- .../51-delete-orphaned-connections/index.js | 28 ++++++++++++++++-- .../51-delete-orphaned-connections.spec.js | 29 +++++++++++++++++-- .../specs/fixtures.js | 23 +++++++++++++-- 3 files changed, 72 insertions(+), 8 deletions(-) diff --git a/app/api/migrations/migrations/51-delete-orphaned-connections/index.js b/app/api/migrations/migrations/51-delete-orphaned-connections/index.js index aa997c742e..0e2dfb4689 100644 --- a/app/api/migrations/migrations/51-delete-orphaned-connections/index.js +++ b/app/api/migrations/migrations/51-delete-orphaned-connections/index.js @@ -1,3 +1,4 @@ +/* eslint-disable no-await-in-loop */ export default { delta: 51, @@ -5,8 +6,29 @@ export default { description: 'Removes all orphaned connections', - async up() { - process.stdout.write(`${this.name}...\r\n`); - return Promise.reject(new Error('error! change this, recently created migration')); + async up(db) { + // For every connection, check if the entity connected exists + // If the entity does not exist, delete the connection + // Check whether there is only one connections in the hub contained in the connection + // If there is only one connection, delete that connection + const cursor = await db.collection('connections').find({}); + + while (await cursor.hasNext()) { + const connection = await cursor.next(); + const { hub: hubId, entity: sharedId } = connection; + + // Checking if entity exists + const entity = await db.collection('entities').findOne({ sharedId }); + if (!entity) { + await db.collection('connections').deleteOne({ _id: connection._id }); + } + + const connectionsInHub = await db.collection('connections').find({ hub: hubId }); + const count = await connectionsInHub.count(); + if (count === 1) { + const { _id } = await connectionsInHub.next(); + await db.collection('connections').deleteOne({ _id }); + } + } }, }; diff --git a/app/api/migrations/migrations/51-delete-orphaned-connections/specs/51-delete-orphaned-connections.spec.js b/app/api/migrations/migrations/51-delete-orphaned-connections/specs/51-delete-orphaned-connections.spec.js index 32eed3d42d..1d42c5a7ac 100644 --- a/app/api/migrations/migrations/51-delete-orphaned-connections/specs/51-delete-orphaned-connections.spec.js +++ b/app/api/migrations/migrations/51-delete-orphaned-connections/specs/51-delete-orphaned-connections.spec.js @@ -16,7 +16,32 @@ describe('migration delete-orphaned-connections', () => { expect(migration.delta).toBe(51); }); - it('should fail', async () => { - await migration.up(); + it('should delete all connections', async () => { + await migration.up(testingDB.mongodb); + const connections = await testingDB.mongodb + .collection('connections') + .find() + .toArray(); + expect(connections.length).toBe(0); + }); + + it('should delete all orphaned connections except two remaining in hub', async () => { + const localFixtures = { + entities: [...fixtures.entities], + connections: [ + ...fixtures.connections, + { + entity: 'sharedid2', + hub: 'hub1', + }, + ], + }; + await testingDB.clearAllAndLoad(localFixtures); + await migration.up(testingDB.mongodb); + const connections = await testingDB.mongodb + .collection('connections') + .find() + .toArray(); + expect(connections.length).toBe(2); }); }); diff --git a/app/api/migrations/migrations/51-delete-orphaned-connections/specs/fixtures.js b/app/api/migrations/migrations/51-delete-orphaned-connections/specs/fixtures.js index de6913240d..a40fe9f5fb 100644 --- a/app/api/migrations/migrations/51-delete-orphaned-connections/specs/fixtures.js +++ b/app/api/migrations/migrations/51-delete-orphaned-connections/specs/fixtures.js @@ -1,5 +1,22 @@ -import db from 'api/utils/testing_db'; - export default { - entities: [{ title: 'test_doc' }] + entities: [ + { + sharedId: 'sharedid1', + title: 'test_doc', + }, + { + sharedId: 'sharedid2', + title: 'test_doc_2', + }, + ], + connections: [ + { + entity: 'sharedid3', + hub: 'hub1', + }, + { + entity: 'sharedid1', + hub: 'hub1', + }, + ], }; From 4c5ed652d101dd5cecb6c6060257cfc9b7131f8f Mon Sep 17 00:00:00 2001 From: Kevin Date: Fri, 17 Sep 2021 18:52:54 +0300 Subject: [PATCH 05/52] added tests --- .../51-delete-orphaned-connections.spec.js | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/app/api/migrations/migrations/51-delete-orphaned-connections/specs/51-delete-orphaned-connections.spec.js b/app/api/migrations/migrations/51-delete-orphaned-connections/specs/51-delete-orphaned-connections.spec.js index 1d42c5a7ac..2ce6db732d 100644 --- a/app/api/migrations/migrations/51-delete-orphaned-connections/specs/51-delete-orphaned-connections.spec.js +++ b/app/api/migrations/migrations/51-delete-orphaned-connections/specs/51-delete-orphaned-connections.spec.js @@ -44,4 +44,22 @@ describe('migration delete-orphaned-connections', () => { .toArray(); expect(connections.length).toBe(2); }); + + it('should not delete any connection', async () => { + const localFixtures = { + entities: [...fixtures.entities], + connections: [ + { ...fixtures.connections[0], entity: 'sharedid1' }, + { ...fixtures.connections[1], entity: 'sharedid2' }, + ], + }; + + await testingDB.clearAllAndLoad(localFixtures); + await migration.up(testingDB.mongodb); + const connections = await testingDB.mongodb + .collection('connections') + .find() + .toArray(); + expect(connections.length).toBe(2); + }); }); From 7e0e44bc094ef408ed9e9750b55ebf404cc77bc5 Mon Sep 17 00:00:00 2001 From: Laszlo Kecskes Date: Mon, 20 Sep 2021 11:08:01 +0200 Subject: [PATCH 06/52] first implementation --- app/api/csv/csvLoader.ts | 20 +++- app/api/csv/importEntity.ts | 126 ++++++++++++++++++++-- app/api/csv/specs/arrangeThesauriTest.csv | 17 +++ app/api/csv/specs/importEntity.spec.js | 107 ++++++++++++++++++ app/api/csv/typeParsers/multiselect.ts | 26 +++++ app/api/csv/typeParsers/select.ts | 13 +++ app/api/files/routes.ts | 11 +- 7 files changed, 309 insertions(+), 11 deletions(-) create mode 100644 app/api/csv/specs/arrangeThesauriTest.csv create mode 100644 app/api/csv/specs/importEntity.spec.js diff --git a/app/api/csv/csvLoader.ts b/app/api/csv/csvLoader.ts index 048a3f89bf..dd2af9be60 100644 --- a/app/api/csv/csvLoader.ts +++ b/app/api/csv/csvLoader.ts @@ -1,3 +1,4 @@ +/* eslint-disable max-statements */ import { EventEmitter } from 'events'; import templates from 'api/templates'; @@ -12,9 +13,11 @@ import { ensure } from 'shared/tsUtils'; import { ObjectId } from 'mongodb'; import csv, { CSVRow } from './csv'; import importFile from './importFile'; -import { importEntity, translateEntity } from './importEntity'; +import { arrangeThesauri, importEntity, translateEntity } from './importEntity'; import { extractEntity, toSafeName } from './entityRow'; +const { performance } = require('perf_hooks') + export class CSVLoader extends EventEmitter { stopOnError: boolean; @@ -46,29 +49,43 @@ export class CSVLoader extends EventEmitter { templateId: ObjectId | string, options = { language: 'en', user: {} } ) { + // console.log('---------------csvLoader.load') + let timeInImportAndTranslate = 0; const template = await templates.getById(templateId); if (!template) { throw new Error('template not found!'); } + // console.log(`-----template\n${JSON.stringify(template, null, 2)}\n-----done`) const file = importFile(csvPath); + // console.log(`-----file\n${file}\n-----done`) const availableLanguages: string[] = ensure( (await settings.get()).languages ).map((l: LanguageSchema) => l.key); const { newNameGeneration = false } = await settings.get(); + // console.log(`-----availableLanguages\n${availableLanguages}\n-----done`) + console.time('time spent in arrageThesauri'); + await arrangeThesauri(file, template); + console.timeEnd('time spent in arrageThesauri'); await csv(await file.readStream(), this.stopOnError) .onRow(async (row: CSVRow) => { + // console.log(row) const { rawEntity, rawTranslations } = extractEntity( row, availableLanguages, options.language, newNameGeneration ); + // console.log(`-----rawEntity\n${JSON.stringify(rawEntity, null, 2)}\n-----done`) + // console.log(`-----rawTranslations\n${JSON.stringify(rawTranslations, null, 2)}\n-----done`) if (rawEntity) { + const t1 = performance.now() const entity = await importEntity(rawEntity, template, file, options); await translateEntity(entity, rawTranslations, template, file); this.emit('entityLoaded', entity); + const t2 = performance.now() + timeInImportAndTranslate += t2 - t1; } }) .onError(async (e: Error, row: CSVRow, index: number) => { @@ -76,6 +93,7 @@ export class CSVLoader extends EventEmitter { this.emit('loadError', e, toSafeName(row), index); }) .read(); + console.log(`Total time spent in importEntity and translateEntity:${timeInImportAndTranslate}`) this.throwErrors(); } diff --git a/app/api/csv/importEntity.ts b/app/api/csv/importEntity.ts index 62fdad0077..fa180d35c8 100644 --- a/app/api/csv/importEntity.ts +++ b/app/api/csv/importEntity.ts @@ -1,3 +1,4 @@ +/* eslint-disable max-statements */ import entities from 'api/entities'; import { search } from 'api/search'; import entitiesModel from 'api/entities/entitiesModel'; @@ -5,18 +6,22 @@ import { processDocument } from 'api/files/processDocument'; import { RawEntity } from 'api/csv/entityRow'; import { TemplateSchema } from 'shared/types/templateType'; import { MetadataSchema, PropertySchema } from 'shared/types/commonTypes'; +import { propertyTypes } from 'shared/propertyTypes'; import { ImportFile } from 'api/csv/importFile'; +import thesauri from 'api/thesauri'; import { EntitySchema } from 'shared/types/entityType'; import { ensure } from 'shared/tsUtils'; - import { attachmentsPath, files } from 'api/files'; -import { propertyTypes } from 'shared/propertyTypes'; import { generateID } from 'shared/IDGenerator'; + +import { normalizeThesaurusLabel } from './typeParsers/select'; +import { splitMultiselectLabels } from './typeParsers/multiselect'; import typeParsers from './typeParsers'; +import csv, { CSVRow } from './csv'; const parse = async (toImportEntity: RawEntity, prop: PropertySchema) => typeParsers[prop.type] - ? typeParsers[prop.type](toImportEntity, prop) + ? typeParsers[prop.type](toImportEntity, prop) // ISSUE_1: go down to select typeparser - db calls each row : typeParsers.text(toImportEntity, prop); const hasValidValue = (prop: PropertySchema, toImportEntity: RawEntity) => @@ -38,7 +43,7 @@ const toMetadata = async ( ); const currentEntityIdentifiers = async (sharedId: string, language: string) => - sharedId ? entities.get({ sharedId, language }, '_id sharedId').then(([e]) => e) : {}; + sharedId ? entities.get({ sharedId, language }, '_id sharedId').then(([e]) => e) : {}; //ISSUE_5 entity.get const titleByTemplate = (template: TemplateSchema, entity: RawEntity) => { const generatedTitle = @@ -66,6 +71,108 @@ type Options = { language: string; }; +const arrangeThesauri = async (file: ImportFile, template: TemplateSchema) => { + const nameToThesauriIdSelects: { [k: string]: string } = {}; + const nameToThesauriIdMultiselects: { [k: string]: string } = {}; + const thesauriIdToExistingValues = new Map(); + const thesauriIdToNewValues: Map> = new Map(); + const thesauriIdToNormalizedNewValues = new Map(); + const thesauriRelatedProperties = template.properties?.filter(p => + [propertyTypes.select, propertyTypes.multiselect].includes(p.type) + ); + thesauriRelatedProperties?.forEach(p => { + if (p.content && p.type) { + if (p.type === propertyTypes.select) { + nameToThesauriIdSelects[p.name] = p.content.toString(); + } else if (p.type === propertyTypes.multiselect) { + nameToThesauriIdMultiselects[p.name] = p.content.toString(); + } + } + }); + const allRelatedThesauri = await Promise.all( + Array.from( + new Set(thesauriRelatedProperties?.map(p => p.content?.toString()).filter(t => t)) + ).map(async id => thesauri.getById(id)) + ); + allRelatedThesauri.forEach(t => { + if (t) { + const id = t._id.toString(); + thesauriIdToExistingValues.set( + id, + new Set(t.values?.map(v => normalizeThesaurusLabel(v.label))) + ); + thesauriIdToNewValues.set(id, new Set()); + thesauriIdToNormalizedNewValues.set(id, new Set()); + } + }); + // console.log(nameToThesauriIdSelects) + // console.log(nameToThesauriIdMultiselects) + // console.log(allRelatedThesauri) + // console.log(thesauriIdToExistingValues) + // console.log(thesauriIdToNewValues); + // console.log(thesauriIdToNormalizedNewValues); + console.log('Reading'); + let readCount = 0; + await csv(await file.readStream()) + .onRow(async (row: CSVRow) => { + readCount += 1; + if (readCount % 1000 === 0) { + console.log(readCount); + } + Object.entries(nameToThesauriIdSelects).forEach(entry => { + const name = entry[0]; + const id = entry[1]; + const label = row[name]; + const normalizedLabel = normalizeThesaurusLabel(label); + if ( + normalizedLabel && + !thesauriIdToExistingValues.get(id).has(normalizedLabel) && + !thesauriIdToNormalizedNewValues.get(id).has(normalizedLabel) + ) { + thesauriIdToNewValues.get(id)?.add(label); + thesauriIdToNormalizedNewValues.get(id).add(normalizedLabel); + } + }); + Object.entries(nameToThesauriIdMultiselects).forEach(([name, id]) => { + const labels = splitMultiselectLabels(row[name]); + Object.entries(labels).forEach(([normalizedLabel, originalLabel]) => { + if ( + normalizedLabel && + !thesauriIdToExistingValues.get(id).has(normalizedLabel) && + !thesauriIdToNormalizedNewValues.get(id).has(normalizedLabel) + ) { + thesauriIdToNewValues.get(id)?.add(originalLabel); + thesauriIdToNormalizedNewValues.get(id).add(normalizedLabel); + } + }); + }); + }) + .onError(async (e: Error, row: CSVRow, index: number) => { + console.log(e); + }) + .read(); + console.log('Saving thesauri'); + await Promise.all( + allRelatedThesauri.map(thesaurus => { + if (thesaurus !== null) { + // console.log(thesaurus.name); + // console.log(thesaurus._id); + const newValues: { label: string }[] = Array.from( + thesauriIdToNewValues.get(thesaurus._id.toString()) || [] + ).map(tval => ({ label: tval })); + // console.log(newValues); + const thesaurusValues = thesaurus.values || []; + return thesauri.save({ + ...thesaurus, + values: thesaurusValues.concat(newValues), + }); + } + }) + ); + // console.log(thesauriIdToNewValues); + // console.log(thesauriIdToNormalizedNewValues); +}; + const importEntity = async ( toImportEntity: RawEntity, template: TemplateSchema, @@ -75,18 +182,20 @@ const importEntity = async ( const { attachments } = toImportEntity; delete toImportEntity.attachments; const eo = await entityObject(toImportEntity, template, { language }); - const entity = await entities.save(eo, { user, language }, true, false); + const entity = await entities.save(eo, { user, language }, true, false); // ISSUE_2: then saves the entity as well + //ISSUE_3: same with documents if (toImportEntity.file && entity.sharedId) { const file = await importFile.extractFile(toImportEntity.file); await processDocument(entity.sharedId, file); } + //ISSUE_4: same with attachments if (attachments && entity.sharedId) { await attachments.split('|').reduce(async (promise: Promise, attachment) => { await promise; const attachmentFile = await importFile.extractFile(attachment, attachmentsPath()); - return files.save({ ...attachmentFile, entity: entity.sharedId, type: 'attachment' }); + return files.save({ ...attachmentFile, entity: entity.sharedId, type: 'attachment' }); // <-----here }, Promise.resolve()); } @@ -101,6 +210,7 @@ const translateEntity = async ( importFile: ImportFile ) => { await entitiesModel.saveMultiple( + // ISSUE_6: this also just maps saves await Promise.all( translations.map(async translatedEntity => entityObject({ ...translatedEntity, id: ensure(entity.sharedId) }, template, { @@ -114,7 +224,7 @@ const translateEntity = async ( translations.map(async translatedEntity => { if (translatedEntity.file) { const file = await importFile.extractFile(translatedEntity.file); - await processDocument(ensure(entity.sharedId), file); + await processDocument(ensure(entity.sharedId), file); //ISSUE_7: process document again } }) ); @@ -122,4 +232,4 @@ const translateEntity = async ( await search.indexEntities({ sharedId: entity.sharedId }, '+fullText'); }; -export { importEntity, translateEntity }; +export { arrangeThesauri, importEntity, translateEntity }; diff --git a/app/api/csv/specs/arrangeThesauriTest.csv b/app/api/csv/specs/arrangeThesauriTest.csv new file mode 100644 index 0000000000..474bf27bed --- /dev/null +++ b/app/api/csv/specs/arrangeThesauriTest.csv @@ -0,0 +1,17 @@ +title,unrelated_property,select_property,multiselect_property +select_1,unrelated_text,B,A +select_2,unrelated_text,C,A +select_3,unrelated_text,b,A +select_4,unrelated_text,B,A +select_5,unrelated_text,d,A +select_6,unrelated_text,D,A +select_7,unrelated_text, b,A +select_8,unrelated_text, ,A +select_8,unrelated_text, ,A +multiselect_1,unrelated_text,A,B +multiselect_2,unrelated_text,A,c +multiselect_3,unrelated_text,A,A|b +multiselect_4,unrelated_text,A,a|B|C +multiselect_5,unrelated_text,A, a| b | +multiselect_6,unrelated_text,A, | | +multiselect_7, unrelated_text,A,A|B|C|D| |E| e| g diff --git a/app/api/csv/specs/importEntity.spec.js b/app/api/csv/specs/importEntity.spec.js new file mode 100644 index 0000000000..a4b7d19f32 --- /dev/null +++ b/app/api/csv/specs/importEntity.spec.js @@ -0,0 +1,107 @@ +import path from 'path'; + +import templates from 'api/templates'; +import { getFixturesFactory } from 'api/utils/fixturesFactory'; +import db from 'api/utils/testing_db'; + +// import fixtures, { template1Id } from './fixtures'; +import { arrangeThesauri } from '../importEntity'; +import importFile from '../importFile'; +import entities from 'api/entities'; +import thesauri from 'api/thesauri'; +// import typeParsers from '../../typeParsers'; + +const fixtureFactory = getFixturesFactory(); + +const fixtures = { + dictionaries: [ + fixtureFactory.thesauri('select_thesaurus', ['A']), + fixtureFactory.thesauri('multiselect_thesaurus', ['A', 'B']), + ], + templates: [ + fixtureFactory.template('template', [ + fixtureFactory.property('unrelated_property', 'text'), + fixtureFactory.property('select_property', 'select', { + content: fixtureFactory.id('select_thesaurus'), + }), + fixtureFactory.property('multiselect_property', 'multiselect', { + content: fixtureFactory.id('multiselect_thesaurus'), + }), + ]), + ], + entities: [ + fixtureFactory.entity('existing_entity_id', 'template', { + unrelated_property: fixtureFactory.metadataValue('unrelated_value'), + select_property: fixtureFactory.metadataValue('A'), + multiselect_property: [fixtureFactory.metadataValue('A'), fixtureFactory.metadataValue('B')], + }), + ], + settings: [ + { + _id: db.id(), + site_name: 'Uwazi', + languages: [{ key: 'en', label: 'English', default: true }], + }, + ], +}; + +describe('arrangeThesauri', () => { + let file; + let fileSpy; + let template; + let selectThesaurus; + let selectLabels; + let selectLabelsSet; + let multiselectThesaurus; + let multiselectLabels; + let multiselectLabelsSet; + + beforeAll(async () => { + await db.clearAllAndLoad(fixtures); + template = await templates.getById(fixtureFactory.id('template')); + file = importFile(path.join(__dirname, '/arrangeThesauriTest.csv')); + await arrangeThesauri(file, template); + selectThesaurus = await thesauri.getById(fixtureFactory.id('select_thesaurus')); + selectLabels = selectThesaurus.values.map(tv => tv.label); + selectLabelsSet = new Set(selectLabels); + multiselectThesaurus = await thesauri.getById(fixtureFactory.id('multiselect_thesaurus')); + multiselectLabels = multiselectThesaurus.values.map(tv => tv.label); + multiselectLabelsSet = new Set(multiselectLabels); + }); + afterAll(async () => { + db.disconnect(); + fileSpy.mockRestore(); + }); + + it('dummy_test_delete_when_done', async () => { + fail('dummy_test_delete_when_done') + }); + + it('should create values in thesauri', async () => { + expect(selectLabels).toEqual(['A', 'B', 'C', 'd']); + expect(multiselectLabels).toEqual(['A', 'B', 'c', 'D', 'E', 'g']); + console.log(selectThesaurus); + console.log(multiselectThesaurus); + }); + + it('should not repeat case sensitive values', async () => { + ['a', 'b', 'c', 'D'].forEach(letter => expect(selectLabelsSet.has(letter)).toBe(false)); + ['a', 'b', 'C', 'd', 'e', 'G'].forEach(letter => + expect(multiselectLabelsSet.has(letter)).toBe(false) + ); + }); + + it('should not add values with trimmable white space or blank values', async () => { + selectLabels.forEach(label => { + expect(label).toBe(label.trim()); + }); + multiselectLabels.forEach(label => { + expect(label).toBe(label.trim()); + }); + }); + + it('should not create repeated values', async () => { + expect(selectLabels.length).toBe(selectLabelsSet.size); + expect(multiselectLabels.length).toBe(multiselectLabelsSet.size); + }); +}); diff --git a/app/api/csv/typeParsers/multiselect.ts b/app/api/csv/typeParsers/multiselect.ts index db38fb6cd2..7eb338e332 100644 --- a/app/api/csv/typeParsers/multiselect.ts +++ b/app/api/csv/typeParsers/multiselect.ts @@ -5,6 +5,32 @@ import { ThesaurusSchema } from 'shared/types/thesaurusType'; import { MetadataObjectSchema, PropertySchema } from 'shared/types/commonTypes'; import { ensure } from 'shared/tsUtils'; +import { normalizeThesaurusLabel } from './select'; + +function labelNotNull(label: string | null): label is string { + return label !== null; +} + +export function splitMultiselectLabels(labelString: string): { [k: string]: string } { + const labels = labelString + .split('|') + .map(l => l.trim()) + .filter(l => l.length > 0); + const result: { [k: string]: string } = {}; + labels.forEach(label => { + const normalizedLabel = normalizeThesaurusLabel(label); + if (labelNotNull(normalizedLabel) && !result.hasOwnProperty(normalizedLabel)) { + result[normalizedLabel] = label; + } + }); + return result; +} + +export function normalizeMultiselectLabels(labelArray: string[]): string[] { + const normalizedLabels = labelArray.map(l => normalizeThesaurusLabel(l)).filter(labelNotNull); + return Array.from(new Set(normalizedLabels)); +} + const multiselect = async ( entityToImport: RawEntity, property: PropertySchema diff --git a/app/api/csv/typeParsers/select.ts b/app/api/csv/typeParsers/select.ts index 9f22043dfb..054cf67da6 100644 --- a/app/api/csv/typeParsers/select.ts +++ b/app/api/csv/typeParsers/select.ts @@ -3,15 +3,26 @@ import { RawEntity } from 'api/csv/entityRow'; import { ThesaurusValueSchema, ThesaurusSchema } from 'shared/types/thesaurusType'; import { MetadataObjectSchema, PropertySchema } from 'shared/types/commonTypes'; import { ensure } from 'shared/tsUtils'; +const { performance } = require('perf_hooks') + +export var timeSpentInSelect = 0; + +export function normalizeThesaurusLabel(label: string): string | null { + const trimmed = label.trim().toLowerCase(); + return trimmed.length > 0 ? trimmed : null; +} const select = async ( entityToImport: RawEntity, property: PropertySchema ): Promise => { + const t1 = performance.now() const currentThesauri = (await thesauri.getById(property.content)) || ({} as ThesaurusSchema); const thesauriValues = currentThesauri.values || []; if (entityToImport[ensure(property.name)].trim() === '') { + const t2 = performance.now() + timeSpentInSelect += t2 - t1; return null; } @@ -32,6 +43,8 @@ const select = async ( }); value = (updated.values || []).find(thesauriMatching); } + const t2 = performance.now() + timeSpentInSelect += t2 - t1; if (value?.id) { return [{ value: value.id, label: value.label }]; diff --git a/app/api/files/routes.ts b/app/api/files/routes.ts index fda81d46a0..7f3fc9b8b8 100644 --- a/app/api/files/routes.ts +++ b/app/api/files/routes.ts @@ -12,6 +12,8 @@ import { fileSchema } from 'shared/types/fileSchema'; import { files } from './files'; import { validation, createError, handleError } from '../utils'; +import { timeSpentInSelect } from 'api/csv/typeParsers/select'; + export default (app: Application) => { app.post( '/api/files/upload/document', @@ -201,7 +203,9 @@ export default (app: Application) => { }, }), - (req, res) => { + async (req, res) => { + console.log('---------------/api/import') + console.time('Entire api call time:'); const loader = new CSVLoader(); let loaded = 0; @@ -215,7 +219,8 @@ export default (app: Application) => { }); req.emitToSessionSocket('IMPORT_CSV_START'); - loader + + await loader .load(req.file.path, req.body.template, { language: req.language, user: req.user }) .then(() => { req.emitToSessionSocket('IMPORT_CSV_END'); @@ -225,6 +230,8 @@ export default (app: Application) => { }); res.json('ok'); + console.log(`Time spent in select formatter: ${timeSpentInSelect}`) + console.timeEnd('Entire api call time:'); } ); }; From 51e7136ffd7ed9524c52f5de47e885f1bfaf8265 Mon Sep 17 00:00:00 2001 From: Laszlo Kecskes Date: Mon, 20 Sep 2021 11:41:46 +0200 Subject: [PATCH 07/52] added extra test --- app/api/csv/specs/importEntity.spec.js | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/app/api/csv/specs/importEntity.spec.js b/app/api/csv/specs/importEntity.spec.js index a4b7d19f32..f4e6506a07 100644 --- a/app/api/csv/specs/importEntity.spec.js +++ b/app/api/csv/specs/importEntity.spec.js @@ -7,6 +7,7 @@ import db from 'api/utils/testing_db'; // import fixtures, { template1Id } from './fixtures'; import { arrangeThesauri } from '../importEntity'; import importFile from '../importFile'; +import { stream } from './helpers'; import entities from 'api/entities'; import thesauri from 'api/thesauri'; // import typeParsers from '../../typeParsers'; @@ -28,6 +29,9 @@ const fixtures = { content: fixtureFactory.id('multiselect_thesaurus'), }), ]), + fixtureFactory.template('no_selects_template', [ + fixtureFactory.property('unrelated_property', 'text'), + ]), ], entities: [ fixtureFactory.entity('existing_entity_id', 'template', { @@ -77,6 +81,15 @@ describe('arrangeThesauri', () => { fail('dummy_test_delete_when_done') }); + it('should not fail on templates with no select or multiselect fields', async () => { + const noselTemplate = templates.getById(fixtureFactory.id('no_selects_template')); + const csv = `title,unrelated_text +first,first +second,second`; + console.log(csv); + await arrangeThesauri(importFile(stream(csv)), noselTemplate); + }); + it('should create values in thesauri', async () => { expect(selectLabels).toEqual(['A', 'B', 'C', 'd']); expect(multiselectLabels).toEqual(['A', 'B', 'c', 'D', 'E', 'g']); From 0de48cefbec5451e690bd30082575a467efaf2b0 Mon Sep 17 00:00:00 2001 From: Laszlo Kecskes Date: Mon, 20 Sep 2021 11:52:14 +0200 Subject: [PATCH 08/52] updated select typeparser --- app/api/csv/specs/fixtures.js | 8 ++++ app/api/csv/typeParsers/select.ts | 22 +++++----- app/api/csv/typeParsers/specs/select.spec.js | 42 ++------------------ 3 files changed, 23 insertions(+), 49 deletions(-) diff --git a/app/api/csv/specs/fixtures.js b/app/api/csv/specs/fixtures.js index c630fffa32..297808cbce 100644 --- a/app/api/csv/specs/fixtures.js +++ b/app/api/csv/specs/fixtures.js @@ -70,6 +70,14 @@ export default { _id: thesauri1Id, name: 'thesauri1', values: [ + { + label: 'value', + id: db.id().toString(), + }, + { + label: 'value2', + id: db.id().toString(), + }, { label: ' value4 ', id: db.id().toString(), diff --git a/app/api/csv/typeParsers/select.ts b/app/api/csv/typeParsers/select.ts index 054cf67da6..8e4263cda8 100644 --- a/app/api/csv/typeParsers/select.ts +++ b/app/api/csv/typeParsers/select.ts @@ -32,17 +32,17 @@ const select = async ( let value = thesauriValues.find(thesauriMatching); - if (!value) { - const updated = await thesauri.save({ - ...currentThesauri, - values: thesauriValues.concat([ - { - label: entityToImport[ensure(property.name)], - }, - ]), - }); - value = (updated.values || []).find(thesauriMatching); - } + // if (!value) { + // const updated = await thesauri.save({ + // ...currentThesauri, + // values: thesauriValues.concat([ + // { + // label: entityToImport[ensure(property.name)], + // }, + // ]), + // }); + // value = (updated.values || []).find(thesauriMatching); + // } const t2 = performance.now() timeSpentInSelect += t2 - t1; diff --git a/app/api/csv/typeParsers/specs/select.spec.js b/app/api/csv/typeParsers/specs/select.spec.js index 3158721233..9759fe9c42 100644 --- a/app/api/csv/typeParsers/specs/select.spec.js +++ b/app/api/csv/typeParsers/specs/select.spec.js @@ -10,57 +10,23 @@ describe('select', () => { beforeEach(async () => db.clearAllAndLoad(fixtures)); afterAll(async () => db.disconnect()); - it('should create thesauri value and return the id', async () => { + it('should find thesauri value and return the id and value', async () => { const templateProp = { name: 'select_prop', content: thesauri1Id }; const value1 = await typeParsers.select({ select_prop: 'value' }, templateProp); const value2 = await typeParsers.select({ select_prop: 'value2' }, templateProp); const thesauri1 = await thesauri.getById(thesauri1Id); - expect(thesauri1.values[1].label).toBe('value'); - expect(thesauri1.values[2].label).toBe('value2'); - expect(value1).toEqual([{ value: thesauri1.values[1].id, label: 'value' }]); - expect(value2).toEqual([{ value: thesauri1.values[2].id, label: 'value2' }]); + expect(value1).toEqual([{ value: thesauri1.values[0].id, label: 'value' }]); + expect(value2).toEqual([{ value: thesauri1.values[1].id, label: 'value2' }]); }); - it('should not repeat case sensitive values', async () => { - const templateProp = { name: 'select_prop', content: thesauri1Id }; - - await typeParsers.select({ select_prop: 'Value' }, templateProp); - await typeParsers.select({ select_prop: 'value ' }, templateProp); - - await typeParsers.select({ select_prop: 'vAlue2' }, templateProp); - await typeParsers.select({ select_prop: 'vAlue2' }, templateProp); - - await typeParsers.select({ select_prop: 'value4' }, templateProp); - await typeParsers.select({ select_prop: 'ValUe4' }, templateProp); - - const thesauri1 = await thesauri.getById(thesauri1Id); - - expect(thesauri1.values.map(v => v.label)).toEqual([' value4 ', 'Value', 'vAlue2']); - }); - - it('should not create repeated values', async () => { - const templateProp = { name: 'select_prop', content: thesauri1Id }; - - await typeParsers.select({ select_prop: 'value4' }, templateProp); - await typeParsers.select({ select_prop: 'value ' }, templateProp); - await typeParsers.select({ select_prop: 'value' }, templateProp); - await typeParsers.select({ select_prop: ' value ' }, templateProp); - await typeParsers.select({ select_prop: 'value4' }, templateProp); - const thesauri1 = await thesauri.getById(thesauri1Id); - - expect(thesauri1.values.length).toBe(2); - }); - - it('should not create blank values', async () => { + it('should return null on blank values', async () => { const templateProp = { name: 'select_prop', content: thesauri1Id }; const rawEntity = { select_prop: ' ' }; const value = await typeParsers.select(rawEntity, templateProp); - const thesauri1 = await thesauri.getById(thesauri1Id); - expect(thesauri1.values.length).toBe(1); expect(value).toBe(null); }); }); From a99bf66e9b9bf8f086bdab2eeb1ffd30380d024f Mon Sep 17 00:00:00 2001 From: Laszlo Kecskes Date: Tue, 21 Sep 2021 13:30:35 +0200 Subject: [PATCH 09/52] updated multiselect typeparser and failing tests --- app/api/csv/csvLoader.ts | 8 ++- app/api/csv/importEntity.ts | 40 ++++++------- app/api/csv/importFile.ts | 4 ++ .../__snapshots__/importFile.spec.ts.snap | 2 +- app/api/csv/specs/arrangeThesauriTest.csv | 2 +- app/api/csv/specs/csvLoader.spec.js | 6 +- app/api/csv/specs/csvLoaderFixtures.ts | 2 +- app/api/csv/specs/fixtures.js | 6 +- app/api/csv/specs/helpers.js | 26 ++++++--- app/api/csv/specs/importEntity.spec.js | 7 --- app/api/csv/specs/test.csv | 2 +- app/api/csv/typeParsers/multiselect.ts | 57 ++++++++++++------- app/api/csv/typeParsers/select.ts | 11 ++-- .../csv/typeParsers/specs/multiselect.spec.js | 29 ++++------ app/api/csv/typeParsers/specs/select.spec.js | 6 +- 15 files changed, 116 insertions(+), 92 deletions(-) diff --git a/app/api/csv/csvLoader.ts b/app/api/csv/csvLoader.ts index dd2af9be60..4026f94a49 100644 --- a/app/api/csv/csvLoader.ts +++ b/app/api/csv/csvLoader.ts @@ -56,7 +56,7 @@ export class CSVLoader extends EventEmitter { throw new Error('template not found!'); } // console.log(`-----template\n${JSON.stringify(template, null, 2)}\n-----done`) - const file = importFile(csvPath); + let file = importFile(csvPath); // console.log(`-----file\n${file}\n-----done`) const availableLanguages: string[] = ensure( (await settings.get()).languages @@ -67,16 +67,18 @@ export class CSVLoader extends EventEmitter { await arrangeThesauri(file, template); console.timeEnd('time spent in arrageThesauri'); + console.log(await thesauri.get()); + await csv(await file.readStream(), this.stopOnError) .onRow(async (row: CSVRow) => { - // console.log(row) + console.log(row) const { rawEntity, rawTranslations } = extractEntity( row, availableLanguages, options.language, newNameGeneration ); - // console.log(`-----rawEntity\n${JSON.stringify(rawEntity, null, 2)}\n-----done`) + console.log(`-----rawEntity\n${JSON.stringify(rawEntity, null, 2)}\n-----done`) // console.log(`-----rawTranslations\n${JSON.stringify(rawTranslations, null, 2)}\n-----done`) if (rawEntity) { diff --git a/app/api/csv/importEntity.ts b/app/api/csv/importEntity.ts index fa180d35c8..0b3f1a8cd7 100644 --- a/app/api/csv/importEntity.ts +++ b/app/api/csv/importEntity.ts @@ -119,32 +119,34 @@ const arrangeThesauri = async (file: ImportFile, template: TemplateSchema) => { if (readCount % 1000 === 0) { console.log(readCount); } - Object.entries(nameToThesauriIdSelects).forEach(entry => { - const name = entry[0]; - const id = entry[1]; + Object.entries(nameToThesauriIdSelects).forEach(([name, id]) => { const label = row[name]; - const normalizedLabel = normalizeThesaurusLabel(label); - if ( - normalizedLabel && - !thesauriIdToExistingValues.get(id).has(normalizedLabel) && - !thesauriIdToNormalizedNewValues.get(id).has(normalizedLabel) - ) { - thesauriIdToNewValues.get(id)?.add(label); - thesauriIdToNormalizedNewValues.get(id).add(normalizedLabel); - } - }); - Object.entries(nameToThesauriIdMultiselects).forEach(([name, id]) => { - const labels = splitMultiselectLabels(row[name]); - Object.entries(labels).forEach(([normalizedLabel, originalLabel]) => { + if (label) { + const normalizedLabel = normalizeThesaurusLabel(label); if ( normalizedLabel && !thesauriIdToExistingValues.get(id).has(normalizedLabel) && !thesauriIdToNormalizedNewValues.get(id).has(normalizedLabel) ) { - thesauriIdToNewValues.get(id)?.add(originalLabel); + thesauriIdToNewValues.get(id)?.add(label); thesauriIdToNormalizedNewValues.get(id).add(normalizedLabel); } - }); + } + }); + Object.entries(nameToThesauriIdMultiselects).forEach(([name, id]) => { + const labels = splitMultiselectLabels(row[name]); + if (labels) { + Object.entries(labels).forEach(([normalizedLabel, originalLabel]) => { + if ( + normalizedLabel && + !thesauriIdToExistingValues.get(id).has(normalizedLabel) && + !thesauriIdToNormalizedNewValues.get(id).has(normalizedLabel) + ) { + thesauriIdToNewValues.get(id)?.add(originalLabel); + thesauriIdToNormalizedNewValues.get(id).add(normalizedLabel); + } + }); + } }); }) .onError(async (e: Error, row: CSVRow, index: number) => { @@ -182,8 +184,8 @@ const importEntity = async ( const { attachments } = toImportEntity; delete toImportEntity.attachments; const eo = await entityObject(toImportEntity, template, { language }); + console.log(eo) const entity = await entities.save(eo, { user, language }, true, false); // ISSUE_2: then saves the entity as well - //ISSUE_3: same with documents if (toImportEntity.file && entity.sharedId) { const file = await importFile.extractFile(toImportEntity.file); diff --git a/app/api/csv/importFile.ts b/app/api/csv/importFile.ts index 5ab2db8a81..52442f48e3 100644 --- a/app/api/csv/importFile.ts +++ b/app/api/csv/importFile.ts @@ -5,6 +5,7 @@ import { Readable } from 'stream'; import { generateFileName, fileFromReadStream, uploadsPath } from 'api/files/filesystem'; import { createError } from 'api/utils'; import zipFile from 'api/utils/zipFile'; +import { ReadableString } from './specs/helpers'; const extractFromZip = async (zipPath: string, fileName: string) => { const readStream = await zipFile(zipPath).findReadStream(entry => entry === fileName); @@ -24,6 +25,9 @@ export class ImportFile { } async readStream(fileName = 'import.csv') { + if (this.filePath instanceof ReadableString) { + return this.filePath.freshCopy(); + } if (this.filePath instanceof Readable) { return this.filePath; } diff --git a/app/api/csv/specs/__snapshots__/importFile.spec.ts.snap b/app/api/csv/specs/__snapshots__/importFile.spec.ts.snap index 748622c07e..44f7c9edb1 100644 --- a/app/api/csv/specs/__snapshots__/importFile.spec.ts.snap +++ b/app/api/csv/specs/__snapshots__/importFile.spec.ts.snap @@ -1,7 +1,7 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP exports[`importFile readStream should return a readable stream for the csv file 1`] = ` -"Title , text label , numeric label, non configured, select label, not defined type, geolocation_geolocation,auto id, additional tag(s) +"Title , text label , numeric label, non configured, select_label, not defined type, geolocation_geolocation,auto id, additional tag(s) title1, text value 1, 1977, ______________, thesauri1 , notType1 , 1|1,,tag1 title2, text value 2, 2019, ______________, thesauri2 , notType2 , ,,tag2 diff --git a/app/api/csv/specs/arrangeThesauriTest.csv b/app/api/csv/specs/arrangeThesauriTest.csv index 474bf27bed..fd3627b235 100644 --- a/app/api/csv/specs/arrangeThesauriTest.csv +++ b/app/api/csv/specs/arrangeThesauriTest.csv @@ -1,4 +1,4 @@ -title,unrelated_property,select_property,multiselect_property +title,unrelated_property, select_property ,multiselect_property select_1,unrelated_text,B,A select_2,unrelated_text,C,A select_3,unrelated_text,b,A diff --git a/app/api/csv/specs/csvLoader.spec.js b/app/api/csv/specs/csvLoader.spec.js index b94f08ca01..abd08a8ab1 100644 --- a/app/api/csv/specs/csvLoader.spec.js +++ b/app/api/csv/specs/csvLoader.spec.js @@ -4,10 +4,11 @@ import entities from 'api/entities'; import path from 'path'; import translations from 'api/i18n'; import { search } from 'api/search'; +import thesauri from 'api/thesauri'; import { CSVLoader } from 'api/csv'; import { templateWithGeneratedTitle } from 'api/csv/specs/csvLoaderFixtures'; -import fixtures, { template1Id } from './csvLoaderFixtures'; +import fixtures, { template1Id, thesauri1Id } from './csvLoaderFixtures'; import { stream } from './helpers'; import typeParsers from '../typeParsers'; @@ -116,7 +117,6 @@ describe('csvLoader', () => { loader.on('entityLoaded', entity => { events.push(entity.title); }); - try { await loader.load(csvFile, template1Id, { language: 'en' }); } catch (e) { @@ -288,7 +288,7 @@ describe('csvLoader', () => { }); describe('when sharedId is provided', () => { - it('should update the entitiy', async () => { + it('should update the entity', async () => { const entity = await entities.save( { title: 'entity4444', template: template1Id }, { user: {}, language: 'en' } diff --git a/app/api/csv/specs/csvLoaderFixtures.ts b/app/api/csv/specs/csvLoaderFixtures.ts index ff02a39835..558c38797d 100644 --- a/app/api/csv/specs/csvLoaderFixtures.ts +++ b/app/api/csv/specs/csvLoaderFixtures.ts @@ -151,4 +151,4 @@ export default { ], }; -export { template1Id, templateWithGeneratedTitle }; +export { template1Id, templateWithGeneratedTitle, thesauri1Id }; diff --git a/app/api/csv/specs/fixtures.js b/app/api/csv/specs/fixtures.js index 297808cbce..10fcf04adc 100644 --- a/app/api/csv/specs/fixtures.js +++ b/app/api/csv/specs/fixtures.js @@ -71,13 +71,17 @@ export default { name: 'thesauri1', values: [ { - label: 'value', + label: 'value1', id: db.id().toString(), }, { label: 'value2', id: db.id().toString(), }, + { + label: 'Value3', + id: db.id().toString(), + }, { label: ' value4 ', id: db.id().toString(), diff --git a/app/api/csv/specs/helpers.js b/app/api/csv/specs/helpers.js index 8d55a7a108..1b2ffbd227 100644 --- a/app/api/csv/specs/helpers.js +++ b/app/api/csv/specs/helpers.js @@ -18,12 +18,22 @@ const createTestingZip = (filesToZip, fileName, directory = __dirname) => .on('error', reject); }); -const stream = string => - new Readable({ - read() { - this.push(string); - this.push(null); - }, - }); +class ReadableString extends Readable { + constructor(input) { + super(); + this.input = input; + } + + freshCopy() { + return new ReadableString(this.input); + } + + _read() { + this.push(this.input); + this.push(null); + } +} + +const stream = string => new ReadableString(string); -export { stream, createTestingZip }; +export { stream, createTestingZip, ReadableString }; diff --git a/app/api/csv/specs/importEntity.spec.js b/app/api/csv/specs/importEntity.spec.js index f4e6506a07..3bc8d5e242 100644 --- a/app/api/csv/specs/importEntity.spec.js +++ b/app/api/csv/specs/importEntity.spec.js @@ -77,24 +77,17 @@ describe('arrangeThesauri', () => { fileSpy.mockRestore(); }); - it('dummy_test_delete_when_done', async () => { - fail('dummy_test_delete_when_done') - }); - it('should not fail on templates with no select or multiselect fields', async () => { const noselTemplate = templates.getById(fixtureFactory.id('no_selects_template')); const csv = `title,unrelated_text first,first second,second`; - console.log(csv); await arrangeThesauri(importFile(stream(csv)), noselTemplate); }); it('should create values in thesauri', async () => { expect(selectLabels).toEqual(['A', 'B', 'C', 'd']); expect(multiselectLabels).toEqual(['A', 'B', 'c', 'D', 'E', 'g']); - console.log(selectThesaurus); - console.log(multiselectThesaurus); }); it('should not repeat case sensitive values', async () => { diff --git a/app/api/csv/specs/test.csv b/app/api/csv/specs/test.csv index 227513f50f..93a6521abe 100644 --- a/app/api/csv/specs/test.csv +++ b/app/api/csv/specs/test.csv @@ -1,4 +1,4 @@ -Title , text label , numeric label, non configured, select label, not defined type, geolocation_geolocation,auto id, additional tag(s) +Title , text label , numeric label, non configured, select_label, not defined type, geolocation_geolocation,auto id, additional tag(s) title1, text value 1, 1977, ______________, thesauri1 , notType1 , 1|1,,tag1 title2, text value 2, 2019, ______________, thesauri2 , notType2 , ,,tag2 diff --git a/app/api/csv/typeParsers/multiselect.ts b/app/api/csv/typeParsers/multiselect.ts index 7eb338e332..f334e84a38 100644 --- a/app/api/csv/typeParsers/multiselect.ts +++ b/app/api/csv/typeParsers/multiselect.ts @@ -38,31 +38,46 @@ const multiselect = async ( const currentThesauri = (await thesauri.getById(property.content)) || ({} as ThesaurusSchema); const thesauriValues = currentThesauri.values || []; - const values = entityToImport[ensure(property.name)] - .split('|') - .map(v => v.trim()) - .filter(emptyString) - .filter(unique); + const values = splitMultiselectLabels(entityToImport[ensure(property.name)]); - const newValues = values.filter( - v => !thesauriValues.find(cv => cv.label.trim().toLowerCase() === v.toLowerCase()) - ); + const result = Object.keys(values) + .map(key => thesauriValues.find(tv => normalizeThesaurusLabel(tv.label) === key)) + .map(tv => tv) + .map(tv => ({ value: ensure(tv?.id), label: ensure(tv?.label) })); - const lowerCaseValues = values.map(v => v.toLowerCase()); - if (!newValues.length) { - return thesauriValues - .filter(value => lowerCaseValues.includes(value.label.trim().toLowerCase())) - .map(value => ({ value: ensure(value.id), label: value.label })); - } + // const result = Object.keys(values).map(key => { + // const found = thesauriValues.find(tv => normalizeThesaurusLabel(tv.label) === key); + // return found !== undefined + // ? { value: ensure(found.id), label: found.label } + // : undefined; + // }); + // const values = entityToImport[ensure(property.name)] + // .split('|') + // .map(v => v.trim()) + // .filter(emptyString) + // .filter(unique); - const updated = await thesauri.save({ - ...currentThesauri, - values: thesauriValues.concat(newValues.map(label => ({ label }))), - }); + // const newValues = values.filter( + // v => !thesauriValues.find(cv => cv.label.trim().toLowerCase() === v.toLowerCase()) + // ); + + // const lowerCaseValues = values.map(v => v.toLowerCase()); + // if (!newValues.length) { + // return thesauriValues + // .filter(value => lowerCaseValues.includes(value.label.trim().toLowerCase())) + // .map(value => ({ value: ensure(value.id), label: value.label })); + // } - return (updated.values || []) - .filter(value => lowerCaseValues.includes(value.label.toLowerCase())) - .map(value => ({ value: ensure(value.id), label: value.label })); + // const updated = await thesauri.save({ + // ...currentThesauri, + // values: thesauriValues.concat(newValues.map(label => ({ label }))), + // }); + + // return (updated.values || []) + // .filter(value => lowerCaseValues.includes(value.label.toLowerCase())) + // .map(value => ({ value: ensure(value.id), label: value.label })); + + return result; }; export default multiselect; diff --git a/app/api/csv/typeParsers/select.ts b/app/api/csv/typeParsers/select.ts index 8e4263cda8..123a829371 100644 --- a/app/api/csv/typeParsers/select.ts +++ b/app/api/csv/typeParsers/select.ts @@ -7,7 +7,10 @@ const { performance } = require('perf_hooks') export var timeSpentInSelect = 0; -export function normalizeThesaurusLabel(label: string): string | null { +export function normalizeThesaurusLabel(label?: string | null): string | null { + if (label === undefined || label === null) { + return null; + } const trimmed = label.trim().toLowerCase(); return trimmed.length > 0 ? trimmed : null; } @@ -20,15 +23,15 @@ const select = async ( const currentThesauri = (await thesauri.getById(property.content)) || ({} as ThesaurusSchema); const thesauriValues = currentThesauri.values || []; - if (entityToImport[ensure(property.name)].trim() === '') { + if (normalizeThesaurusLabel(entityToImport[ensure(property.name)]) === '') { const t2 = performance.now() timeSpentInSelect += t2 - t1; return null; } const thesauriMatching = (v: ThesaurusValueSchema) => - v.label.trim().toLowerCase() === - entityToImport[ensure(property.name)].trim().toLowerCase(); + normalizeThesaurusLabel(v.label) === + normalizeThesaurusLabel(entityToImport[ensure(property.name)]); let value = thesauriValues.find(thesauriMatching); diff --git a/app/api/csv/typeParsers/specs/multiselect.spec.js b/app/api/csv/typeParsers/specs/multiselect.spec.js index da055fa4d9..dfe21122e4 100644 --- a/app/api/csv/typeParsers/specs/multiselect.spec.js +++ b/app/api/csv/typeParsers/specs/multiselect.spec.js @@ -42,30 +42,21 @@ describe('multiselect', () => { thesauri1 = await thesauri.getById(thesauri1Id); }); - it('should create thesauri values and return an array of ids', async () => { - expect(thesauri1.values[0].label).toBe(' value4 '); - expect(thesauri1.values[1].label).toBe('Value1'); - expect(thesauri1.values[2].label).toBe('value3'); - expect(thesauri1.values[3].label).toBe('value2'); - - expect(value1).toEqual([{ value: thesauri1.values[0].id, label: ' value4 ' }]); + it('should find values in thesauri and return an array of ids and labels', async () => { + expect(value1).toEqual([{ value: thesauri1.values[3].id, label: ' value4 ' }]); expect(value2).toEqual([ - { value: thesauri1.values[1].id, label: 'Value1' }, - { value: thesauri1.values[2].id, label: 'value3' }, + { value: thesauri1.values[0].id, label: 'value1' }, + { value: thesauri1.values[2].id, label: 'Value3' }, ]); expect(value3).toEqual([ - { value: thesauri1.values[1].id, label: 'Value1' }, - { value: thesauri1.values[2].id, label: 'value3' }, - { value: thesauri1.values[3].id, label: 'value2' }, + { value: thesauri1.values[0].id, label: 'value1' }, + { value: thesauri1.values[1].id, label: 'value2' }, + { value: thesauri1.values[2].id, label: 'Value3' }, ]); expect(value4).toEqual([ - { value: thesauri1.values[0].id, label: ' value4 ' }, - { value: thesauri1.values[1].id, label: 'Value1' }, - { value: thesauri1.values[3].id, label: 'value2' }, + { value: thesauri1.values[0].id, label: 'value1' }, + { value: thesauri1.values[1].id, label: 'value2' }, + { value: thesauri1.values[3].id, label: ' value4 ' }, ]); }); - - it('should not create blank values, or repeat values', async () => { - expect(thesauri1.values.length).toBe(4); - }); }); diff --git a/app/api/csv/typeParsers/specs/select.spec.js b/app/api/csv/typeParsers/specs/select.spec.js index 9759fe9c42..a7e573e26f 100644 --- a/app/api/csv/typeParsers/specs/select.spec.js +++ b/app/api/csv/typeParsers/specs/select.spec.js @@ -13,11 +13,11 @@ describe('select', () => { it('should find thesauri value and return the id and value', async () => { const templateProp = { name: 'select_prop', content: thesauri1Id }; - const value1 = await typeParsers.select({ select_prop: 'value' }, templateProp); - const value2 = await typeParsers.select({ select_prop: 'value2' }, templateProp); + const value1 = await typeParsers.select({ select_prop: 'value1' }, templateProp); + const value2 = await typeParsers.select({ select_prop: 'vAlUe2' }, templateProp); const thesauri1 = await thesauri.getById(thesauri1Id); - expect(value1).toEqual([{ value: thesauri1.values[0].id, label: 'value' }]); + expect(value1).toEqual([{ value: thesauri1.values[0].id, label: 'value1' }]); expect(value2).toEqual([{ value: thesauri1.values[1].id, label: 'value2' }]); }); From baa367e06b4623bb0eb473c2b1c6b036c8abf992 Mon Sep 17 00:00:00 2001 From: Laszlo Kecskes Date: Tue, 21 Sep 2021 13:45:41 +0200 Subject: [PATCH 10/52] code cleanup --- app/api/csv/csvLoader.ts | 20 +-------- app/api/csv/importEntity.ts | 61 ++++++++------------------ app/api/csv/specs/importEntity.spec.js | 5 +-- app/api/csv/typeParsers/multiselect.ts | 33 -------------- app/api/csv/typeParsers/select.ts | 22 +--------- app/api/files/routes.ts | 6 +-- 6 files changed, 23 insertions(+), 124 deletions(-) diff --git a/app/api/csv/csvLoader.ts b/app/api/csv/csvLoader.ts index 4026f94a49..59b1cb226f 100644 --- a/app/api/csv/csvLoader.ts +++ b/app/api/csv/csvLoader.ts @@ -16,8 +16,6 @@ import importFile from './importFile'; import { arrangeThesauri, importEntity, translateEntity } from './importEntity'; import { extractEntity, toSafeName } from './entityRow'; -const { performance } = require('perf_hooks') - export class CSVLoader extends EventEmitter { stopOnError: boolean; @@ -49,45 +47,30 @@ export class CSVLoader extends EventEmitter { templateId: ObjectId | string, options = { language: 'en', user: {} } ) { - // console.log('---------------csvLoader.load') - let timeInImportAndTranslate = 0; const template = await templates.getById(templateId); if (!template) { throw new Error('template not found!'); } - // console.log(`-----template\n${JSON.stringify(template, null, 2)}\n-----done`) - let file = importFile(csvPath); - // console.log(`-----file\n${file}\n-----done`) + const file = importFile(csvPath); const availableLanguages: string[] = ensure( (await settings.get()).languages ).map((l: LanguageSchema) => l.key); const { newNameGeneration = false } = await settings.get(); - // console.log(`-----availableLanguages\n${availableLanguages}\n-----done`) - console.time('time spent in arrageThesauri'); await arrangeThesauri(file, template); - console.timeEnd('time spent in arrageThesauri'); - - console.log(await thesauri.get()); await csv(await file.readStream(), this.stopOnError) .onRow(async (row: CSVRow) => { - console.log(row) const { rawEntity, rawTranslations } = extractEntity( row, availableLanguages, options.language, newNameGeneration ); - console.log(`-----rawEntity\n${JSON.stringify(rawEntity, null, 2)}\n-----done`) - // console.log(`-----rawTranslations\n${JSON.stringify(rawTranslations, null, 2)}\n-----done`) if (rawEntity) { - const t1 = performance.now() const entity = await importEntity(rawEntity, template, file, options); await translateEntity(entity, rawTranslations, template, file); this.emit('entityLoaded', entity); - const t2 = performance.now() - timeInImportAndTranslate += t2 - t1; } }) .onError(async (e: Error, row: CSVRow, index: number) => { @@ -95,7 +78,6 @@ export class CSVLoader extends EventEmitter { this.emit('loadError', e, toSafeName(row), index); }) .read(); - console.log(`Total time spent in importEntity and translateEntity:${timeInImportAndTranslate}`) this.throwErrors(); } diff --git a/app/api/csv/importEntity.ts b/app/api/csv/importEntity.ts index 0b3f1a8cd7..558be7cad1 100644 --- a/app/api/csv/importEntity.ts +++ b/app/api/csv/importEntity.ts @@ -21,7 +21,7 @@ import csv, { CSVRow } from './csv'; const parse = async (toImportEntity: RawEntity, prop: PropertySchema) => typeParsers[prop.type] - ? typeParsers[prop.type](toImportEntity, prop) // ISSUE_1: go down to select typeparser - db calls each row + ? typeParsers[prop.type](toImportEntity, prop) : typeParsers.text(toImportEntity, prop); const hasValidValue = (prop: PropertySchema, toImportEntity: RawEntity) => @@ -43,7 +43,7 @@ const toMetadata = async ( ); const currentEntityIdentifiers = async (sharedId: string, language: string) => - sharedId ? entities.get({ sharedId, language }, '_id sharedId').then(([e]) => e) : {}; //ISSUE_5 entity.get + sharedId ? entities.get({ sharedId, language }, '_id sharedId').then(([e]) => e) : {}; const titleByTemplate = (template: TemplateSchema, entity: RawEntity) => { const generatedTitle = @@ -71,6 +71,8 @@ type Options = { language: string; }; + + const arrangeThesauri = async (file: ImportFile, template: TemplateSchema) => { const nameToThesauriIdSelects: { [k: string]: string } = {}; const nameToThesauriIdMultiselects: { [k: string]: string } = {}; @@ -78,7 +80,7 @@ const arrangeThesauri = async (file: ImportFile, template: TemplateSchema) => { const thesauriIdToNewValues: Map> = new Map(); const thesauriIdToNormalizedNewValues = new Map(); const thesauriRelatedProperties = template.properties?.filter(p => - [propertyTypes.select, propertyTypes.multiselect].includes(p.type) + ['select', 'multiselect'].includes(p.type) ); thesauriRelatedProperties?.forEach(p => { if (p.content && p.type) { @@ -105,46 +107,30 @@ const arrangeThesauri = async (file: ImportFile, template: TemplateSchema) => { thesauriIdToNormalizedNewValues.set(id, new Set()); } }); - // console.log(nameToThesauriIdSelects) - // console.log(nameToThesauriIdMultiselects) - // console.log(allRelatedThesauri) - // console.log(thesauriIdToExistingValues) - // console.log(thesauriIdToNewValues); - // console.log(thesauriIdToNormalizedNewValues); - console.log('Reading'); - let readCount = 0; + function handleLabels(id: string, original: string, normalized: string | null) { + if ( + normalized && + !thesauriIdToExistingValues.get(id).has(normalized) && + !thesauriIdToNormalizedNewValues.get(id).has(normalized) + ) { + thesauriIdToNewValues.get(id)?.add(original); + thesauriIdToNormalizedNewValues.get(id).add(normalized); + } + } await csv(await file.readStream()) .onRow(async (row: CSVRow) => { - readCount += 1; - if (readCount % 1000 === 0) { - console.log(readCount); - } Object.entries(nameToThesauriIdSelects).forEach(([name, id]) => { const label = row[name]; if (label) { const normalizedLabel = normalizeThesaurusLabel(label); - if ( - normalizedLabel && - !thesauriIdToExistingValues.get(id).has(normalizedLabel) && - !thesauriIdToNormalizedNewValues.get(id).has(normalizedLabel) - ) { - thesauriIdToNewValues.get(id)?.add(label); - thesauriIdToNormalizedNewValues.get(id).add(normalizedLabel); - } + handleLabels(id, label, normalizedLabel); } }); Object.entries(nameToThesauriIdMultiselects).forEach(([name, id]) => { const labels = splitMultiselectLabels(row[name]); if (labels) { Object.entries(labels).forEach(([normalizedLabel, originalLabel]) => { - if ( - normalizedLabel && - !thesauriIdToExistingValues.get(id).has(normalizedLabel) && - !thesauriIdToNormalizedNewValues.get(id).has(normalizedLabel) - ) { - thesauriIdToNewValues.get(id)?.add(originalLabel); - thesauriIdToNormalizedNewValues.get(id).add(normalizedLabel); - } + handleLabels(id, originalLabel, normalizedLabel); }); } }); @@ -153,16 +139,12 @@ const arrangeThesauri = async (file: ImportFile, template: TemplateSchema) => { console.log(e); }) .read(); - console.log('Saving thesauri'); await Promise.all( allRelatedThesauri.map(thesaurus => { if (thesaurus !== null) { - // console.log(thesaurus.name); - // console.log(thesaurus._id); const newValues: { label: string }[] = Array.from( thesauriIdToNewValues.get(thesaurus._id.toString()) || [] ).map(tval => ({ label: tval })); - // console.log(newValues); const thesaurusValues = thesaurus.values || []; return thesauri.save({ ...thesaurus, @@ -171,8 +153,6 @@ const arrangeThesauri = async (file: ImportFile, template: TemplateSchema) => { } }) ); - // console.log(thesauriIdToNewValues); - // console.log(thesauriIdToNormalizedNewValues); }; const importEntity = async ( @@ -184,15 +164,13 @@ const importEntity = async ( const { attachments } = toImportEntity; delete toImportEntity.attachments; const eo = await entityObject(toImportEntity, template, { language }); - console.log(eo) const entity = await entities.save(eo, { user, language }, true, false); // ISSUE_2: then saves the entity as well - //ISSUE_3: same with documents + if (toImportEntity.file && entity.sharedId) { const file = await importFile.extractFile(toImportEntity.file); await processDocument(entity.sharedId, file); } - //ISSUE_4: same with attachments if (attachments && entity.sharedId) { await attachments.split('|').reduce(async (promise: Promise, attachment) => { await promise; @@ -212,7 +190,6 @@ const translateEntity = async ( importFile: ImportFile ) => { await entitiesModel.saveMultiple( - // ISSUE_6: this also just maps saves await Promise.all( translations.map(async translatedEntity => entityObject({ ...translatedEntity, id: ensure(entity.sharedId) }, template, { @@ -226,7 +203,7 @@ const translateEntity = async ( translations.map(async translatedEntity => { if (translatedEntity.file) { const file = await importFile.extractFile(translatedEntity.file); - await processDocument(ensure(entity.sharedId), file); //ISSUE_7: process document again + await processDocument(ensure(entity.sharedId), file); } }) ); diff --git a/app/api/csv/specs/importEntity.spec.js b/app/api/csv/specs/importEntity.spec.js index 3bc8d5e242..b40a93b31b 100644 --- a/app/api/csv/specs/importEntity.spec.js +++ b/app/api/csv/specs/importEntity.spec.js @@ -1,16 +1,13 @@ import path from 'path'; import templates from 'api/templates'; +import thesauri from 'api/thesauri'; import { getFixturesFactory } from 'api/utils/fixturesFactory'; import db from 'api/utils/testing_db'; -// import fixtures, { template1Id } from './fixtures'; import { arrangeThesauri } from '../importEntity'; import importFile from '../importFile'; import { stream } from './helpers'; -import entities from 'api/entities'; -import thesauri from 'api/thesauri'; -// import typeParsers from '../../typeParsers'; const fixtureFactory = getFixturesFactory(); diff --git a/app/api/csv/typeParsers/multiselect.ts b/app/api/csv/typeParsers/multiselect.ts index f334e84a38..9e208cf428 100644 --- a/app/api/csv/typeParsers/multiselect.ts +++ b/app/api/csv/typeParsers/multiselect.ts @@ -1,5 +1,4 @@ import thesauri from 'api/thesauri'; -import { unique, emptyString } from 'api/utils/filters'; import { RawEntity } from 'api/csv/entityRow'; import { ThesaurusSchema } from 'shared/types/thesaurusType'; import { MetadataObjectSchema, PropertySchema } from 'shared/types/commonTypes'; @@ -45,38 +44,6 @@ const multiselect = async ( .map(tv => tv) .map(tv => ({ value: ensure(tv?.id), label: ensure(tv?.label) })); - // const result = Object.keys(values).map(key => { - // const found = thesauriValues.find(tv => normalizeThesaurusLabel(tv.label) === key); - // return found !== undefined - // ? { value: ensure(found.id), label: found.label } - // : undefined; - // }); - // const values = entityToImport[ensure(property.name)] - // .split('|') - // .map(v => v.trim()) - // .filter(emptyString) - // .filter(unique); - - // const newValues = values.filter( - // v => !thesauriValues.find(cv => cv.label.trim().toLowerCase() === v.toLowerCase()) - // ); - - // const lowerCaseValues = values.map(v => v.toLowerCase()); - // if (!newValues.length) { - // return thesauriValues - // .filter(value => lowerCaseValues.includes(value.label.trim().toLowerCase())) - // .map(value => ({ value: ensure(value.id), label: value.label })); - // } - - // const updated = await thesauri.save({ - // ...currentThesauri, - // values: thesauriValues.concat(newValues.map(label => ({ label }))), - // }); - - // return (updated.values || []) - // .filter(value => lowerCaseValues.includes(value.label.toLowerCase())) - // .map(value => ({ value: ensure(value.id), label: value.label })); - return result; }; diff --git a/app/api/csv/typeParsers/select.ts b/app/api/csv/typeParsers/select.ts index 123a829371..bf914ebc56 100644 --- a/app/api/csv/typeParsers/select.ts +++ b/app/api/csv/typeParsers/select.ts @@ -3,9 +3,6 @@ import { RawEntity } from 'api/csv/entityRow'; import { ThesaurusValueSchema, ThesaurusSchema } from 'shared/types/thesaurusType'; import { MetadataObjectSchema, PropertySchema } from 'shared/types/commonTypes'; import { ensure } from 'shared/tsUtils'; -const { performance } = require('perf_hooks') - -export var timeSpentInSelect = 0; export function normalizeThesaurusLabel(label?: string | null): string | null { if (label === undefined || label === null) { @@ -19,13 +16,10 @@ const select = async ( entityToImport: RawEntity, property: PropertySchema ): Promise => { - const t1 = performance.now() const currentThesauri = (await thesauri.getById(property.content)) || ({} as ThesaurusSchema); const thesauriValues = currentThesauri.values || []; if (normalizeThesaurusLabel(entityToImport[ensure(property.name)]) === '') { - const t2 = performance.now() - timeSpentInSelect += t2 - t1; return null; } @@ -33,21 +27,7 @@ const select = async ( normalizeThesaurusLabel(v.label) === normalizeThesaurusLabel(entityToImport[ensure(property.name)]); - let value = thesauriValues.find(thesauriMatching); - - // if (!value) { - // const updated = await thesauri.save({ - // ...currentThesauri, - // values: thesauriValues.concat([ - // { - // label: entityToImport[ensure(property.name)], - // }, - // ]), - // }); - // value = (updated.values || []).find(thesauriMatching); - // } - const t2 = performance.now() - timeSpentInSelect += t2 - t1; + const value = thesauriValues.find(thesauriMatching); if (value?.id) { return [{ value: value.id, label: value.label }]; diff --git a/app/api/files/routes.ts b/app/api/files/routes.ts index 7f3fc9b8b8..db7016bc79 100644 --- a/app/api/files/routes.ts +++ b/app/api/files/routes.ts @@ -204,8 +204,6 @@ export default (app: Application) => { }), async (req, res) => { - console.log('---------------/api/import') - console.time('Entire api call time:'); const loader = new CSVLoader(); let loaded = 0; @@ -219,7 +217,7 @@ export default (app: Application) => { }); req.emitToSessionSocket('IMPORT_CSV_START'); - + await loader .load(req.file.path, req.body.template, { language: req.language, user: req.user }) .then(() => { @@ -230,8 +228,6 @@ export default (app: Application) => { }); res.json('ok'); - console.log(`Time spent in select formatter: ${timeSpentInSelect}`) - console.timeEnd('Entire api call time:'); } ); }; From 9f5a68ef7fac7bdc9f45949bdac3bb9641a7915b Mon Sep 17 00:00:00 2001 From: Laszlo Kecskes Date: Tue, 21 Sep 2021 13:55:20 +0200 Subject: [PATCH 11/52] added error handling --- app/api/csv/csvLoader.ts | 2 +- app/api/csv/importEntity.ts | 13 +++++++------ app/api/files/routes.ts | 2 -- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/app/api/csv/csvLoader.ts b/app/api/csv/csvLoader.ts index 59b1cb226f..89fd4f115d 100644 --- a/app/api/csv/csvLoader.ts +++ b/app/api/csv/csvLoader.ts @@ -56,7 +56,7 @@ export class CSVLoader extends EventEmitter { (await settings.get()).languages ).map((l: LanguageSchema) => l.key); const { newNameGeneration = false } = await settings.get(); - await arrangeThesauri(file, template); + await arrangeThesauri(file, template, this); await csv(await file.readStream(), this.stopOnError) .onRow(async (row: CSVRow) => { diff --git a/app/api/csv/importEntity.ts b/app/api/csv/importEntity.ts index 558be7cad1..ee04384d12 100644 --- a/app/api/csv/importEntity.ts +++ b/app/api/csv/importEntity.ts @@ -3,7 +3,7 @@ import entities from 'api/entities'; import { search } from 'api/search'; import entitiesModel from 'api/entities/entitiesModel'; import { processDocument } from 'api/files/processDocument'; -import { RawEntity } from 'api/csv/entityRow'; +import { RawEntity, toSafeName } from 'api/csv/entityRow'; import { TemplateSchema } from 'shared/types/templateType'; import { MetadataSchema, PropertySchema } from 'shared/types/commonTypes'; import { propertyTypes } from 'shared/propertyTypes'; @@ -71,9 +71,7 @@ type Options = { language: string; }; - - -const arrangeThesauri = async (file: ImportFile, template: TemplateSchema) => { +const arrangeThesauri = async (file: ImportFile, template: TemplateSchema, errorContext?: any) => { const nameToThesauriIdSelects: { [k: string]: string } = {}; const nameToThesauriIdMultiselects: { [k: string]: string } = {}; const thesauriIdToExistingValues = new Map(); @@ -117,7 +115,7 @@ const arrangeThesauri = async (file: ImportFile, template: TemplateSchema) => { thesauriIdToNormalizedNewValues.get(id).add(normalized); } } - await csv(await file.readStream()) + await csv(await file.readStream(), errorContext?.stopOnError) .onRow(async (row: CSVRow) => { Object.entries(nameToThesauriIdSelects).forEach(([name, id]) => { const label = row[name]; @@ -136,7 +134,10 @@ const arrangeThesauri = async (file: ImportFile, template: TemplateSchema) => { }); }) .onError(async (e: Error, row: CSVRow, index: number) => { - console.log(e); + if (errorContext) { + errorContext._errors[index] = e; + errorContext.emit('loadError', e, toSafeName(row), index); + } }) .read(); await Promise.all( diff --git a/app/api/files/routes.ts b/app/api/files/routes.ts index db7016bc79..063d3bef0c 100644 --- a/app/api/files/routes.ts +++ b/app/api/files/routes.ts @@ -12,8 +12,6 @@ import { fileSchema } from 'shared/types/fileSchema'; import { files } from './files'; import { validation, createError, handleError } from '../utils'; -import { timeSpentInSelect } from 'api/csv/typeParsers/select'; - export default (app: Application) => { app.post( '/api/files/upload/document', From 4d01edd60a18cb037b741c2d053cfaa768ad2ea1 Mon Sep 17 00:00:00 2001 From: Laszlo Kecskes Date: Tue, 21 Sep 2021 14:04:38 +0200 Subject: [PATCH 12/52] fix code climate issues --- app/api/csv/specs/csvLoader.spec.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/app/api/csv/specs/csvLoader.spec.js b/app/api/csv/specs/csvLoader.spec.js index abd08a8ab1..3d3624eb48 100644 --- a/app/api/csv/specs/csvLoader.spec.js +++ b/app/api/csv/specs/csvLoader.spec.js @@ -4,11 +4,10 @@ import entities from 'api/entities'; import path from 'path'; import translations from 'api/i18n'; import { search } from 'api/search'; -import thesauri from 'api/thesauri'; import { CSVLoader } from 'api/csv'; import { templateWithGeneratedTitle } from 'api/csv/specs/csvLoaderFixtures'; -import fixtures, { template1Id, thesauri1Id } from './csvLoaderFixtures'; +import fixtures, { template1Id } from './csvLoaderFixtures'; import { stream } from './helpers'; import typeParsers from '../typeParsers'; From b43cfb3cce93c065593588bd9e5ec1d1ae5b5dac Mon Sep 17 00:00:00 2001 From: Laszlo Kecskes Date: Tue, 21 Sep 2021 14:11:04 +0200 Subject: [PATCH 13/52] further code cleanup --- app/api/csv/importEntity.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/api/csv/importEntity.ts b/app/api/csv/importEntity.ts index ee04384d12..5bfc54373c 100644 --- a/app/api/csv/importEntity.ts +++ b/app/api/csv/importEntity.ts @@ -165,7 +165,7 @@ const importEntity = async ( const { attachments } = toImportEntity; delete toImportEntity.attachments; const eo = await entityObject(toImportEntity, template, { language }); - const entity = await entities.save(eo, { user, language }, true, false); // ISSUE_2: then saves the entity as well + const entity = await entities.save(eo, { user, language }, true, false); if (toImportEntity.file && entity.sharedId) { const file = await importFile.extractFile(toImportEntity.file); @@ -176,7 +176,7 @@ const importEntity = async ( await attachments.split('|').reduce(async (promise: Promise, attachment) => { await promise; const attachmentFile = await importFile.extractFile(attachment, attachmentsPath()); - return files.save({ ...attachmentFile, entity: entity.sharedId, type: 'attachment' }); // <-----here + return files.save({ ...attachmentFile, entity: entity.sharedId, type: 'attachment' }); }, Promise.resolve()); } From 72d516ddd79bf59cf414a603fd12c8ca0457ff6f Mon Sep 17 00:00:00 2001 From: Santiago Date: Mon, 20 Sep 2021 16:31:29 -0300 Subject: [PATCH 14/52] search only happens when templates with geolocation exists --- app/react/Library/helpers/requestState.js | 82 ++++++++++++----------- 1 file changed, 43 insertions(+), 39 deletions(-) diff --git a/app/react/Library/helpers/requestState.js b/app/react/Library/helpers/requestState.js index 2697f30a4b..b7942d37ad 100644 --- a/app/react/Library/helpers/requestState.js +++ b/app/react/Library/helpers/requestState.js @@ -55,50 +55,54 @@ export default function requestState(request, globalResources, calculateTableCol const documentsRequest = request.set( tocGenerationUtils.aggregations(docsQuery, globalResources.settings.collection.toJS()) ); + const templatesWithGeolocation = globalResources.templates.find(template => + template.get('properties').find(property => property.get('type') === 'geolocation') + ); const markersRequest = request.set({ ...docsQuery, geolocation: true }); - return Promise.all([api.search(documentsRequest), api.search(markersRequest)]).then( - ([documents, markers]) => { - const templates = globalResources.templates.toJS(); - const filterState = libraryHelpers.URLQueryToState( - documentsRequest.data, - templates, - globalResources.thesauris.toJS(), - globalResources.relationTypes.toJS(), - request.data.quickLabelThesaurus - ? getThesaurusPropertyNames(request.data.quickLabelThesaurus, templates) - : [] - ); + return Promise.all([ + api.search(documentsRequest), + templatesWithGeolocation && api.search(markersRequest), + ]).then(([documents, markers]) => { + const templates = globalResources.templates.toJS(); + const filterState = libraryHelpers.URLQueryToState( + documentsRequest.data, + templates, + globalResources.thesauris.toJS(), + globalResources.relationTypes.toJS(), + request.data.quickLabelThesaurus + ? getThesaurusPropertyNames(request.data.quickLabelThesaurus, templates) + : [] + ); - const state = { - library: { - filters: { - documentTypes: documentsRequest.data.types || [], - properties: filterState.properties, - }, - aggregations: documents.aggregations, - search: filterState.search, - documents, - markers, + const state = { + library: { + filters: { + documentTypes: documentsRequest.data.types || [], + properties: filterState.properties, }, - }; + aggregations: documents.aggregations, + search: filterState.search, + documents, + markers, + }, + }; - const addinsteadOfSet = Boolean(docsQuery.from); + const addinsteadOfSet = Boolean(docsQuery.from); - const dispatchedActions = [ - setReduxState(state, 'library', addinsteadOfSet), - actions.set('library.sidepanel.quickLabelState', { - thesaurus: request.data.quickLabelThesaurus, - autoSave: false, - }), - ]; - if (calculateTableColumns) { - const tableViewColumns = getTableColumns(documents, templates, documentsRequest.data.types); - dispatchedActions.push(dispatch => - wrapDispatch(dispatch, 'library')(setTableViewColumns(tableViewColumns)) - ); - } - return dispatchedActions; + const dispatchedActions = [ + setReduxState(state, 'library', addinsteadOfSet), + actions.set('library.sidepanel.quickLabelState', { + thesaurus: request.data.quickLabelThesaurus, + autoSave: false, + }), + ]; + if (calculateTableColumns) { + const tableViewColumns = getTableColumns(documents, templates, documentsRequest.data.types); + dispatchedActions.push(dispatch => + wrapDispatch(dispatch, 'library')(setTableViewColumns(tableViewColumns)) + ); } - ); + return dispatchedActions; + }); } From 8cf7de17968be182aefdca98c16a03fe54f92621 Mon Sep 17 00:00:00 2001 From: Laszlo Kecskes Date: Tue, 21 Sep 2021 15:19:52 +0200 Subject: [PATCH 15/52] removing unneccessary async --- app/api/files/routes.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/api/files/routes.ts b/app/api/files/routes.ts index 063d3bef0c..ae234e18f1 100644 --- a/app/api/files/routes.ts +++ b/app/api/files/routes.ts @@ -201,7 +201,7 @@ export default (app: Application) => { }, }), - async (req, res) => { + (req, res) => { const loader = new CSVLoader(); let loaded = 0; @@ -216,7 +216,7 @@ export default (app: Application) => { req.emitToSessionSocket('IMPORT_CSV_START'); - await loader + loader .load(req.file.path, req.body.template, { language: req.language, user: req.user }) .then(() => { req.emitToSessionSocket('IMPORT_CSV_END'); From cf637967e468c6cc39139c1165a69934c815b25e Mon Sep 17 00:00:00 2001 From: Santiago Date: Tue, 21 Sep 2021 10:29:54 -0300 Subject: [PATCH 16/52] fixed unit test --- app/react/Library/helpers/specs/resquestState.spec.js | 1 + 1 file changed, 1 insertion(+) diff --git a/app/react/Library/helpers/specs/resquestState.spec.js b/app/react/Library/helpers/specs/resquestState.spec.js index f64df8e357..c80ba11891 100644 --- a/app/react/Library/helpers/specs/resquestState.spec.js +++ b/app/react/Library/helpers/specs/resquestState.spec.js @@ -14,6 +14,7 @@ describe('static requestState()', () => { properties: [ { name: 'p', filter: true, type: 'text', prioritySorting: true }, { name: 'country', filter: false, type: 'select', content: 'countries' }, + { name: 'location', filter: false, type: 'geolocation' }, ], }, { name: 'Ruling', _id: 'abc2', properties: [] }, From d1a9cb16d52be1480ab8b0b4411498b80b767f6f Mon Sep 17 00:00:00 2001 From: Santiago Date: Tue, 21 Sep 2021 10:55:33 -0300 Subject: [PATCH 17/52] small change in implementation --- app/react/Library/helpers/requestState.js | 84 ++++++++++++----------- 1 file changed, 43 insertions(+), 41 deletions(-) diff --git a/app/react/Library/helpers/requestState.js b/app/react/Library/helpers/requestState.js index b7942d37ad..c0b813e9e4 100644 --- a/app/react/Library/helpers/requestState.js +++ b/app/react/Library/helpers/requestState.js @@ -58,51 +58,53 @@ export default function requestState(request, globalResources, calculateTableCol const templatesWithGeolocation = globalResources.templates.find(template => template.get('properties').find(property => property.get('type') === 'geolocation') ); - const markersRequest = request.set({ ...docsQuery, geolocation: true }); + const markersRequest = request.set({ + ...docsQuery, + geolocation: templatesWithGeolocation && true, + }); - return Promise.all([ - api.search(documentsRequest), - templatesWithGeolocation && api.search(markersRequest), - ]).then(([documents, markers]) => { - const templates = globalResources.templates.toJS(); - const filterState = libraryHelpers.URLQueryToState( - documentsRequest.data, - templates, - globalResources.thesauris.toJS(), - globalResources.relationTypes.toJS(), - request.data.quickLabelThesaurus - ? getThesaurusPropertyNames(request.data.quickLabelThesaurus, templates) - : [] - ); + return Promise.all([api.search(documentsRequest), api.search(markersRequest)]).then( + ([documents, markers]) => { + const templates = globalResources.templates.toJS(); + const filterState = libraryHelpers.URLQueryToState( + documentsRequest.data, + templates, + globalResources.thesauris.toJS(), + globalResources.relationTypes.toJS(), + request.data.quickLabelThesaurus + ? getThesaurusPropertyNames(request.data.quickLabelThesaurus, templates) + : [] + ); - const state = { - library: { - filters: { - documentTypes: documentsRequest.data.types || [], - properties: filterState.properties, + const state = { + library: { + filters: { + documentTypes: documentsRequest.data.types || [], + properties: filterState.properties, + }, + aggregations: documents.aggregations, + search: filterState.search, + documents, + markers, }, - aggregations: documents.aggregations, - search: filterState.search, - documents, - markers, - }, - }; + }; - const addinsteadOfSet = Boolean(docsQuery.from); + const addinsteadOfSet = Boolean(docsQuery.from); - const dispatchedActions = [ - setReduxState(state, 'library', addinsteadOfSet), - actions.set('library.sidepanel.quickLabelState', { - thesaurus: request.data.quickLabelThesaurus, - autoSave: false, - }), - ]; - if (calculateTableColumns) { - const tableViewColumns = getTableColumns(documents, templates, documentsRequest.data.types); - dispatchedActions.push(dispatch => - wrapDispatch(dispatch, 'library')(setTableViewColumns(tableViewColumns)) - ); + const dispatchedActions = [ + setReduxState(state, 'library', addinsteadOfSet), + actions.set('library.sidepanel.quickLabelState', { + thesaurus: request.data.quickLabelThesaurus, + autoSave: false, + }), + ]; + if (calculateTableColumns) { + const tableViewColumns = getTableColumns(documents, templates, documentsRequest.data.types); + dispatchedActions.push(dispatch => + wrapDispatch(dispatch, 'library')(setTableViewColumns(tableViewColumns)) + ); + } + return dispatchedActions; } - return dispatchedActions; - }); + ); } From 9a85e2d78d852f4952827b912b5d3062d14f0222 Mon Sep 17 00:00:00 2001 From: Santiago Date: Tue, 21 Sep 2021 16:08:32 -0300 Subject: [PATCH 18/52] avoid getting markers when no template has geolocation --- app/react/Library/components/LibraryModeToggleButtons.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/app/react/Library/components/LibraryModeToggleButtons.js b/app/react/Library/components/LibraryModeToggleButtons.js index a505f147d6..1b2c1277a1 100644 --- a/app/react/Library/components/LibraryModeToggleButtons.js +++ b/app/react/Library/components/LibraryModeToggleButtons.js @@ -143,7 +143,9 @@ export function mapStateToProps(state, props) { templates.find(_t => _t.get('properties').find(p => p.get('type') === 'geolocation')) ); - const numberOfMarkers = numberOfMarkersSelector({ state, storeKey: props.storeKey }); + const numberOfMarkers = showGeolocation + ? numberOfMarkersSelector({ state, storeKey: props.storeKey }) + : 0; return { searchUrl: encodedSearch(state[props.storeKey]), From 1ab2637fec36f9d16e5a31f20392320649ca95ce Mon Sep 17 00:00:00 2001 From: Laszlo Kecskes Date: Wed, 22 Sep 2021 09:20:55 +0200 Subject: [PATCH 19/52] removed Readable option from importFile --- app/api/csv/importFile.ts | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/app/api/csv/importFile.ts b/app/api/csv/importFile.ts index 52442f48e3..2ee03a7ffd 100644 --- a/app/api/csv/importFile.ts +++ b/app/api/csv/importFile.ts @@ -1,11 +1,9 @@ import fs from 'fs'; import path from 'path'; -import { Readable } from 'stream'; import { generateFileName, fileFromReadStream, uploadsPath } from 'api/files/filesystem'; import { createError } from 'api/utils'; import zipFile from 'api/utils/zipFile'; -import { ReadableString } from './specs/helpers'; const extractFromZip = async (zipPath: string, fileName: string) => { const readStream = await zipFile(zipPath).findReadStream(entry => entry === fileName); @@ -18,19 +16,13 @@ const extractFromZip = async (zipPath: string, fileName: string) => { }; export class ImportFile { - filePath: string | Readable; + filePath: string; - constructor(filePath: string | Readable) { + constructor(filePath: string) { this.filePath = filePath; } async readStream(fileName = 'import.csv') { - if (this.filePath instanceof ReadableString) { - return this.filePath.freshCopy(); - } - if (this.filePath instanceof Readable) { - return this.filePath; - } if (path.extname(this.filePath) === '.zip') { return extractFromZip(this.filePath, fileName); } @@ -50,6 +42,6 @@ export class ImportFile { } } -const importFile = (filePath: string | Readable) => new ImportFile(filePath); +const importFile = (filePath: string) => new ImportFile(filePath); export default importFile; From a6cef0f28152b6b3b7e12aba12ca10a7398df65e Mon Sep 17 00:00:00 2001 From: Laszlo Kecskes Date: Wed, 22 Sep 2021 10:12:25 +0200 Subject: [PATCH 20/52] corrected failing unit tests --- app/api/csv/specs/csvLoader.spec.js | 46 +++++++++++++++------ app/api/csv/specs/csvLoaderThesauri.spec.js | 6 ++- app/api/csv/specs/helpers.js | 5 ++- app/api/csv/specs/importEntity.spec.js | 6 ++- 4 files changed, 45 insertions(+), 18 deletions(-) diff --git a/app/api/csv/specs/csvLoader.spec.js b/app/api/csv/specs/csvLoader.spec.js index 3d3624eb48..22a5ce6218 100644 --- a/app/api/csv/specs/csvLoader.spec.js +++ b/app/api/csv/specs/csvLoader.spec.js @@ -1,14 +1,15 @@ /* eslint-disable max-lines */ +import path from 'path'; + import db from 'api/utils/testing_db'; import entities from 'api/entities'; -import path from 'path'; import translations from 'api/i18n'; import { search } from 'api/search'; import { CSVLoader } from 'api/csv'; import { templateWithGeneratedTitle } from 'api/csv/specs/csvLoaderFixtures'; import fixtures, { template1Id } from './csvLoaderFixtures'; -import { stream } from './helpers'; +import { mockCsvFileReadStream } from './helpers'; import typeParsers from '../typeParsers'; describe('csvLoader', () => { @@ -33,6 +34,7 @@ describe('csvLoader', () => { describe('load translations', () => { let csv; + let readStreamMock; beforeEach(async () => { await db.clearAllAndLoad(fixtures); @@ -44,8 +46,13 @@ describe('csvLoader', () => { original 3, value 3, valor 3, valeur 3, 3 ,`; }); + afterEach(() => { + readStreamMock.mockRestore(); + }); + it('should set all translations from csv', async () => { - await loader.loadTranslations(stream(csv), 'System'); + readStreamMock = mockCsvFileReadStream(csv); + await loader.loadTranslations('mockedFileFromString', 'System'); const [english, spanish, french] = await translations.get(); expect(english.contexts[0].values).toEqual({ 'original 1': 'value 1', @@ -65,8 +72,9 @@ describe('csvLoader', () => { }); it('should not update a language that exists in the system but not in csv', async () => { + readStreamMock = mockCsvFileReadStream(csv); await translations.addLanguage('aa'); - await loader.loadTranslations(stream(csv), 'System'); + await loader.loadTranslations('mockedFileFromString', 'System'); const [afar] = await translations.get({ locale: 'aa' }); expect(afar.contexts[0].values).toEqual({ 'original 1': 'original 1', @@ -78,8 +86,8 @@ describe('csvLoader', () => { it('should not remove translations that are not in the csv', async () => { const localCsv = `Key, English, original 1, value 1`; - - await loader.loadTranslations(stream(localCsv), 'System'); + readStreamMock = mockCsvFileReadStream(localCsv); + await loader.loadTranslations('mockedFileFromString', 'System'); const [english] = await translations.get(); expect(english.contexts[0].values).toEqual({ @@ -91,8 +99,8 @@ describe('csvLoader', () => { it('should not import empty language translations', async () => { const localCsv = `Key, English, Spanish original 1,, sp value 1`; - - await loader.loadTranslations(stream(localCsv), 'System'); + readStreamMock = mockCsvFileReadStream(localCsv); + await loader.loadTranslations('mockedFileFromString', 'System'); const [english, spanish] = await translations.get(); expect(english.contexts[0].values).toEqual({ @@ -295,28 +303,34 @@ describe('csvLoader', () => { const csv = `id , title , ${entity.sharedId}, new title, , title2 ,`; - + const readStreamMock = mockCsvFileReadStream(csv); const testingLoader = new CSVLoader(); - await testingLoader.load(stream(csv), template1Id, { language: 'en' }); + await testingLoader.load('mockedFileFromString', template1Id, { language: 'en' }); const [expected] = await entities.get({ sharedId: entity.sharedId, language: 'en', }); expect(expected.title).toBe('new title'); + readStreamMock.mockRestore(); }); }); describe('when the title is not provided', () => { + let readStreamMock; + afterEach(() => { + readStreamMock.mockRestore(); + }); describe('title not marked with generated Id option', () => { it('should throw a validation error', async () => { const csv = `title , numeric label , 10 title2, 10`; + readStreamMock = mockCsvFileReadStream(csv); const testingLoader = new CSVLoader(); try { - await testingLoader.load(stream(csv), template1Id, { language: 'en' }); + await testingLoader.load('mockedFileFromString', template1Id, { language: 'en' }); } catch (e) { expect(e.message).toEqual('validation failed'); expect(e.errors[0].dataPath).toEqual('.title'); @@ -328,9 +342,12 @@ describe('csvLoader', () => { const csv = `title , numeric label , 10 title2, 10`; + readStreamMock = mockCsvFileReadStream(csv); const testingLoader = new CSVLoader(); - await testingLoader.load(stream(csv), templateWithGeneratedTitle, { language: 'en' }); + await testingLoader.load('mockedFileFromString', templateWithGeneratedTitle, { + language: 'en', + }); const result = await entities.get({ 'metadata.numeric_label.value': 10, language: 'en', @@ -342,8 +359,11 @@ describe('csvLoader', () => { const csv = `numeric label 20 22`; + readStreamMock = mockCsvFileReadStream(csv); const testingLoader = new CSVLoader(); - await testingLoader.load(stream(csv), templateWithGeneratedTitle, { language: 'en' }); + await testingLoader.load('mockedFileFromString', templateWithGeneratedTitle, { + language: 'en', + }); const result = await entities.get({ 'metadata.numeric_label.value': { $in: [20, 22] }, language: 'en', diff --git a/app/api/csv/specs/csvLoaderThesauri.spec.js b/app/api/csv/specs/csvLoaderThesauri.spec.js index 4442ed7e08..b7ceaed8b0 100644 --- a/app/api/csv/specs/csvLoaderThesauri.spec.js +++ b/app/api/csv/specs/csvLoaderThesauri.spec.js @@ -5,7 +5,7 @@ import settings from 'api/settings'; import { CSVLoader } from '../csvLoader'; import fixtures from './fixtures'; -import { stream } from './helpers'; +import { mockCsvFileReadStream } from './helpers'; describe('csvLoader thesauri', () => { const loader = new CSVLoader(); @@ -37,7 +37,9 @@ describe('csvLoader thesauri', () => { value 3, valor 3, valeur 3, 3 ,`; thesauriId = _id; - result = await loader.loadThesauri(stream(csv), _id, { language: 'en' }); + const mockedFile = mockCsvFileReadStream(csv); + result = await loader.loadThesauri('mockedFileFromString', _id, { language: 'en' }); + mockedFile.mockRestore(); }); const getTranslation = async lang => diff --git a/app/api/csv/specs/helpers.js b/app/api/csv/specs/helpers.js index 1b2ffbd227..b4ed9ac2c6 100644 --- a/app/api/csv/specs/helpers.js +++ b/app/api/csv/specs/helpers.js @@ -36,4 +36,7 @@ class ReadableString extends Readable { const stream = string => new ReadableString(string); -export { stream, createTestingZip, ReadableString }; +const mockCsvFileReadStream = str => + jest.spyOn(fs, 'createReadStream').mockImplementation(() => stream(str)); + +export { stream, createTestingZip, mockCsvFileReadStream }; diff --git a/app/api/csv/specs/importEntity.spec.js b/app/api/csv/specs/importEntity.spec.js index b40a93b31b..bf7f0200be 100644 --- a/app/api/csv/specs/importEntity.spec.js +++ b/app/api/csv/specs/importEntity.spec.js @@ -7,7 +7,7 @@ import db from 'api/utils/testing_db'; import { arrangeThesauri } from '../importEntity'; import importFile from '../importFile'; -import { stream } from './helpers'; +import { mockCsvFileReadStream } from './helpers'; const fixtureFactory = getFixturesFactory(); @@ -79,7 +79,9 @@ describe('arrangeThesauri', () => { const csv = `title,unrelated_text first,first second,second`; - await arrangeThesauri(importFile(stream(csv)), noselTemplate); + const readStreamMock = mockCsvFileReadStream(csv); + await arrangeThesauri(importFile('mockedFile'), noselTemplate); + readStreamMock.mockRestore(); }); it('should create values in thesauri', async () => { From 1a64e59382b33efac6664eeadfaffea16743c10e Mon Sep 17 00:00:00 2001 From: Kevin Date: Wed, 22 Sep 2021 11:19:22 +0300 Subject: [PATCH 21/52] Updated migration delta --- .../index.js | 2 +- .../specs/53-delete-orphaned-connections.spec.js} | 2 +- .../specs/fixtures.js | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename app/api/migrations/migrations/{51-delete-orphaned-connections => 53-delete-orphaned-connections}/index.js (98%) rename app/api/migrations/migrations/{51-delete-orphaned-connections/specs/51-delete-orphaned-connections.spec.js => 53-delete-orphaned-connections/specs/53-delete-orphaned-connections.spec.js} (97%) rename app/api/migrations/migrations/{51-delete-orphaned-connections => 53-delete-orphaned-connections}/specs/fixtures.js (100%) diff --git a/app/api/migrations/migrations/51-delete-orphaned-connections/index.js b/app/api/migrations/migrations/53-delete-orphaned-connections/index.js similarity index 98% rename from app/api/migrations/migrations/51-delete-orphaned-connections/index.js rename to app/api/migrations/migrations/53-delete-orphaned-connections/index.js index 0e2dfb4689..f84113d35a 100644 --- a/app/api/migrations/migrations/51-delete-orphaned-connections/index.js +++ b/app/api/migrations/migrations/53-delete-orphaned-connections/index.js @@ -1,6 +1,6 @@ /* eslint-disable no-await-in-loop */ export default { - delta: 51, + delta: 53, name: 'delete-orphaned-connections', diff --git a/app/api/migrations/migrations/51-delete-orphaned-connections/specs/51-delete-orphaned-connections.spec.js b/app/api/migrations/migrations/53-delete-orphaned-connections/specs/53-delete-orphaned-connections.spec.js similarity index 97% rename from app/api/migrations/migrations/51-delete-orphaned-connections/specs/51-delete-orphaned-connections.spec.js rename to app/api/migrations/migrations/53-delete-orphaned-connections/specs/53-delete-orphaned-connections.spec.js index 2ce6db732d..3c90a297e3 100644 --- a/app/api/migrations/migrations/51-delete-orphaned-connections/specs/51-delete-orphaned-connections.spec.js +++ b/app/api/migrations/migrations/53-delete-orphaned-connections/specs/53-delete-orphaned-connections.spec.js @@ -13,7 +13,7 @@ describe('migration delete-orphaned-connections', () => { }); it('should have a delta number', () => { - expect(migration.delta).toBe(51); + expect(migration.delta).toBe(53); }); it('should delete all connections', async () => { diff --git a/app/api/migrations/migrations/51-delete-orphaned-connections/specs/fixtures.js b/app/api/migrations/migrations/53-delete-orphaned-connections/specs/fixtures.js similarity index 100% rename from app/api/migrations/migrations/51-delete-orphaned-connections/specs/fixtures.js rename to app/api/migrations/migrations/53-delete-orphaned-connections/specs/fixtures.js From 8ae1ef17a14a63e645bc3eec8be265f47fedf551 Mon Sep 17 00:00:00 2001 From: Laszlo Kecskes Date: Wed, 22 Sep 2021 11:59:32 +0200 Subject: [PATCH 22/52] added smoke test --- app/api/csv/specs/importEntity.spec.js | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/app/api/csv/specs/importEntity.spec.js b/app/api/csv/specs/importEntity.spec.js index bf7f0200be..d6b09add54 100644 --- a/app/api/csv/specs/importEntity.spec.js +++ b/app/api/csv/specs/importEntity.spec.js @@ -84,6 +84,16 @@ second,second`; readStreamMock.mockRestore(); }); + it('should not fail if the select or multiselect fields are missing from the csv', async () => { + const noselTemplate = templates.getById(fixtureFactory.id('template')); + const csv = `title,unrelated_property +first,first +second,second`; + const readStreamMock = mockCsvFileReadStream(csv); + await arrangeThesauri(importFile('mockedFile'), noselTemplate); + readStreamMock.mockRestore(); + }); + it('should create values in thesauri', async () => { expect(selectLabels).toEqual(['A', 'B', 'C', 'd']); expect(multiselectLabels).toEqual(['A', 'B', 'c', 'D', 'E', 'g']); From 9d60bfc0826fa1b312355408132ebf4c54dfca91 Mon Sep 17 00:00:00 2001 From: Kevin Date: Wed, 22 Sep 2021 13:27:04 +0300 Subject: [PATCH 23/52] Added an e2e test: Incomplete --- e2e/suites/convert-entity-template.test.ts | 95 +++++++++++++++++++++ e2e/suites/test_files/batman.jpg | Bin 0 -> 27761 bytes 2 files changed, 95 insertions(+) create mode 100644 e2e/suites/convert-entity-template.test.ts create mode 100644 e2e/suites/test_files/batman.jpg diff --git a/e2e/suites/convert-entity-template.test.ts b/e2e/suites/convert-entity-template.test.ts new file mode 100644 index 0000000000..757f78460a --- /dev/null +++ b/e2e/suites/convert-entity-template.test.ts @@ -0,0 +1,95 @@ +/*global page*/ + +import { ElementHandle } from 'puppeteer'; +import { adminLogin, logout } from '../helpers/login'; +import proxyMock from '../helpers/proxyMock'; +import { host } from '../config'; +import insertFixtures from '../helpers/insertFixtures'; +import disableTransitions from '../helpers/disableTransitions'; + +const setupPreFlights = async (): Promise => { + await insertFixtures(); + await proxyMock(); + await adminLogin(); + await disableTransitions(); +}; + +const createTemplate = async (name: string) => { + await page.goto(`${host}/en/settings/account`); + await expect(page).toClick('span', { text: 'Templates' }); + await expect(page).toClick('span', { text: 'Add template' }); + await expect(page).toFill('input[name="template.data.name"]', name); + if (name === 'With image') { + const imageElement = await page.$$('ul.property-options-list > .list-group-item'); + const button = await imageElement[11].$$('button'); + await button[0].click(); + } + await expect(page).toClick('button[type="submit"]'); + await expect(page).toClick('span', { text: 'Saved successfully.' }); +}; + +const createEntity = async (templateName: string) => { + await page.goto(`${host}`); + await expect(page).toClick('button', { text: 'Create entity' }); + await expect(page).toFill('textarea[name="library.sidepanel.metadata.title"]', templateName); + let options: ElementHandle[] = []; + await page.$$('select.form-control > option').then(selects => { + options = selects; + }); + + // @ts-ignore + options.forEach(async (option: ElementHandle): void => { + const value = await option.evaluate(optionEl => ({ + text: optionEl.textContent, + value: optionEl.getAttribute('value') as string, + })); + if (value.text === templateName) { + await page.select('select.form-control', value.value); + } + }); + await expect(page).toMatchElement('button[form="metadataForm"]', { text: 'Save' }); + await expect(page).toClick('button[form="metadataForm"]', { text: 'Save' }); + await expect(page).toClick('span', { text: 'Entity created' }); +}; + +const uploadPDFToEntity = async (entityName: string) => { + await page.goto(`${host}`); + await expect(page).toClick('span', { text: 'Restricted' }); + await expect(page).toClick('div.item-name > span', { text: entityName }); + await expect(page).toUploadFile('#upload-button-input', `${__dirname}/test_files/valid.pdf`); +}; + +const UploadSupportingFileToEntity = async (entityName: string): Promise => { + await page.goto(`${host}`); + await expect(page).toClick('span', { text: 'Restricted' }); + await expect(page).toClick('div.item-name > span', { text: entityName }); + console.log('clicked entity...'); + await expect(page).toClick('button.upload-button > span', { text: 'Add supporting file' }); + console.log('clicked supporting files button...'); + const [fileChooser] = await Promise.all([ + page.waitForFileChooser(), + page.click('div.attachments-modal__dropzone'), + ]); + await fileChooser.accept([`${__dirname}/test_files/batman.jpg`]); + // await expect(page).toUploadFile('#upload-button-input', `${__dirname}/test_files/valid.pdf`); +}; + +describe('Convert entity template', () => { + beforeAll(async () => { + await setupPreFlights(); + await createTemplate('Without image'); + await createTemplate('With image'); + await createEntity('Without image'); + await uploadPDFToEntity('Without image'); + console.log('Beginning uploading supporting file...'); + await UploadSupportingFileToEntity('Without image'); + }); + + it('Should create new entity', async () => { + expect(true).toEqual(true); + }); + + afterAll(async () => { + await logout(); + }); +}); diff --git a/e2e/suites/test_files/batman.jpg b/e2e/suites/test_files/batman.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d4ad77f4e729e9123a5e96d7ce8978bcb80e0021 GIT binary patch literal 27761 zcmb4qV|Zjiw{^#uxMSP4ZQGb66Hm;EGqF9fZ5tEYwrv|<=DqiR`1w`;=;zcvwW_+F zTKnw1&+5;m&n*Cww78Tw00aaCAo=wHe69jS0FdAikPzUIkPwhiP>|5DNU*RlFtDfy zh;T?4sF;`-sOad}cw_|FI3&2}=!CR{B;*uS)Kpjmbc}SAjAWEll>b0LprD{&p<#ir zus}*|bZpB1@AlaPK!ygn1kncrAqIdVgMcA}eD(ow0U!WSuz$<_-wgs13>*p+1p3R0 z{nh{f8UY|+;GhtYpDO@3u&_p`w_D2J}QT-ev_%5w?SFl%Gjxs$gFK$lYv^YRNP$ z+gVk<=gsiMx_{iAU5|_}edo>O6CFCXw$hSl_r8*FSkzYi1mHOn9lUh?1K#0liWsw) zueDHM(QL@SH>xVirJQ}NRJ8f9IeK$fR$Z;*`c|NpHq~5jTejG4z~@7vM~pOtS~8 z8Pw%Am|fwr8>`oQmhc4ymB}=4)W-DM*tlsN#*^_L*+m;XDmwX`mK-A9-XZZ}4I8`) z&qj@vy^}wGX_la4{vjWDu9n<%U@^MSskv6Ur9ajgmFc=X-)AUvGTYqm9cG8h_DbuC zg;8sW%-k508}64(r)hYIzrEDjzpiN^|E;=cb2J(a?&PKr+FLi9y!&0|`vnS?;OOB+ zqt4(@-$haW>)>@a`so<=6Oh@^CpqEPC~sQZ%*QuwtJ4=6ADNJ+zL0s)@$Agv_N@1` zeD8K@hOYa^zHaR=w|CI(J8o_y;TItBJ~eQXsZ6u1)d5m3GQ*0r|Mi#&e{!F9QKEE) z=-c$sJE)Fko*A;r^&vyemY}qR@#r#ULW<9GcLnpUfJ4n!kpv3Ktz zm0#eEHQj2XFYMgt(Dpt#yr1-OX>fd0=c3bmYyAWW!}0&yQ%tvDrTtaUlR_Z=Rj7fv zhDEq}a=(6^n#x0+fP8TJk`l)^1Ev;d{JTBl4h9012`>-M4Robw6%UE#ZKEt+P$CSaR(xlim(#X%9`Sx8903zLb(`N2px>E9TwTb47mrY$4RYkujCjplP(4sx3 z-O6uGhR)hFz04m;ZQ1hZMM|L@waWZr=B&H{krEi;HUMK8){cQ5HB>FGy zz-1eo&iu7i3j^z0+Sa3XgAU!4Ifu^efbji?z$U+OK1m^chkxk9q`mj3?Fd(* zNlQ`9J;%0e%-#b+d-muVx#K0)qx-nUK(+GP_OOnbn^r?@ALnhQu>MbwTpgohwfe$4 zSf7Bj=?skj+xbh9FLBKrmyZSC)QcJS`oV)f(S`+9_uDiRT_Rq5 z@FRjs5g9pkw+@|}eyGOI@WtYIs~pE~-P7Cwf;65<$akd3D3z6`#)ytG4T~&;gHJ%p zyH!8DeSxOi=^f!@RjAC8rMrXS+1qWl>HHblBsYF5i@RT@%;EI2%U}^r5rJW1WZZZ) zEVzVlZeEe&g}?e`l$vIpdcDbslH(_Ua+=2~EU|J6@V2}0%ST=O6F_yc_Sngdv@<~l z;~ChP%34`yWqxss>M>yL_6y47utThAtlbB12!NO+h=5_(EnvcazHXg*0V3IUj8i?! z#_F)rcP3u$<*wY~StPzO8ST6RK*jN{T<%b7uu21z3*KS3a&AP-d%wLy)qL2W7X(qO ziT`sfR33k1&99sC+GF6c%Wtjrya`>t_Y?Tjdui~mKeye<4s;a>0{FcEfZk*Wn_shB z9nXh9c1c)w-f{Cb7~Ku=z90agknUu-B;%b3Z!7K*H`Cp-jPuqEcNn}UD6r@M>o+|jdhQPvRSYXVm3=Y0SaL}!^vXyZXm1E-;>-i?v`={ z>})&a#2mWx7Ek4pvy%v!H{;-^YI7L!ay)*jD6(70KPXIAmy~>akl^^z^A)0}v ze{2xmy+C0YMz{cm-{0BXGl!hqAYdC!ts6J+Bd z;@YL1vhOts)w+9b`G8&BK?MVyMe+tmdGwOMMMEE|r#EC44lT&KS2{;CVpuVXq`|O9 zW!(3CLGdu&VkdPTqq&Gtyx~ilD&^`hA1LJ>=Dhdk|KLN`Yn^fz(x@=qh zL%}cNsuJw|b_Zf_vT9tx;gn%ru~>Ftnn5^zwlS8Gme9u^K_PV5Aj=s_==sQ^OvnXT zJW{T)-WX%v5NCP1%{VhsbPGea3+OF7be%fn10QG$I(w;t@ri-fLNh-Wh8^*!={t(J zu$NebHR3cegJDdniJBt=gSo|Yb~?$VWeb5@nKZIbUMvmp9(XLq8j5~L@#>*6D7$Ys z5mD;!#VIIO^RcNZL>lCXcC~~SR%A@jP+BR;bg}*Ncf;GxalSuu*=`ATKLOmf@o(O$ z2t|d~pwsWv8B@BqlmOPkB>}RrlWsf_*9gb+? z7}wi1?vbN`GU3^roX*`QEcQzo{ZMDsU1mHo8#_${%m=6S%Q^o0iU9>ErZzIt2w6Lq z4ikN(q7+Y*G!E`P4(XucKqy~#lQ^0{^?)g_v^}xz4|b{i^F}&~M0hXf9aU+R7x<+) zM5&}*n7>D_PsMG4;VxoC2f58XW1fQVc`(}7BgVewg`~}6V0yZ>@Z>PjGQZGpT=dje z^!v1pUXF&franx`D4ZzF$PAN?T09=XsDg<|@>p9gaD$@?nJ_{i3rRHz!DJ(yyQI6s zqq+8y%Q7s+=(&QlnAgN39kKWgcCX5|J^{#&-!oEWUWs3A<>aEyg&Y1_US3k0CQ;T} zN5?m1AP7MT+36RBW)N;vg@7ZY8(D7Jn>qKFnEdH>6pn_?kVxH(?{(H4I0ipz1+xZap6AU5X)tMKInG* zeNd)1GthD&iCbJM0sX;xNip?Gdx`9(F2?;m-+qMq-AJXX|W^c*-wQPW3@p3?B;O(|F^7bVc9H^Cl<$YLcO-CHaqhB z0n0;@+}@`(GH+f1y-MOk$csxkZ5eUBmOG~rHDvxe&t~+bR+cumZ~Mo`0rKkC7q}sW z;e54~xp$hM00KW1CM}-9rmC|_hDGXtdn__Ocm$qDFgE5yv0K|oP`6ocHUgQ^n{w9r}=@0zuz=IUJ%x${G%(On!;ex zis+SIeohX%t*j?JgAeJU*I7$9zvRunx#R790`O_C9ZZwW+c2VS9^^(F`n~wr2Q3n2 z_S#8aY~p-K3W7J9oPh_yyIs0$<&F(X;}4ndGiw?rPJ=G~e_15_NK&GObT0y2a-F)^ zS9>2yHgu(~rIz2Y_Qhmp5whGD&(_CltVM?@sxG*5B7em+)a>y&m~Y;@Z&lhKWlCLl zZ=zmI9O)ki{jgSeu;=2Bl+oRD7JF`E8^UrGCB#lObY4w5p0?oUtQwtV&pk-T+LaWG zCaRR+Sn9-ZITqI)ok1>IK_68FA6d(12P{mZ-PQ`K)ff+RRb>y`}H+H>t`W?_1flR88j)X*I* zV{CZxL1lhv6Ou~oeUg$5R)r{-8w&>I0d6#KeQg7M3ZI!0ni%eUU`>*ewPS2l?5>?` z!F`|u2bOYT<-_pR}Z>6RYy368v#KD+xd!rE^PW7skPv?0r{IQIY2V{5(_oy?`<$pkFB#(KaCTFA3CD_q@P22>H~+upigOybB+ne>IiWiP1G7J?0U)ga@?+V!9ZpI6*A;- zjeq@+$3YYkLAm;&1M4??YBSTI7Rho3tl|yK-vUhL*^$yZ0Z#9DWhHg9Bup%xncZb} zTq*0AQ4BQB<(T8_W71>i#VMUNynNrE^EiZ%9h?iCHp-Ah53Dcr7c`G48+rRZcJ0HK ztFOA(Sgt#p)N;xS=7ZyDG$3SW?6@qFf9fozt}c7^Q~KC3J-2*YwsUi(&%m5uZ*B-o zD;##gKd%)im6Jw))2u3qta|S8xpdRpaj0Xgt7~5OA2IR{fBJ*E(1N>xfqvRFg~7NC zq|v}Nf^Ylt&D0gPBiiPit0ky<*ICcjZ>jBjtO-u;Awg=>d|>O(laze&GV-gox}>(Y zli$DJ@;tBlk?;F@X$nsxgwjoURNRKXA>h#0^lg=-BDWI2`&e6i0&bpm39Xv80`3cvXSkLeJsOLAWp$C1V1 z!NY%bhGMa)2f8m#$C(Y>@s-tyRPz{FG}*U;r8IqyZqkf#uc%XDq?iCIR7D1+>!`5m zaN8%dDl~mS=q(o^*$woJQ6N51g`PGsRKX@4*|qr_y(*G8#CAkz@?oSMJ!C*I2Tp%S zq_ZoVFvuSnpSFTopvG^bH9d-)jqtBrwSD{A5-waOO4qZiz_IVoUIS3Yn3J>VL!iWCe%Ypw_^#)4&uvS1 z@A-IO0Z*iFu#)qr^Av&(FTOotEF(r*IR)UfjGd;ys*KPEQM(gUYH(iF?Y*CwPF7KQ z*{{GyKAkJ$m97ccIURMrFHrD-KbaaeFW~r=Thgb?IpFnV#-rb zZ_VgtF2`=ccrjzcjjP@Rb~LNweM4AH5or|0V~R9;`1_S*iEY~t&vuZ(Au^b_+wx^4n zmDO080a-DPOVwS~OpHDE=oCkqKpM;iApm!#W9biTT5LCA8?@5P@3fheR_=gJ3oxUS zJBW50F0px}y}yDtEpoM(EK@3X^D1fK*Yj<(Vk+U{oKe7VNtMV|f7p&n;Df5d`Jk?C zV)3_cPBakta`Vj+wUosKv5&hkhb3UAdAe*65&<5SOFY6Qna(Vmb; zhQvAEEpC>2f0)$(2NZeg49Ieu=$S`FXSpsKdSG5|H0#-x**KXIJJ*@e4`O)n$C(2M zs#W6z5LQ)WiO_qT^D;4-HonzOFc|1PQGgFK>ObOuWnDk^=vSpIo=<=mb&!e6_nb#NN5^ky^1mE1m+%tJ>ys428IASVg_O9j zf0s0XU(It&SZ4B`t5$wcFC><#n^`B|*F@HKegfPUBSGY*jI~z>oh2Q`Oyp@Nu!C+H zKkQ;Vas5sVjb@*F>9yR-Aa=JBTatxyALjU%wqu(bfgCKb2?zenUkOYmgK1k;hU0EXOtA$HCoG_InMdG_M2W+i3sq3x^ckMEbef}EKr!=kC{I|JZQ#EUL*L5nr2*(0-Dpkf+7Rc!+6nBpd1>h(c z8pM@w1126U9AZ=P41@wXuqkbTaujDma?789dt)f5VKAlaESx-^gJbMz6I^4j-!AlN z#Sm{lK3HI>Nou!>UXb6)H5(0L&Ovt&Qfqnk9}}zh~ww@ zkKcZ`7|N}eyifmP;!uXS_@dHeG50Oc{a#mO$Ad+B)9l7xaf}Po`(6OJv0ur=LhJUplN0EWMKc(OaKb zApahIV>oM#L}zP|h+Xa>hkFmKy+jq|81 z$QpbH2Q2AzxynYDE`|629;A%;q)w4Q9;d&h-b-b}J@_Z`LwWJ)bbr{Ky4DrGVR}f! z2MHjNq~H$`X0FTSi#dH?ZK1(BpK&K_|m1A5U<)H-e;lJUQVU@O}%!@+$KHKIQ`y=&)7>C35#?!=v%(ZcUbI?M`!Y6DmFfsiRBoJ#q# z>fE`kTkiIF6U#EE{Gvt8oK%62zk~|bD3^$^H^T4RR1#ieE+=Cj47TO5x6Q*=hgmq? z<%h<0Lah}Qyt8>D_3GOlAOQKl;+x+MzNjSrRa*oFg@A&F`g#@ppJ(3xO(h{ELwmn3 zJ}JlM)O3>w3L800U$H2v7;j%gpgR7}B^9adaq^Gb`R{roqyXqW3FNQs#%8Z+QNtcq z8N7L^YwX$jlJFj5CrsU}taVi3)$~<)s82wmM+CI;QT>K%qStR%IiJF?Bl$=8gcV&^ zZK84ZCYg&c60gN);>{oGW>K)<`fVVunL%EQpMZIO?GupPA4f1(>y8P?^Yt5_fR>}I zZ=ZmZ=xjga;7SI(&})PUP#@w~^ng&zY$wX|ll~h5$7z0yud$-(8V!rcEnF3$@aM7Z zYyQUy`-qCLXep8b=aaCr>0r)r`Ged{Z;g{3rqf2(LcdW1rQJ8k zN}v7QT*FuqHT!6BE9ov!@TDqbgeW|f6P)X9lpMvPT1@wu73JD3b$(e*5jLp{roM!u$kS+382HMJ&>~g0G0`9C~h;kxuB7rE)r^({J9&IwN18 zi-rfz3IlI*@x$+~7cwthHd9ScGFDQROiTAPZst<@eHr;@g?~!QBN$hcEI^>+tMC;N%T-v+6Q z7(Jq%yE#jjL1NY;5u&N6@SYECqB|yev)0SS`P`O51()yKg~f`7lktv4U6ihJ(&UCJ z`&cWT(;cPb*f_a7M*CDYkQ8+=rLw^Bh8yL4r4|lc=$Ver`3IE^l@&&-q03^5b888c zH`G!`6(h^$5?o~euWc?4{tz)JO-6#K9t3@ifb16zhZU{Ky?|z$wYu&F4WUb~4}kue z>J(Q5oZ+#776~+|9QBJ6cgLq(1uYL!Gm8BkAfMeuSFm; z?I*NRp=-?=Uyw`q~_B|>RNW)jg+7KQBry)4vVUEUe7c43o=Ur z^tNF$H-C^=f7El1o2Qjj`|_Qdc@W^g63PVEfCtO+y^ULwki?Xy#VgN4MHVtn)67bk zH#TU&T5Il4X(;k#WQ)Mzo^uX&YNoF7;4+=d3X09pspf@K99Jld27qY z4GvCbtm!fh#1OkEYe9dNG)CQ#kD53%yWSRQ4G`ei(?~O{@}wd^zz!<2y97|Bo+*_K z*&?iR90EWhLijDtSlH!e&u)|&Lh;}HIvWK7<)|&9wpGQQx1R_W=crht`(0hYT6{R5 zm%VfR6APO$Y&)cRWE(2g@wRp~zsp&@(*z~uAdsCorjvD{YNd`H;9LN|+;LAe{vbS2 zGi>qsqXE=ujYEu%IhWra{0L=(haW^}krDJ9wIfiPk{C(V%*r`)dUB|Yti1whZFWy1 zryiugNKCoUl7@q6rJQYFuuP8<3_7)uOMin-8KP=Q-MQFAt_s0&8W?%pgq|5? zxA-#5zB-CEP3}cd)~2$pBoszQRCi*+&FCBwg+enU>iC!*6}#0ucOJ3;Yq1a_syb+= z(~~D46-04@TIBJ^!$$0o)f`06ZmnFZN*jAW6BkjFm~Pc(DnP_!TJM}o>N>oUm3GQR z9c9y@<6MsduiHu2HQ~B3uUjEXv;kEEJIq4pZ_p#h+)m2=h#|gs;Xvuc|BkS*V?gL}L?&l4`NRw~sbH;Yh^NFKtKVoy(8m zpU~-btvT_VD$;0Rm3uf(5hRmyBp!a{x3>yB4pJ<&-8nol&hN@0cCpyZ{+i^wIU*su z$1t(WiZu4mIjz;dGfN-B=g|p!6Yw5?y#$!QyK+6tQRKg!RNyWpLlF=Cmd65#>08s# zCh8wLBN@nzQd$eN0m~l);xpQ1@kxd-!7*=yEYtoGIYlE^gWTRv8#lu<9bD z5Vg9%hW&ZB79wxaUrB_N#%rQsyXY-hvTN;;;HIYm)|?hU zkgZlh&!V_IP$bOLqb$$E+&%NQhm-AL zG0}g`i^`76&_0T5Zp4CQIZ|(&N!?Ty^ zl7qlC{ve+Z8#>0~Ef(S_ee6=g!!{oiC{Gnb}gV`hdn%Ae<$Y5*(zLfwTk# zyrXteF^QOs^o-D1&OVXf9#68E*3t7X2;1A)%|5N@siY>Etbs@AKHffLvFS6+gyqBC z+P3cW)^ZBTFb}TQ+gIF705++kt+f2HDJkFaTU2^f?H?S6LMBmMRZ*iov(3=0XFAF~ zgntpvn}HJLejlVJmCuujcUA39?B=l_;vKbg;DJ{#t2&7Dd=*L`KfcNeX%X^h{jTM5?IK5GMYd9;f&`FFSdmpEJ7+aoi-HatUdre!`8I-B z)VQYM0c8@&^_WUQ8=Ca%+u{GKa{mT~c$&%99s%kCBN=++Q zX=Zzt&K61`7(zbCJ`xJbAQzjg(> z^<9ZnBs_8qc>K{1mi-Bk++Y*^N^h=U7nMo$_78F*ZOt#so*SLEof;c+Q8B!Xii<1= zb}a49?_%m!4}iKQihAc=B<<9U&VBsDS3{v78&l0E)O;vxlLPu!&CfrSy|J>L)ex=H zeTEiEen*18A`mIvv#y+#k`na(GP01{$Wa<#$nw+3170mD{E?*=#OP(0-&G}WYkn-1 zaatm!rq4smx%2fq}6*9{-2nCoq96$$Qympe*|%*^ctmgS}T?Iip;67l9nr_}$VX5!{n z^^bXzxqXK+iE}#6^6TE9f5CX0cUW=WS}Ng|8ah^gtiK1m)1UAHq?pQCRgPoBER&!4 zaE3S^z&oHs>-Ismvd&Dvu}k*0A8^Q%GWVxjtifpi@$~xeB{ol4th_}Z7Ez7jCm?qd zF2r7(adXZ4cOhQnZD}}$ULgTNsi#F9tYJ8 z3ez@u%{$VGTJWepWFh0p%}tfipS?cChTL;>QZHviUpo8~5N%fXsA^w|N%$}N3?zdbY_Js6rF{CAN;R5HLjLAqy=ql&vm_iPM!(YKUr(dV8_%7v_5hf?<&`;+R_IRYXs+^Hkp zv|X(H?>EDKM?}j=sH;tSXen->U+u;IF=AnG@+TmR2j+YLS82)SnewJT&cRng(anUL z)tEf$&UoM$KioY}xpoy{=R3cdC4rEsd?RxrCS|N+B-tQgeoQ38P4H`PP*y+MJ^`wb`S@3yI#J%dN1WDz_g<+7ROK$D>;u~CtpZ))*>iIxb#HBCc4=8Sqwn%KBe z%OY51=T2&mD63Yn&_x6+!H$+$8G+7`xVhB$vF-J z6AnVwd*NLC26ZV7@og`d&#Raljuqk_gUHR(XE#3bjz^h|p1PSAp5gZhzVGs>GB!-9 zV_KDPmC(#B{#=3Gny<_>_ZCpDXEy z0SU-#^uLfq#cho!7|ecTfjL7ZUFD+$E2#2yF5iUI;lqBIzzt-S3hnjBgyu>AssG{X zgDSA%Y;Xqo3qlSL)!v<$%Ar+p(7wba7W5LOf7ZI#<&xDS+EO~;FVEg%-o?4bIvT}W z--QO~vZ+I=X`NIL{`c3FG+}9KIIFPt5Q&2TAuNJ>K}FdRgnd$L6H>=i=)6yWV*x;~ z3bACoSpE-Jr@VE`)B&K`jxQ=uGjbr5P)C}bTE$YAu+9o5FlSfL;)j3IwYFTd^nyu( z9kaBq6%@mBD5KydFlJCqQk?s?Mn!$Z zX@J%qsB7iwKtbPCsDDsPf2X($PGztIQ+d0c@|Ud*rn;4N10l3+z*n<%&PK#lbaX{qI2;vlm=} zeP{r>ctS85sGUv*(ss`G+j!YC)C$HVUnOke*qkmp>O%EXFavkk7!xnb>fhJY*e};c zDo3e@{g!9*{W%O^YzfOAbF^IS7H!cx-j^y8mRS*Z#b#Est4rU(*hTKmZ2s=>=oVil zxQ3-;+NWi_AL9kU_e+d}v_MxnG{C%bne;21$UyVi@TmA|Rz_Zf z(8~ij-qqGyI3!+$&257z`aG4AdTO1VAV_crx%Zv5)vwhnUAdI~HUi3IhNfqt8C-jO zdYFv`y>{0}D#4}58_AUNhi^&0BhkDjrb?x>D0R9ze*#M5Kw?W5pK^%i9J7@fD)R7%oQ}y{MN(G&pCHaf|oHN>HlPEYQPsViFf>fz$^r&3dnRvW5^6Boum9xQN{Cd;$Pyj+{$)!5yYR09=F89*Z&pPW=s<6UxQU z*-UB-U?LqI_>Ho-=omQH;IV?u)kH*8+f*&gd^E2|8Oc`l=%zNZiUc4xDprae<8mO} z+(qO%FdDnsa?9jPkHwmzR>J?@q1V?|^^kFJZ|*pffD>BAA+DRM3@`u8GRKec#G>W2 z&C*3Z*MEzg177GPoyj?5@G22uo?JqhJDQF1s#*!h;vX9;p7_ijotjaqvNF6Re3r{H&iMBUfXoEQ^g1pto^5PzcT_pLUGfVqeJtS(li#K%M~?9fn~(8Znw+ zn_@D&kdJ$x!)z;OGSOVAdOfMVD&{4!A;3v4Vj6)+bvGM&DNUzUr6{A3;TiQYk&0XK z*QI#WS6UZ*H$D2&k}h9tez9KMg-+}P;g(oF$cFuy5RClbPC2rPkMqx*T|y>xj31W(OHnsu#xWeiUZrv8os(2$U}vIPbG2_R zEVK2LZdH4KdqT`qbyw%}EA|Y+n09~bk?M6NOXy6@Onw9nTnCJrEtUZ5_vTJ*rw}ng z+z*8c*>D5>SvAH^db1R=*>*=VC z2*S$5$9gJo$^&26ELCX;hCy6^{$@toc%D;q8xI#P0+ ztH1&*6vBk1fLm`cqVM3}$IqR=sjGkl3Xhs*Oj5I_Pt6%(LP}L1t8fFVzGVt@t!{J4 z;FHYbvgy`e2PDF9pmgyaPrs0ml<*AufCdy^_JZ(SYlu|cJ}M0-e5Yq6#j~K)J#q}I zlDqN`%vXbc&@GQ!K2bREaZwc7r%C*4P%tpftq2g*WK652FUbY*9OX8l%KHO0q79nm zt1_8Ol|i0Y=zzPE06v_@WmYit`WRV~krYV63YD$`)XXQV_nc3SXtds*Rd^w-1v*YqP33;K^J=UT^{3=Vs$ zLkMw~=&UeFO_>BTN6*>VhWJd}%(uL666?XcQvuY~Hu7HuMOr)~C=#^Z_A7(!w?*~! z*k^bN7pVL}ob9k67981Sg`XXzY#cIWx>sf2#$NTabz4*ZAgzO(y&I-)@$dgi4v&OE zmn)HWT+9Hc>4e=W+$(#=h@$bx8ckl~0-sBgzgw^5s*-8n*$E)i#G~GoEuf9)x-0HZ zOlO%gjR}w0O5mcz4smH`xCCn>*sfCSe<0^MUOutaYWauUNyIe+#PJj76|ILE#CX4!o7on%|ej`%FdM; zQlxEPKC|TeJtT778+5$uI`JEzV&B|oE13893_-ZhW;oKak*PL>^GB}lxo;W ziuM#Gzit+`AIF15pkv6P4+=FTIBl zMQ!Gp(Q>Rg^o=sXGIBf*<}|WnFf8R@^s&WImc5@!F0*d7=^(6b$CoJfx4{3iS8v;Es=}Iti4Moz6eDEg@ zZ{{=0D;YbRFvmNM11|pZSDnvF2y0RW$ii z4|6MUv^?=qNz)dYM0T%aIlbV}Q9@BSFjTJXoa`9GQkW%6{L>9GajU~9qzI>M8nO|{ zj9+ST|`pc3j043OFpB-VQCC2e497jknucWzT55VUC=@k2r>E&?QvKBZvCIOrb<- zB^}*Bk`kd|0E^u4%+{#LoA6vlDc8$(D?gUxi5s=aP539@n&T+}UE3v3S@$r#`#eHOV=@$#w;Jg|1tv>9A?j=kC>YUX zvU2$o37XUqd$h0fH4jCA?z+9-Iy*z zSX=fb+$EPcsQfVFqhVKF)pO=)0A6?tHBJo{R9l;Zph3>r+@GBF8}&Z%+5pk%@Zbb@ z*NYR}_2N!&cXua92=49#%_HA?KdDzWwKY3+rf2q_o~iEBXTKKN;=&w;6~uzFPwcSv z_Fj^%C4ek^QbavgPmkIPo2zp{2P&vXg0tp>!Wcrq;H*jep-rbq5eZ3!+{9yz+aW-- z$H*K`lR=!fN*&3^#?gZk48og;oC-Hl3+JtoQ~3|_BFe`lT3kI*wE{tuQHP*1ESb~o z#{7rw+ZaA(`oVXEIaKjyqEpk7CHKn<&DKfeMK3nCeTCr9PU$P9tCgs9;n1(aEq%D@ zlIO?3AVwyGJ6kMN*37N>w^ps?bbpbEy=0l-cX6pSO-H4ZTaK#=J#%I=mvZ`&5nl;w zY-?IRZ}70(PF_i#bOT&#yZ3v3jIE`xobY0apUEJ~+yVGE_VmrmMzPOPZV4UfD&d6= z#3xb&@p;p$x0onenA&Bk15{K+6_jeoVc{HM3Zb#Mo&d07Z6{z>2j z{+>*iNVEmypn~ZOUVpHx@aJpp-H$d7!BvUf6T(x8--a;`gwxG$wqrv3n^9aVk z4x!AEEH}{>jei_nZwzCKSt`7YrA8BkLGZdYyXDxut!7ZqNQMNaYmuTiXG?3*9~CF~ zT@pVYCJ?FOzZ4&}wJyJ1%DryW?qiWG+F9}389pzxkPOhoY}1=R@Mi>sa@q>9<#A^_ zxzTswbD3__YjA2m*(jh;mvnxnh({-vEv$i&n3*nb>VQT-Or=w&Q7~B95O7e(Ju&Tq zAft?jeT7)>rKApVDdyN`#SpRCyD+4&XvN#|Ra=ir04~+3HK4vxcXEj}n$>)R=Fi=6`;T4+;2(_=?RG}!c804@<>UE3?V17if4|<&04D!GKwZ@U zOXvS!ri>g-2-?p7%fd8SopPOWj0oX>&1zK4WBNWCx}^I5rcBI_igB%|0eLoG|1$!- z5yLwe<}eA)`M-eXbi-H^lH~)c6TSb~oM8Kg^IeS2r9Yg-@;^2k9NXJSSs~HBMr-}w zMTy_G%^oipm*&jdHvcmc*NvaoO*zfdE~xj<=6L5z+N}QgL|!}ej>A7dS3vb&K(P17 z#hZio%Rj`3|I1&%3d0t6&+^i{x$no&%VnEixPQx2*Awa8KSbC2bHS0%W#0Zo>zmvM zK&0ccoANK9`}x%MF5u7krO?f(|4VNF%Vm4Zrk_Ui&F##`{q-_BZ?pUPFMzz}fswaI zX#4#AW0|Cv%WjQR_l0nSzkmsmxy#Nw!Hnl>!)fv+_aB|t_peVXoiigREHyQim%b{~ z+eUrgPg{Hoo2t62VL#8TH^2(X1hcx#;O|bzJVy&%&%kR;2-4*V-N+01+1vixsg}@7 zaKP;h((MG-qtU;wy=m#~LjBjdK1u%v_@9R4^soOBEv85RU;zRkVPOAT0R{*_CubK& z```m6)RD0T<@e29iEUs1Cs_QC0FU;*C4-lz~ivuEHUkJXt%id)_-@1VasoVeUe_oj+UJl zKE*Q|w2i#vXHSFQ>0h5pF<;E2zYCs%cENvI%iD89>AFwtBrBy1z3{x9yoEcg)iQ$# zLMeeoPH~F!JTp2o4p*x$zeshNd`_PTS-;7q?l%e!>Dh~I2QQc-@i9$`6gc|~-oL3% zdNI~s-I9s6i_1Hu9Zy_Nll(5BJw&6P(wx)SiZnxGq1`oJkZ|`DLA~}b^Mm)YmLuoz>n4j++#!v@@-jA5J7*t%6{252l zV+#!y+-zl@sQ$AN`{R!ouw=wKcf8!@5_We-067|+y#+djfy}lCmy%@V+rESOUq*%9&dv(4JC9H~uLH{R{r!tl3yH_#rjG495} z1Xf|^Sc3p$U#!ciyL9~@IaRHmbe*IS?TTUpR-jyiw(;`=5=7x%5p$)Ht+n=cJY@15 zPZr(+O>st~wn?TR8Y)Vr4SxZ(LzsC9K)fll9Z$abdhn$|tJ?POReQ+(t@!~-KcgQn zz-1|OmlDd;qWMkacP@}yjpZzkeIE(TeRkeSVvdK^9R4klV$`5DL54f$zTZr-RXl9z&Ciz{Op_(w z4%z*&=f^C6$YmZPXW$qN8~$#lP7$aIOA6cgB(vkpofw{lT-4$5dR`wmHtpEcVS&xz zbi#Y7r~a^;m2`S&-w?CQz_5wz38A(nvuQ5*lje`cTo|qT7i3c0ZH|%NvSzl*&)%;I z-}O4JJ=ut2)X@t~BFCO5hSS34&I`PsMH??g=~}h*TH@)CN$&4SDKSp>o_DA5%tRzp z(XelE9^-WVbQno8r4%an=+{ZP(FpEZ_rC`)d;Tkrq5J0+fIwJC=>O7%`j1;cBm0*{ zy2`JcV+%^?+ZOweSNO-cNBaw~p6>?U62rvK`f+$&zDo!7nG_CXxOHNElyqxmlg2Ie zeM;pD70F{gUd@p?M`{+?+?wlhBI&G2sD35I3tze0U4!Fh%uRv^(5^q%NN`8B{r0E8 z3B1)+^j<1<-adt*CNK5R)?xC}+WhVRMf!#0{8>IrXktOQkeZ%! zv0o=Y*_Sv#ad^G3LP)sOOjHUu2FrF?zPfu%55nTnu0zZ}Iz5NP60}?~UTbdksX}yp zSEjRv=9EFJ6)IMK5(A%J--)#h%y0#gcrq>j+z$C7u=Se$GSXOOnai&g+d#K?>UR1W zjZWM%v8bctkex;7QK*%97gl? za(>oH>{?FtL3Y+{Mo+hi`{4zHngKIz!@kH;_sN2n8V`vpNh9m_3+<=KLgRW<73OGc zIvM0O+|6q?Vh=v$mt?;g>q5Jqg$?Wn7%c1_UK|&{1tI_Q7&9;y? z_tFGD_tMB=uNZ|#;mDRWqvT22P~Tlm8C55Fp;m|1$b6!?QlO_SdGSKQ!@*wF?Z6V2ernp(~KI>DP?fB6`tWlOPm8L-G&iH<#GIT|fcF`A}$+@ns zQfqcn>Ame2jZ}k5Q30D>vno6VBZl`q9phIX5gd>kgra@t(b-u|V8KQh8-AEf_(gnu$NvAON5|CJI(dyChFUyX;m*I4`u z80D02*^kqCxvJC5PzkLfGcog-&lQ9r`~8;pglrAZO61q64#`h4+G>o*$DC145vHzn zFu<(hq6-T?`UJ{I;p#&}TEOpl7vDvk8g{jYr^8rP{DJf}UgUt?vR{LZJiAZ#knt}d zRx&MQ)Z}K?j0ex$5rt1R9Dr{f^NmMdHL}pVEogX7=ADm5IPZ%YGH(}9L><8$od8HM zMMJ=h1CBwZ%B;hJk7xSolN?Pg`)l9fUBH338Ok?dC*cO}>ze8+@7#2pmf3oJ6?%($ z{wT(>01vM0$`sU*G>RCKS)uy4i#qGb%)8KmDx{hGD+8qr4g`k-uo!BOj1ZMdVVT^< z%M)-Qohe-a$4Ob3Na4re^Q`iPn`IK7-qfv{7`~sSAx0gmJMcoQpm^bi*IM#irf@QA zOA~#131!@O;bAgpg<(SzpK9bC8YJ~7Ck(!or@!2CS$NJj%o4GKV&$LT8Z1v$qY|N64I{uQq9LvInQuHO91Q^pOUBh7D|%#E1V$@ z47Nccr>;(gtY|{`JQy!8Hfsnsv{PD$e;@@a=I8_7=fSa7QKho*-z#aI&;}vdc+&(3@=b2!DzmZSo1{Wwceb5Ra2x~C zv@v>hkg$S|_+eC^3JId^Mf{-bXOsDw3+$5irdm`jh$7*RDa=#FpeZf-t#q*2_y1r~ zvKJzoO^{#fHf0pP<042n?Ze%uRGLT}M2;?;`*uSA zl@6RYaB^c(oy@Tbx6-t+WtVVqt4Wxt&oAq&os3m*dpU>S;jLQCb&-+FNu9d-p>Ev{ zeWm;{SE%7te5VDclskW^hKA=1nuA=Tq(TWab%=U1Y~3|CTvqrGoqNgI@xj2W6rO;| z*|xD#NpOyA05;+g7yC8j(jWM1wL;`+LTlx$4|R<6N2sL$jg#tHXA$&A@j%_>L87~r zO__5I+70P=k56BR30JWBp_C=rzj~BySyRI;>g0@WMCx2L^X}>C6fi@OeIHFZ)2s+x z$BmzHJC=o09fkQio773LxYALPDe!8o-HGd_N?cM1F$yG$iPNo&`~-BMnHW=Ul;44^ zTV=07nDSsy?P$OPN_QrU+9cbHySFTnv4`X?hc1yd&|O$PQvL=YQ7zVmuf`dB zfYCZb;sku1~Ax0V1s2NjmZ_B4DQu;WTkZ}#fOP2fa9`#L%5fXT{J zB9%&qVzs_9Q7QiOiVXGw z`U|Koy@-nDbP1a~^UO7J$U@QMCnc965mQ!D!MP;8yZSby67;K_-g!^d zoRV6p`iy;}^g5eiSV=E6>&-J5>u91$eiD+Y7G3>KN^nkKyxN>OM2f-zXT89KEVj`s z-++9*CgGsaCf7!sl|$=Ljd?oSD>GFM-&%}$&Sytt&sSRRL^xVPj&M2e@&c@1lJXXK zb=Bgw7jhD78<@hyea{VX*7(5`LJf{bXCYB}Y{zD+@D-lvdSh@0{{jm93|K$v>&=uj z;}mywl_+RMA&@~?Ag!ajoFrd+){nI-0xMEbTV3ED%xc5cp5|7p&A?$!!B|+eGtb+k=<>!5N%PEHc1GfBGHifiNtNvE7A(i zP9e2)If?=gQwa<$$a%X}=<;XtMVZNyctR8#VxQ}*G;!1OEwI2-NRNKsd9PxIJeSvt zrTO5b;s>LN$(>+)_IQr~g)CWi(iDPAQag$g!`%0l^@}S_6q$GnWN%H@11jk>z$A4h z;F~5{B0_XoXwji~l)&cpS~Q9v9)|Jlr=Gq21h(voD;`&3C7>r8BcklS2-es9qc7Bh zOQ?{wut9nQn1$%4LCK>v*QAu~Io>cEX>dm~6(K}S-=>2^@k*Xag)plNNt*p~k8a87 zXD!fcWoFLka`3(fm#ts zmJ+j55p3n+WOAZ`PclhWv6@2zC7sm>7Y6{x7DyK8$#uvOp)v9YCay(HCAQNK3As;& z@^ksntF0VLuF6Ekkbj2yE;)60X#5A%^-f`YSY}JJn{ei@~Wg{){ z<`PyLe-zeiRB85^9dM6T5&~yG*d-)QmTH21bNe%ShKvF0wIN^5Q=(h*Rmq*x(_m6_ ze%3rmL`e+%c&cUsc=ur1s*iJWB^#+xfqJ9e$EYdC3^aiewMLi4#I6EiXnOX&L3%?K z#+C4(WH}%juRzwq`(fWF$n+>1CbSd%%20!Q-U9}^sNCb7gQa#u!JTNVYqo<<2BIF#uDXKfD3-t5t=e?p#!tXT1KanMv3H2ks z+(1DkV#t`*80^GDGE*|(NEq*zYy0!7;mm=Kk?|-6ehGhZ{U8K^2s_*65kb<)3h~NT z6tb}zNgpA6*|H!zdEugMS@=Xbrtdi1m80->(C*CM;zcyN%|)=86q_{cO^KdXe!t}? z#Cc&&zX%_m0@97UW44==kQN*mslLJxR?FX zakq&5SnhnT^$*;R(P+*TJ`7um;LDlGtCXso&aSx3lo#`iVoWmGn8@eLyB+zS)%l!T zgY2kLHmDcmmYeJu zGR0xj{w=-adJ=B(Kr{?YjYf5aj~}yiQ^5QHbVp5@(9nR#lj#gEDRBY%uwijvFj3`U zyUcb=1YYfQgC0LQoRQ4TCpPE@-f8}l9DT3M25Dw$=Fy(yDywy%HWj&oAb0Xz5i$px zr*>(bmG3V11_eWC?HC8=!z>KPx!|O;!FVRsIf{;kCR5vAbYW&SroP5!**U z^zY^~5g=CB{MiF^ai6x0@h;-?+!$dnYkvg(xE5!?Ro{xVzi9PR()ohABr9l4tV6P2 zI1uQuRhWMkmm#PI0S6l(@$la7ibk7xKORu4?*GB9ch`jF$w<%a0Ln3>+D|;4lMBZI z%xfnK;?hg>4-Nymn~VhvxN2Ev)^|;_LA!P%LmEo<9-FVCEl4tZE##z5 zQNt=mX0$)?+{T#`gz!^>?usLO)`}Q~ZEBTq%>XCLy392k=PoY8kGst$4^v;{xsBF` zi-*iQM&81^XpkBlwdnwEy+nf|e0jD~+k25^kbU0eaNNzDk;!P!ZxV_2~8 z1p?WB0Z6oBYy_40(>3&fVN~(kqfnP$JASON--F|jb?&Khzr8G?s?p{WYfGa*p;yr; z+;d)3PpPflB|C*1i8Pf6-UFbi@kn$wZKYHqV}j` z#tjPW#p6TF)YOlwmcd|a$x&7==5IH`Sod$-D%c6TkG|>n6=K{~ral>Ov5PuH&L49_ z^?=2vN-ddFp_zeoi#5y2OF58RvptKW@~B&uhS)gn^#}G61f57S#UkYAl&7YAW67T5qyH}Y~>x`ID`m1as*v{m5hjHglWXy=@82G?d2?_HEY$!xYx7nm=S(fg$xVoS)>#p^4ZGf#r;6N%HyDL?a1i#_I>M~Hajt7?)LsDA@sHUu5m9A1=+`D2Em zqKDH9V+O_;XRGp^IPE{&&j;9(D)gakg?nvt@>sc0bV^fLMIA_<&OFabD6By{&y+U~oT zu%zn=-Y?lBS4yv3$k5i_eP64((DDx7@7-c)P$tb7$$@g*2l*-FwuYf|Xv%>yM;+~_ z<6dk!zk!+{m+#@3S%SN|5km1~Z$%-Wn1;r^NGrrBDP02NlM|`Nq-pR)@?Wg%$UbyS z)35*3r5jB+JnMBq@yFY-AR#6%h5Xq58PyciQ0s6KfZ|8a4>t|FW%Ath;Mr*sHACHgkOyiN5l;XCwx}j&`Z4y9y!elC9 z*^-%Ul0Gr91af*@8tEmOWzsZ0^l>N-@*}EbcS#bogzIMosMfOU1Q~%pLu*C9MO5ye zl#vwL^Y6I^*ens!ZC4M9t!KA@>|^Nr?4`)1&=C=SpWnDsUTd7F?IIq+pG$pB!3}5I zXa`lqeAB#q&LC80(l}o)`WE=vWa`%C;5)Wkm7Zg~y>BzZjBA0OXh)y_fO|Igbd$KT zBk`mMXx07#q)k6Zf#~hnh9D;84vOsM_!OV~gRxacRW{+WYQN3$-Fj}1&L5kc30;ff zW5;c1shbcTt0qJvWy2M8P5eKYz~(w&-9LLk5cKt_1Qug9;B@I_mVL zdu@HMX3e{FL4Qyd@92B0{t5OGX*f%k#)q~W^o)~qDB9&n@#q#Y9q=s_)*I>TR(yh> ze-vGMU8BZ*OW*X3m%sS^QDmgc>=fE4eRAydRoC$~pm$x(4}Bvr*ZFA=rqPdSfO_ug z;T=ReEB)?i$cyZ~31S&Rxsv%blYTy|)FA{Tn`1zqTJ&=+pzp^K_CZ1*#{xr<2KeY4 z)1YCWd`7r8^71{@$1;L*SkI--;E1CB0>T{Mzlul+ z;GG3Nmh)PErumKHvvcoQw;^tqaH}RrISe=$LMnh_!-zA=r$RTxQ; zQ*a{hCn#x}rzE(dzkwcY0M39 z8SQl1kx0sUWke|#M#GiU;UpmrinvJ-$uEpD-jJiIk7>7Xm`qaLMKePj_5q?7nyTqi z>-#flC^B65!3G3&(XLD{d=iR00?jX_1~z2?O~YQs`D9@5O-5*AnF=LbvglNZrN&-m z&D~k6!qn>xJhgTGM^RpPnUHAQ*lYx3qoJSbC<8}*^^$>hik5vR*_QMLWw=j=>i`a1 zZRW3M!~_VnxIp4R0A(~6)2DMe`4sz_PAt6!?TP8fOI>n)u#)J+(yiIm(aMJ-+1dkn z8&wE~($Xz#3C^J-Kn!{hwegq#{3R%!H1wjC1=B8KzW2J?ESIwo#xEz#x*xayLhANA zt~5GuU07{iHH;RYY~NunSy$B?5AyLcc#pV89fz@4hzKa= zv@zq?={N)#&5OMU2r-sRE)1KuKfja>4=ldD$Cw&9`u~B%Pgjn9$;>5{4Zl4NYOIA^ zxY2g5A(T}saNB2lns)98GX{cz3`K3mmMC z%^+@h*o>t^HW{K??)WMtrLjX3XpFeyFVRMtKPz%#q5wL%@9_KVfH2qcvk5SR zaWwCD@Nv-m1(>Gjs>{3hpfHYZ=XYEkDy2ll@FzUH^hEv2_gYi%EHx0$CLO){?&@0s ziW=ULgE@b~IN^?jfU%O-0YaERBnTbG9E6R!)kdymru+qfJc(nZg+gAem&d&vSq{i8 zRMbY5A?fI#A{#Wy8~6geFcWRB)2oo=a21kBy<=ogtUZQ|u5{>FtRdw@joXK_l`^6& zQ8Od8Sr=!PT=WHyk0$sHM=q++F$|%JKD&Csj=3qwJ5f;;Hpbs!-l}-qelD}%o_8`B zbjUtd%2iMa^16Qt;@{}TP zn$rc_oEM^M(T;;-SN7OkWr@|3i|66WH@I88r^sFEIy_wqohL3#R$O9$V0|hg8B%Sf2Y;yHny}m~%8$^}~-JX8{hy@>V&l=k33U!^Qa^zi4-wouIx@)`%?X! zF^evJ5&X?+lelpg2gdg34$|Aa_aoHEs66|ll`Sf(Y5z}?zUY}x(V^-v3p_k#3!w}B zZ*_Xviz)kCTAeN_3f|kyfRE6Obk-US->4VwJ8mwl%|4!vURBEJG?=LH^_)9KL;sA# zBXo98U3}X<|7F*20tY^GMfrRGZkb~Rh5J5>iDy}R8 z!Bmn-X(FBEY~k+v;tbCx+c=5rS_~S@u(m(k8s$J|1X(_;13buD)%dHs=-EY;Loht; z%odTBa1zZ}!W>Lu3Npng&7^-E8i77h=7Zt9%uX@CrTgb?x>>Qxinx}mPUFzpiTN~8 zM3=oU#u9-Em-r^w(@zPi)*|%xhIZf}Q6aU+(rn;{8SWm1@|P1wsJ(^ExV9}Q^yzwq z!`>^+_XV?5_`rG5{Y`3n+m6SQ-;QsR~d>r%aD~!aczIXLr)6WOh0l*REVtU_lJl> zO6w}RL#OmR$5ImV>Zc8Pu9*9ghPOol)2pv(Tye_DdB?f*s0A=^xcN6*1i!BlFz0sO z2#JB~J4FG2+2qcRs4dkJJk-H?M4^UJPY>5&MK z@#;i+7$4<%B<^r4G~O8N$wcO1k#ErlUp<6Cch(#X;_X;MlosbEOIO02SjN;Mf z@Yx~8jpGXHrSP;?!M@7M`DEF}B#HUjB699g9-mI0vHvNEj2Jz#@pQokq!{Vq`PF+s zWhwpHp!Y8T@w0Wj@oUr9;)SFqBo;KN?tvvf!CFMs1Nia#*ZQ#ADGE}xBvtyG7&KSUYSwa1;;QdLlyg0tyUdk5o#O4h`U4#m3i1im*+8}E zqg2q!3`b3pVlH*a_G2Ge1ZHn@rn)AUBYe`gk*H5Q8o_LDJ^B^;fs?mu{zvl#ylsC0 z@<8z@v(WRO1(>&0ZQpaeXZPwGWZYe11_j>-nZcZ@H{kpYx_%}4Z*uk%UYR&5Er=r) zHXHgMKaTmHpA4M(q_~4%AXHAyw3;lmlvF={>OSIk1Ve-j?+9-!&{Lyn`g>Vgy&RfQ z^o9;g9b6w9lDp*hA(bbciH}>Py2}UWY(o)-r-2fMlofmB-sFm*s6xbT@&zKj2ar{&)CZk9t}n@g-uv2dX;m=fGG+qt-zy+enS`8h9b2fk!t)Gz-6gVkujR^YvC_|Xxj!5#2x;{wit@r pA>tS^MRBN+ME Date: Wed, 22 Sep 2021 14:43:49 +0300 Subject: [PATCH 24/52] Updated fixtures --- .../53-delete-orphaned-connections.spec.js | 39 +++---------------- .../specs/fixtures.js | 32 +++++++++++++++ 2 files changed, 38 insertions(+), 33 deletions(-) diff --git a/app/api/migrations/migrations/53-delete-orphaned-connections/specs/53-delete-orphaned-connections.spec.js b/app/api/migrations/migrations/53-delete-orphaned-connections/specs/53-delete-orphaned-connections.spec.js index 3c90a297e3..304ef76522 100644 --- a/app/api/migrations/migrations/53-delete-orphaned-connections/specs/53-delete-orphaned-connections.spec.js +++ b/app/api/migrations/migrations/53-delete-orphaned-connections/specs/53-delete-orphaned-connections.spec.js @@ -16,50 +16,23 @@ describe('migration delete-orphaned-connections', () => { expect(migration.delta).toBe(53); }); - it('should delete all connections', async () => { + it('should delete all connections which do not have an existing entity', async () => { await migration.up(testingDB.mongodb); const connections = await testingDB.mongodb .collection('connections') .find() .toArray(); - expect(connections.length).toBe(0); + expect(connections.length).toBe(3); }); - it('should delete all orphaned connections except two remaining in hub', async () => { - const localFixtures = { - entities: [...fixtures.entities], - connections: [ - ...fixtures.connections, - { - entity: 'sharedid2', - hub: 'hub1', - }, - ], - }; - await testingDB.clearAllAndLoad(localFixtures); + it('should delete all connections who are alone in a hub', async () => { await migration.up(testingDB.mongodb); const connections = await testingDB.mongodb .collection('connections') .find() .toArray(); - expect(connections.length).toBe(2); - }); - - it('should not delete any connection', async () => { - const localFixtures = { - entities: [...fixtures.entities], - connections: [ - { ...fixtures.connections[0], entity: 'sharedid1' }, - { ...fixtures.connections[1], entity: 'sharedid2' }, - ], - }; - - await testingDB.clearAllAndLoad(localFixtures); - await migration.up(testingDB.mongodb); - const connections = await testingDB.mongodb - .collection('connections') - .find() - .toArray(); - expect(connections.length).toBe(2); + expect(connections).toEqual( + expect.arrayContaining([expect.not.objectContaining({ hub: 'hub3' })]) + ); }); }); diff --git a/app/api/migrations/migrations/53-delete-orphaned-connections/specs/fixtures.js b/app/api/migrations/migrations/53-delete-orphaned-connections/specs/fixtures.js index a40fe9f5fb..8ac4ecd5b8 100644 --- a/app/api/migrations/migrations/53-delete-orphaned-connections/specs/fixtures.js +++ b/app/api/migrations/migrations/53-delete-orphaned-connections/specs/fixtures.js @@ -8,6 +8,22 @@ export default { sharedId: 'sharedid2', title: 'test_doc_2', }, + { + sharedId: 'sharedid1', + title: 'test_doc', + }, + { + sharedId: 'sharedid3', + title: 'test_doc_2', + }, + { + sharedId: 'sharedid1', + title: 'test_doc', + }, + { + sharedId: 'sharedid3', + title: 'test_doc_2', + }, ], connections: [ { @@ -18,5 +34,21 @@ export default { entity: 'sharedid1', hub: 'hub1', }, + { + entity: 'sharedid2', + hub: 'hub1', + }, + { + entity: 'sharedid3', + hub: 'hub2', + }, + { + entity: 'sharedid1', + hub: 'hub3', + }, + { + entity: 'shareid4', + hub: 'hub3', + }, ], }; From af075d5eba7b882bdc89a156d2be987289a239d9 Mon Sep 17 00:00:00 2001 From: Laszlo Kecskes Date: Wed, 22 Sep 2021 14:01:47 +0200 Subject: [PATCH 25/52] handling languages --- app/api/csv/csvLoader.ts | 3 +- app/api/csv/importEntity.ts | 39 ++++++++++++++++++++++---- app/api/csv/specs/importEntity.spec.js | 11 ++++++-- 3 files changed, 42 insertions(+), 11 deletions(-) diff --git a/app/api/csv/csvLoader.ts b/app/api/csv/csvLoader.ts index 89fd4f115d..c03659688e 100644 --- a/app/api/csv/csvLoader.ts +++ b/app/api/csv/csvLoader.ts @@ -56,7 +56,7 @@ export class CSVLoader extends EventEmitter { (await settings.get()).languages ).map((l: LanguageSchema) => l.key); const { newNameGeneration = false } = await settings.get(); - await arrangeThesauri(file, template, this); + await arrangeThesauri(file, template, availableLanguages, this); await csv(await file.readStream(), this.stopOnError) .onRow(async (row: CSVRow) => { @@ -66,7 +66,6 @@ export class CSVLoader extends EventEmitter { options.language, newNameGeneration ); - if (rawEntity) { const entity = await importEntity(rawEntity, template, file, options); await translateEntity(entity, rawTranslations, template, file); diff --git a/app/api/csv/importEntity.ts b/app/api/csv/importEntity.ts index 5bfc54373c..d3c926b168 100644 --- a/app/api/csv/importEntity.ts +++ b/app/api/csv/importEntity.ts @@ -71,9 +71,24 @@ type Options = { language: string; }; -const arrangeThesauri = async (file: ImportFile, template: TemplateSchema, errorContext?: any) => { - const nameToThesauriIdSelects: { [k: string]: string } = {}; - const nameToThesauriIdMultiselects: { [k: string]: string } = {}; +const filterJSObject = (input: { [k: string]: any }, keys: string[]): { [k: string]: any } => { + const result: { [k: string]: any } = {}; + keys.forEach(k => { + if (input.hasOwnProperty(k)) { + result[k] = input[k]; + } + }); + return result; +}; + +const arrangeThesauri = async ( + file: ImportFile, + template: TemplateSchema, + languages?: string[], + errorContext?: any +) => { + let nameToThesauriIdSelects: { [k: string]: string } = {}; + let nameToThesauriIdMultiselects: { [k: string]: string } = {}; const thesauriIdToExistingValues = new Map(); const thesauriIdToNewValues: Map> = new Map(); const thesauriIdToNormalizedNewValues = new Map(); @@ -82,10 +97,17 @@ const arrangeThesauri = async (file: ImportFile, template: TemplateSchema, error ); thesauriRelatedProperties?.forEach(p => { if (p.content && p.type) { + const thesarusID = p.content.toString(); if (p.type === propertyTypes.select) { - nameToThesauriIdSelects[p.name] = p.content.toString(); + nameToThesauriIdSelects[p.name] = thesarusID; + languages?.forEach(suffix => { + nameToThesauriIdSelects[`${p.name}__${suffix}`] = thesarusID; + }); } else if (p.type === propertyTypes.multiselect) { - nameToThesauriIdMultiselects[p.name] = p.content.toString(); + nameToThesauriIdMultiselects[p.name] = thesarusID; + languages?.forEach(suffix => { + nameToThesauriIdMultiselects[`${p.name}__${suffix}`] = thesarusID; + }); } } }); @@ -116,7 +138,12 @@ const arrangeThesauri = async (file: ImportFile, template: TemplateSchema, error } } await csv(await file.readStream(), errorContext?.stopOnError) - .onRow(async (row: CSVRow) => { + .onRow(async (row: CSVRow, index: number) => { + if (index === 0) { + const columnnames = Object.keys(row); + nameToThesauriIdSelects = filterJSObject(nameToThesauriIdSelects, columnnames); + nameToThesauriIdMultiselects = filterJSObject(nameToThesauriIdMultiselects, columnnames); + } Object.entries(nameToThesauriIdSelects).forEach(([name, id]) => { const label = row[name]; if (label) { diff --git a/app/api/csv/specs/importEntity.spec.js b/app/api/csv/specs/importEntity.spec.js index d6b09add54..5b901f8020 100644 --- a/app/api/csv/specs/importEntity.spec.js +++ b/app/api/csv/specs/importEntity.spec.js @@ -2,6 +2,7 @@ import path from 'path'; import templates from 'api/templates'; import thesauri from 'api/thesauri'; +import settings from 'api/settings'; import { getFixturesFactory } from 'api/utils/fixturesFactory'; import db from 'api/utils/testing_db'; @@ -61,7 +62,8 @@ describe('arrangeThesauri', () => { await db.clearAllAndLoad(fixtures); template = await templates.getById(fixtureFactory.id('template')); file = importFile(path.join(__dirname, '/arrangeThesauriTest.csv')); - await arrangeThesauri(file, template); + const languages = (await settings.get()).languages.map(l => l.key); + await arrangeThesauri(file, template, languages); selectThesaurus = await thesauri.getById(fixtureFactory.id('select_thesaurus')); selectLabels = selectThesaurus.values.map(tv => tv.label); selectLabelsSet = new Set(selectLabels); @@ -85,12 +87,11 @@ second,second`; }); it('should not fail if the select or multiselect fields are missing from the csv', async () => { - const noselTemplate = templates.getById(fixtureFactory.id('template')); const csv = `title,unrelated_property first,first second,second`; const readStreamMock = mockCsvFileReadStream(csv); - await arrangeThesauri(importFile('mockedFile'), noselTemplate); + await arrangeThesauri(importFile('mockedFile'), template); readStreamMock.mockRestore(); }); @@ -119,4 +120,8 @@ second,second`; expect(selectLabels.length).toBe(selectLabelsSet.size); expect(multiselectLabels.length).toBe(multiselectLabelsSet.size); }); + + it('dummy test', () => { + console.log('there'); + }) }); From ee3eb2ce514e422e74ce8145a694c363e36d077f Mon Sep 17 00:00:00 2001 From: Laszlo Kecskes Date: Wed, 22 Sep 2021 14:22:55 +0200 Subject: [PATCH 26/52] updated test with languages --- app/api/csv/specs/arrangeThesauriTest.csv | 34 +++++++++++------------ app/api/csv/specs/importEntity.spec.js | 32 +++++++++++++++------ 2 files changed, 40 insertions(+), 26 deletions(-) diff --git a/app/api/csv/specs/arrangeThesauriTest.csv b/app/api/csv/specs/arrangeThesauriTest.csv index fd3627b235..20efec600f 100644 --- a/app/api/csv/specs/arrangeThesauriTest.csv +++ b/app/api/csv/specs/arrangeThesauriTest.csv @@ -1,17 +1,17 @@ -title,unrelated_property, select_property ,multiselect_property -select_1,unrelated_text,B,A -select_2,unrelated_text,C,A -select_3,unrelated_text,b,A -select_4,unrelated_text,B,A -select_5,unrelated_text,d,A -select_6,unrelated_text,D,A -select_7,unrelated_text, b,A -select_8,unrelated_text, ,A -select_8,unrelated_text, ,A -multiselect_1,unrelated_text,A,B -multiselect_2,unrelated_text,A,c -multiselect_3,unrelated_text,A,A|b -multiselect_4,unrelated_text,A,a|B|C -multiselect_5,unrelated_text,A, a| b | -multiselect_6,unrelated_text,A, | | -multiselect_7, unrelated_text,A,A|B|C|D| |E| e| g +title,unrelated_property,select_property__en, select_property__es,multiselect_property__en, multiselect_property__es +select_1,unrelated_text,B,Bes,A,Aes +select_2,unrelated_text,C,Ces,A,Aes +select_3,unrelated_text,b,bes,A,Aes +select_4,unrelated_text,B,Bes,A,Aes +select_5,unrelated_text,d,des,A,Aes +select_6,unrelated_text,D,Des,A,Aes +select_7,unrelated_text, b,bes,A,Aes +select_8,unrelated_text, , ,A,Aes +select_8,unrelated_text, , ,A,Aes +multiselect_1,unrelated_text,A,Aes,B,Bes +multiselect_2,unrelated_text,A,Aes,c,ces +multiselect_3,unrelated_text,A,Aes,A|b,Aes|bes +multiselect_4,unrelated_text,A,Aes,a|B|C,aes|Bes|Ces +multiselect_5,unrelated_text,A,Aes, a| b | , aes| bes | +multiselect_6,unrelated_text,A,Aes, | | , | | +multiselect_7, unrelated_text,A,Aes,A|B|C|D| |E| e| g ,Aes|Bes|Ces|Des| |Ees| ees| ges diff --git a/app/api/csv/specs/importEntity.spec.js b/app/api/csv/specs/importEntity.spec.js index 5b901f8020..d735a54f10 100644 --- a/app/api/csv/specs/importEntity.spec.js +++ b/app/api/csv/specs/importEntity.spec.js @@ -42,7 +42,10 @@ const fixtures = { { _id: db.id(), site_name: 'Uwazi', - languages: [{ key: 'en', label: 'English', default: true }], + languages: [ + { key: 'en', label: 'English', default: true }, + { key: 'es', label: 'Spanish' }, + ], }, ], }; @@ -96,13 +99,28 @@ second,second`; }); it('should create values in thesauri', async () => { - expect(selectLabels).toEqual(['A', 'B', 'C', 'd']); - expect(multiselectLabels).toEqual(['A', 'B', 'c', 'D', 'E', 'g']); + expect(selectLabels).toEqual(['A', 'B', 'Bes', 'C', 'Ces', 'd', 'des', 'Aes']); + expect(multiselectLabels).toEqual([ + 'A', + 'B', + 'Aes', + 'Bes', + 'c', + 'ces', + 'D', + 'E', + 'g', + 'Des', + 'Ees', + 'ges', + ]); }); it('should not repeat case sensitive values', async () => { - ['a', 'b', 'c', 'D'].forEach(letter => expect(selectLabelsSet.has(letter)).toBe(false)); - ['a', 'b', 'C', 'd', 'e', 'G'].forEach(letter => + ['a', 'aes', 'b', 'bes', 'c', 'ces', 'D', 'Des'].forEach(letter => + expect(selectLabelsSet.has(letter)).toBe(false) + ); + ['a', 'aes', 'b', 'bes', 'C', 'Ces', 'd', 'des', 'e', 'ees', 'G', 'Ges'].forEach(letter => expect(multiselectLabelsSet.has(letter)).toBe(false) ); }); @@ -120,8 +138,4 @@ second,second`; expect(selectLabels.length).toBe(selectLabelsSet.size); expect(multiselectLabels.length).toBe(multiselectLabelsSet.size); }); - - it('dummy test', () => { - console.log('there'); - }) }); From cdad29e907a234db1dda6fab2ab4544b19d7ce78 Mon Sep 17 00:00:00 2001 From: Kevin Date: Wed, 22 Sep 2021 16:14:50 +0300 Subject: [PATCH 27/52] More tests --- e2e/suites/convert-entity-template.test.ts | 33 ++++++++++++++-------- 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/e2e/suites/convert-entity-template.test.ts b/e2e/suites/convert-entity-template.test.ts index 757f78460a..b90992d7f9 100644 --- a/e2e/suites/convert-entity-template.test.ts +++ b/e2e/suites/convert-entity-template.test.ts @@ -63,30 +63,39 @@ const UploadSupportingFileToEntity = async (entityName: string): Promise = await page.goto(`${host}`); await expect(page).toClick('span', { text: 'Restricted' }); await expect(page).toClick('div.item-name > span', { text: entityName }); - console.log('clicked entity...'); - await expect(page).toClick('button.upload-button > span', { text: 'Add supporting file' }); - console.log('clicked supporting files button...'); + await expect(page).toClick('button[type="button"].upload-button'); const [fileChooser] = await Promise.all([ page.waitForFileChooser(), - page.click('div.attachments-modal__dropzone'), + page.click('div.attachments-modal__dropzone > button'), ]); await fileChooser.accept([`${__dirname}/test_files/batman.jpg`]); - // await expect(page).toUploadFile('#upload-button-input', `${__dirname}/test_files/valid.pdf`); +}; + +const convertEntityTemplate = async (entityName: string, targetTemplate: string): Promise => { + await page.goto(`${host}`); + await expect(page).toClick('span', { text: 'Restricted' }); + await expect(page).toClick('div.item-name > span', { text: entityName }); + await expect(page).toClick('div.sidepanel-footer > button[type="button"].edit-metadata'); +}; + +const setupTest = async () => { + await createTemplate('Without image'); + await createTemplate('With image'); + await createEntity('Without image'); + await uploadPDFToEntity('Without image'); + await UploadSupportingFileToEntity('Without image'); }; describe('Convert entity template', () => { beforeAll(async () => { await setupPreFlights(); - await createTemplate('Without image'); - await createTemplate('With image'); - await createEntity('Without image'); - await uploadPDFToEntity('Without image'); - console.log('Beginning uploading supporting file...'); - await UploadSupportingFileToEntity('Without image'); + + await convertEntityTemplate('Without image', 'With image'); }); it('Should create new entity', async () => { - expect(true).toEqual(true); + await setupTest(); + await expect(page).toClick('a[(type = "button")].btn'); }); afterAll(async () => { From 92f197014e4d733a7943fb42f8819d76ce6f4118 Mon Sep 17 00:00:00 2001 From: Santiago Date: Wed, 22 Sep 2021 14:06:29 -0300 Subject: [PATCH 28/52] avoid making the second call to api.search --- app/react/Library/helpers/requestState.js | 81 ++++++++++++----------- 1 file changed, 41 insertions(+), 40 deletions(-) diff --git a/app/react/Library/helpers/requestState.js b/app/react/Library/helpers/requestState.js index c0b813e9e4..e2b56e647f 100644 --- a/app/react/Library/helpers/requestState.js +++ b/app/react/Library/helpers/requestState.js @@ -60,51 +60,52 @@ export default function requestState(request, globalResources, calculateTableCol ); const markersRequest = request.set({ ...docsQuery, - geolocation: templatesWithGeolocation && true, + geolocation: true, }); - return Promise.all([api.search(documentsRequest), api.search(markersRequest)]).then( - ([documents, markers]) => { - const templates = globalResources.templates.toJS(); - const filterState = libraryHelpers.URLQueryToState( - documentsRequest.data, - templates, - globalResources.thesauris.toJS(), - globalResources.relationTypes.toJS(), - request.data.quickLabelThesaurus - ? getThesaurusPropertyNames(request.data.quickLabelThesaurus, templates) - : [] - ); + return Promise.all([ + api.search(documentsRequest), + templatesWithGeolocation ? api.search(markersRequest) : { rows: [] }, + ]).then(([documents, markers]) => { + const templates = globalResources.templates.toJS(); + const filterState = libraryHelpers.URLQueryToState( + documentsRequest.data, + templates, + globalResources.thesauris.toJS(), + globalResources.relationTypes.toJS(), + request.data.quickLabelThesaurus + ? getThesaurusPropertyNames(request.data.quickLabelThesaurus, templates) + : [] + ); - const state = { - library: { - filters: { - documentTypes: documentsRequest.data.types || [], - properties: filterState.properties, - }, - aggregations: documents.aggregations, - search: filterState.search, - documents, - markers, + const state = { + library: { + filters: { + documentTypes: documentsRequest.data.types || [], + properties: filterState.properties, }, - }; + aggregations: documents.aggregations, + search: filterState.search, + documents, + markers, + }, + }; - const addinsteadOfSet = Boolean(docsQuery.from); + const addinsteadOfSet = Boolean(docsQuery.from); - const dispatchedActions = [ - setReduxState(state, 'library', addinsteadOfSet), - actions.set('library.sidepanel.quickLabelState', { - thesaurus: request.data.quickLabelThesaurus, - autoSave: false, - }), - ]; - if (calculateTableColumns) { - const tableViewColumns = getTableColumns(documents, templates, documentsRequest.data.types); - dispatchedActions.push(dispatch => - wrapDispatch(dispatch, 'library')(setTableViewColumns(tableViewColumns)) - ); - } - return dispatchedActions; + const dispatchedActions = [ + setReduxState(state, 'library', addinsteadOfSet), + actions.set('library.sidepanel.quickLabelState', { + thesaurus: request.data.quickLabelThesaurus, + autoSave: false, + }), + ]; + if (calculateTableColumns) { + const tableViewColumns = getTableColumns(documents, templates, documentsRequest.data.types); + dispatchedActions.push(dispatch => + wrapDispatch(dispatch, 'library')(setTableViewColumns(tableViewColumns)) + ); } - ); + return dispatchedActions; + }); } From 0d1a057b4b4b41ad2147c2197e4ff42b37824f3f Mon Sep 17 00:00:00 2001 From: Santiago Date: Wed, 22 Sep 2021 16:26:15 -0300 Subject: [PATCH 29/52] test expanded --- .../__snapshots__/resquestState.spec.js.snap | 13 +++ .../helpers/specs/resquestState.spec.js | 96 +++++++++++++------ 2 files changed, 78 insertions(+), 31 deletions(-) diff --git a/app/react/Library/helpers/specs/__snapshots__/resquestState.spec.js.snap b/app/react/Library/helpers/specs/__snapshots__/resquestState.spec.js.snap index 62d23cf128..04ba361b51 100644 --- a/app/react/Library/helpers/specs/__snapshots__/resquestState.spec.js.snap +++ b/app/react/Library/helpers/specs/__snapshots__/resquestState.spec.js.snap @@ -25,3 +25,16 @@ Array [ }, ] `; + +exports[`static requestState() when is for geolocation should work when there are no geolocation type properties 1`] = ` +Array [ + [Function], + Object { + "type": "library.sidepanel.quickLabelState/SET", + "value": Object { + "autoSave": false, + "thesaurus": undefined, + }, + }, +] +`; diff --git a/app/react/Library/helpers/specs/resquestState.spec.js b/app/react/Library/helpers/specs/resquestState.spec.js index c80ba11891..2ab83f975f 100644 --- a/app/react/Library/helpers/specs/resquestState.spec.js +++ b/app/react/Library/helpers/specs/resquestState.spec.js @@ -6,38 +6,41 @@ import rison from 'rison-node'; import requestState, { processQuery } from '../requestState'; describe('static requestState()', () => { - const aggregations = { buckets: [] }; - const templates = [ - { - name: 'Decision', - _id: 'abc1', - properties: [ - { name: 'p', filter: true, type: 'text', prioritySorting: true }, - { name: 'country', filter: false, type: 'select', content: 'countries' }, - { name: 'location', filter: false, type: 'geolocation' }, - ], - }, - { name: 'Ruling', _id: 'abc2', properties: [] }, - ]; - const relationTypes = [ - { name: 'Victim', _id: 'abc3', properties: [{ name: 'p', filter: true, type: 'text' }] }, - ]; - - const thesauris = [{ name: 'countries', _id: '1', values: [] }]; - const documents = { - rows: [{ title: 'Something to publish' }, { title: 'My best recipes' }], - totalRows: 2, - aggregations, - }; - const globalResources = { - templates: Immutable.fromJS(templates), - settings: { collection: Immutable.fromJS({ features: {} }) }, - thesauris: Immutable.fromJS(thesauris), - relationTypes: Immutable.fromJS(relationTypes), - user: Immutable.fromJS({}), - }; - + let globalResources; + let templates; beforeEach(() => { + const aggregations = { buckets: [] }; + templates = [ + { + name: 'Decision', + _id: 'abc1', + properties: [ + { name: 'p', filter: true, type: 'text', prioritySorting: true }, + { name: 'country', filter: false, type: 'select', content: 'countries' }, + { name: 'location', filter: false, type: 'geolocation' }, + ], + }, + { name: 'Ruling', _id: 'abc2', properties: [] }, + ]; + const relationTypes = [ + { name: 'Victim', _id: 'abc3', properties: [{ name: 'p', filter: true, type: 'text' }] }, + ]; + + const thesauris = [{ name: 'countries', _id: '1', values: [] }]; + const documents = { + rows: [{ title: 'Something to publish' }, { title: 'My best recipes' }], + totalRows: 2, + aggregations, + }; + + globalResources = { + templates: Immutable.fromJS(templates), + settings: { collection: Immutable.fromJS({ features: {} }) }, + thesauris: Immutable.fromJS(thesauris), + relationTypes: Immutable.fromJS(relationTypes), + user: Immutable.fromJS({}), + }; + spyOn(searchAPI, 'search').and.returnValue(Promise.resolve(documents)); }); @@ -83,6 +86,37 @@ describe('static requestState()', () => { expect(searchAPI.search).toHaveBeenCalledWith(expectedSearch); expect(actions).toMatchSnapshot(); }); + + it('should work when there are no geolocation type properties', async () => { + templates = [ + { + name: 'Appeal', + _id: 'abc2', + properties: [ + { name: 'p', filter: true, type: 'text', prioritySorting: true }, + { name: 'description', filter: false, type: 'markdown' }, + ], + }, + ]; + globalResources.templates = Immutable.fromJS(templates); + const query = { q: rison.encode({ filters: { something: 1 }, types: [] }) }; + const expectedSearch = { + data: { + sort: prioritySortingCriteria.get({ templates: Immutable.fromJS(templates) }).sort, + order: prioritySortingCriteria.get({ templates: Immutable.fromJS(templates) }).order, + filters: { something: 1 }, + types: [], + view: undefined, + }, + headers: {}, + }; + + const request = new RequestParams(query); + const actions = await requestState(request, globalResources); + + expect(searchAPI.search).toHaveBeenCalledWith(expectedSearch); + expect(actions).toMatchSnapshot(); + }); }); describe('processQuery()', () => { From 2b554b5700d654d58f413233981e9af529d88015 Mon Sep 17 00:00:00 2001 From: Santiago Date: Wed, 22 Sep 2021 17:42:09 -0300 Subject: [PATCH 30/52] moved ternary to const for readability --- app/react/Library/helpers/requestState.js | 80 +++++++++++------------ 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/app/react/Library/helpers/requestState.js b/app/react/Library/helpers/requestState.js index e2b56e647f..cbca19052e 100644 --- a/app/react/Library/helpers/requestState.js +++ b/app/react/Library/helpers/requestState.js @@ -62,50 +62,50 @@ export default function requestState(request, globalResources, calculateTableCol ...docsQuery, geolocation: true, }); + const markersRequestSearch = templatesWithGeolocation ? api.search(markersRequest) : { rows: [] }; - return Promise.all([ - api.search(documentsRequest), - templatesWithGeolocation ? api.search(markersRequest) : { rows: [] }, - ]).then(([documents, markers]) => { - const templates = globalResources.templates.toJS(); - const filterState = libraryHelpers.URLQueryToState( - documentsRequest.data, - templates, - globalResources.thesauris.toJS(), - globalResources.relationTypes.toJS(), - request.data.quickLabelThesaurus - ? getThesaurusPropertyNames(request.data.quickLabelThesaurus, templates) - : [] - ); + return Promise.all([api.search(documentsRequest), markersRequestSearch]).then( + ([documents, markers]) => { + const templates = globalResources.templates.toJS(); + const filterState = libraryHelpers.URLQueryToState( + documentsRequest.data, + templates, + globalResources.thesauris.toJS(), + globalResources.relationTypes.toJS(), + request.data.quickLabelThesaurus + ? getThesaurusPropertyNames(request.data.quickLabelThesaurus, templates) + : [] + ); - const state = { - library: { - filters: { - documentTypes: documentsRequest.data.types || [], - properties: filterState.properties, + const state = { + library: { + filters: { + documentTypes: documentsRequest.data.types || [], + properties: filterState.properties, + }, + aggregations: documents.aggregations, + search: filterState.search, + documents, + markers, }, - aggregations: documents.aggregations, - search: filterState.search, - documents, - markers, - }, - }; + }; - const addinsteadOfSet = Boolean(docsQuery.from); + const addinsteadOfSet = Boolean(docsQuery.from); - const dispatchedActions = [ - setReduxState(state, 'library', addinsteadOfSet), - actions.set('library.sidepanel.quickLabelState', { - thesaurus: request.data.quickLabelThesaurus, - autoSave: false, - }), - ]; - if (calculateTableColumns) { - const tableViewColumns = getTableColumns(documents, templates, documentsRequest.data.types); - dispatchedActions.push(dispatch => - wrapDispatch(dispatch, 'library')(setTableViewColumns(tableViewColumns)) - ); + const dispatchedActions = [ + setReduxState(state, 'library', addinsteadOfSet), + actions.set('library.sidepanel.quickLabelState', { + thesaurus: request.data.quickLabelThesaurus, + autoSave: false, + }), + ]; + if (calculateTableColumns) { + const tableViewColumns = getTableColumns(documents, templates, documentsRequest.data.types); + dispatchedActions.push(dispatch => + wrapDispatch(dispatch, 'library')(setTableViewColumns(tableViewColumns)) + ); + } + return dispatchedActions; } - return dispatchedActions; - }); + ); } From 3049ca49a214d49192d03da8de400caf791d91fb Mon Sep 17 00:00:00 2001 From: Santiago Date: Wed, 22 Sep 2021 17:42:34 -0300 Subject: [PATCH 31/52] removed unneeded expect --- .../specs/__snapshots__/resquestState.spec.js.snap | 13 ------------- .../Library/helpers/specs/resquestState.spec.js | 6 +++--- 2 files changed, 3 insertions(+), 16 deletions(-) diff --git a/app/react/Library/helpers/specs/__snapshots__/resquestState.spec.js.snap b/app/react/Library/helpers/specs/__snapshots__/resquestState.spec.js.snap index 04ba361b51..62d23cf128 100644 --- a/app/react/Library/helpers/specs/__snapshots__/resquestState.spec.js.snap +++ b/app/react/Library/helpers/specs/__snapshots__/resquestState.spec.js.snap @@ -25,16 +25,3 @@ Array [ }, ] `; - -exports[`static requestState() when is for geolocation should work when there are no geolocation type properties 1`] = ` -Array [ - [Function], - Object { - "type": "library.sidepanel.quickLabelState/SET", - "value": Object { - "autoSave": false, - "thesaurus": undefined, - }, - }, -] -`; diff --git a/app/react/Library/helpers/specs/resquestState.spec.js b/app/react/Library/helpers/specs/resquestState.spec.js index 2ab83f975f..fbe6bc1211 100644 --- a/app/react/Library/helpers/specs/resquestState.spec.js +++ b/app/react/Library/helpers/specs/resquestState.spec.js @@ -84,6 +84,7 @@ describe('static requestState()', () => { const actions = await requestState(request, globalResources); expect(searchAPI.search).toHaveBeenCalledWith(expectedSearch); + expect(actions).toMatchSnapshot(); }); @@ -110,12 +111,11 @@ describe('static requestState()', () => { }, headers: {}, }; - const request = new RequestParams(query); - const actions = await requestState(request, globalResources); + + await requestState(request, globalResources); expect(searchAPI.search).toHaveBeenCalledWith(expectedSearch); - expect(actions).toMatchSnapshot(); }); }); From a5d8728b63d8f3d8069aa0519913d6c98618f2d7 Mon Sep 17 00:00:00 2001 From: Santiago Date: Wed, 22 Sep 2021 17:54:06 -0300 Subject: [PATCH 32/52] moved ternary to const for readability --- app/react/Library/helpers/requestState.js | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/app/react/Library/helpers/requestState.js b/app/react/Library/helpers/requestState.js index cbca19052e..e65eaf987f 100644 --- a/app/react/Library/helpers/requestState.js +++ b/app/react/Library/helpers/requestState.js @@ -55,16 +55,21 @@ export default function requestState(request, globalResources, calculateTableCol const documentsRequest = request.set( tocGenerationUtils.aggregations(docsQuery, globalResources.settings.collection.toJS()) ); + const templatesWithGeolocation = globalResources.templates.find(template => template.get('properties').find(property => property.get('type') === 'geolocation') ); - const markersRequest = request.set({ - ...docsQuery, - geolocation: true, - }); - const markersRequestSearch = templatesWithGeolocation ? api.search(markersRequest) : { rows: [] }; - return Promise.all([api.search(documentsRequest), markersRequestSearch]).then( + const markersRequest = templatesWithGeolocation + ? api.search( + request.set({ + ...docsQuery, + geolocation: true, + }) + ) + : { rows: [] }; + + return Promise.all([api.search(documentsRequest), markersRequest]).then( ([documents, markers]) => { const templates = globalResources.templates.toJS(); const filterState = libraryHelpers.URLQueryToState( From 05abbd0d9883ff7db18a869b7962c37d661b93d0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 23 Sep 2021 07:42:04 +0000 Subject: [PATCH 33/52] Bump tmpl from 1.0.4 to 1.0.5 Bumps [tmpl](https://github.com/daaku/nodejs-tmpl) from 1.0.4 to 1.0.5. - [Release notes](https://github.com/daaku/nodejs-tmpl/releases) - [Commits](https://github.com/daaku/nodejs-tmpl/commits/v1.0.5) --- updated-dependencies: - dependency-name: tmpl dependency-type: indirect ... Signed-off-by: dependabot[bot] --- yarn.lock | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/yarn.lock b/yarn.lock index 57b6fc1ea8..c7f7a12951 100644 --- a/yarn.lock +++ b/yarn.lock @@ -14499,8 +14499,9 @@ tmp@^0.2.1: rimraf "^3.0.0" tmpl@1.0.x: - version "1.0.4" - resolved "https://registry.yarnpkg.com/tmpl/-/tmpl-1.0.4.tgz#23640dd7b42d00433911140820e5cf440e521dd1" + version "1.0.5" + resolved "https://registry.yarnpkg.com/tmpl/-/tmpl-1.0.5.tgz#8683e0b902bb9c20c4f726e3c0b69f36518c07cc" + integrity sha512-3f0uOEAQwIqGuWW2MVzYg8fV/QNnc/IpuJNG837rLuczAaLVHslWHZQj4IGiEl5Hs3kkbhwL9Ab7Hrsmuj+Smw== to-arraybuffer@^1.0.0: version "1.0.1" From a4bc6c3182af8353171a5136bd86c2794dcaee38 Mon Sep 17 00:00:00 2001 From: Laszlo Kecskes Date: Thu, 23 Sep 2021 14:59:57 +0200 Subject: [PATCH 34/52] corrected missing translation problem --- app/api/csv/importEntity.ts | 20 +++-- .../__snapshots__/importFile.spec.ts.snap | 8 +- app/api/csv/specs/csvLoader.spec.js | 18 +++- app/api/csv/specs/csvLoaderFixtures.ts | 82 +++++++++++-------- app/api/csv/specs/test.csv | 8 +- app/api/csv/typeParsers/select.ts | 18 ++-- 6 files changed, 93 insertions(+), 61 deletions(-) diff --git a/app/api/csv/importEntity.ts b/app/api/csv/importEntity.ts index d3c926b168..4d6b45823d 100644 --- a/app/api/csv/importEntity.ts +++ b/app/api/csv/importEntity.ts @@ -167,20 +167,22 @@ const arrangeThesauri = async ( } }) .read(); - await Promise.all( - allRelatedThesauri.map(thesaurus => { - if (thesaurus !== null) { - const newValues: { label: string }[] = Array.from( - thesauriIdToNewValues.get(thesaurus._id.toString()) || [] - ).map(tval => ({ label: tval })); + for (let i = 0; i < allRelatedThesauri.length; i += 1) { + const thesaurus = allRelatedThesauri[i]; + if (thesaurus !== null) { + const newValues: { label: string }[] = Array.from( + thesauriIdToNewValues.get(thesaurus._id.toString()) || [] + ).map(tval => ({ label: tval })); + if (newValues.length > 0) { const thesaurusValues = thesaurus.values || []; - return thesauri.save({ + // eslint-disable-next-line no-await-in-loop + await thesauri.save({ ...thesaurus, values: thesaurusValues.concat(newValues), }); } - }) - ); + } + } }; const importEntity = async ( diff --git a/app/api/csv/specs/__snapshots__/importFile.spec.ts.snap b/app/api/csv/specs/__snapshots__/importFile.spec.ts.snap index 44f7c9edb1..5b4864d04b 100644 --- a/app/api/csv/specs/__snapshots__/importFile.spec.ts.snap +++ b/app/api/csv/specs/__snapshots__/importFile.spec.ts.snap @@ -1,11 +1,11 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP exports[`importFile readStream should return a readable stream for the csv file 1`] = ` -"Title , text label , numeric label, non configured, select_label, not defined type, geolocation_geolocation,auto id, additional tag(s) +"Title , text label , numeric label, non configured, select_label, not defined type, geolocation_geolocation,auto id, additional tag(s), multi_select_label -title1, text value 1, 1977, ______________, thesauri1 , notType1 , 1|1,,tag1 -title2, text value 2, 2019, ______________, thesauri2 , notType2 , ,,tag2 -title3, text value 3, 2020, ______________, thesauri2 , notType3 , 0|0,,tag3 +title1, text value 1, 1977, ______________, thesauri1 , notType1 , 1|1,,tag1, multivalue1 +title2, text value 2, 2019, ______________, thesauri2 , notType2 , ,,tag2, multivalue2 +title3, text value 3, 2020, ______________, thesauri2 , notType3 , 0|0,,tag3, multivalue1|multivalue3 " `; diff --git a/app/api/csv/specs/csvLoader.spec.js b/app/api/csv/specs/csvLoader.spec.js index 22a5ce6218..7a800e2410 100644 --- a/app/api/csv/specs/csvLoader.spec.js +++ b/app/api/csv/specs/csvLoader.spec.js @@ -19,7 +19,6 @@ describe('csvLoader', () => { beforeAll(async () => { await db.clearAllAndLoad(fixtures); spyOn(search, 'indexEntities').and.returnValue(Promise.resolve()); - spyOn(translations, 'updateContext').and.returnValue(Promise.resolve()); }); afterAll(async () => db.disconnect()); @@ -160,6 +159,7 @@ describe('csvLoader', () => { 'geolocation_geolocation', 'auto_id', 'additional_tag(s)', + 'multi_select_label', ]); }); @@ -169,6 +169,22 @@ describe('csvLoader', () => { expect(textValues.length).toEqual(0); }); + it('should arrange translations for selects and multiselects', async () => { + const trs = await translations.get(); + trs.forEach(tr => { + expect(tr.contexts.find(c => c.label === 'thesauri1').values).toMatchObject({ + thesauri1: 'thesauri1', + thesauri2: 'thesauri2', + }); + expect(tr.contexts.find(c => c.label === 'multi_select_thesaurus').values).toMatchObject({ + multi_select_thesaurus: 'multi_select_thesaurus', + multivalue1: 'multivalue1', + multivalue2: 'multivalue2', + multivalue3: 'multivalue3', + }); + }); + }); + describe('metadata parsing', () => { it('should parse metadata properties by type using typeParsers', () => { const textValues = imported.map(i => i.metadata.text_label[0].value); diff --git a/app/api/csv/specs/csvLoaderFixtures.ts b/app/api/csv/specs/csvLoaderFixtures.ts index 558c38797d..95ee8f4796 100644 --- a/app/api/csv/specs/csvLoaderFixtures.ts +++ b/app/api/csv/specs/csvLoaderFixtures.ts @@ -3,10 +3,38 @@ import { propertyTypes } from 'shared/propertyTypes'; import { templateUtils } from 'api/templates'; const template1Id = db.id(); +const multiSelectThesaurusId = db.id(); const thesauri1Id = db.id(); const templateToRelateId = db.id(); const templateWithGeneratedTitle = db.id(); +const commonTranslationContexts = [ + { + id: 'System', + label: 'System', + values: [ + { key: 'original 1', value: 'original 1' }, + { key: 'original 2', value: 'original 2' }, + { key: 'original 3', value: 'original 3' }, + ], + }, + { + id: thesauri1Id.toString(), + label: 'thesauri1', + values: [{ key: 'thesauri1', value: 'thesauri1' }], + type: 'Dictionary', + }, + { + id: multiSelectThesaurusId.toString(), + label: 'multi_select_thesaurus', + values: [ + { key: 'multi_select_thesaurus', value: 'multi_select_thesaurus' }, + { key: 'multivalue1', value: 'multivalue1' }, + ], + type: 'Dictionary', + }, +]; + export default { templates: [ { @@ -32,7 +60,7 @@ export default { type: propertyTypes.select, label: 'select label', name: templateUtils.safeName('select label'), - content: thesauri1Id, + content: thesauri1Id.toString(), }, { type: 'non_defined_type', @@ -60,6 +88,12 @@ export default { label: 'additional tag(s)', name: templateUtils.safeName('additional tag(s)', true), }, + { + type: propertyTypes.multiselect, + label: 'multi_select_label', + name: templateUtils.safeName('multi_select_label'), + content: multiSelectThesaurusId.toString(), + }, ], }, { @@ -87,6 +121,16 @@ export default { }, ], }, + { + _id: multiSelectThesaurusId, + name: 'multi_select_thesaurus', + values: [ + { + label: 'multivalue1', + id: db.id().toString(), + }, + ], + }, ], settings: [ @@ -106,47 +150,17 @@ export default { { _id: db.id(), locale: 'en', - contexts: [ - { - id: 'System', - label: 'System', - values: [ - { key: 'original 1', value: 'original 1' }, - { key: 'original 2', value: 'original 2' }, - { key: 'original 3', value: 'original 3' }, - ], - }, - ], + contexts: commonTranslationContexts, }, { _id: db.id(), locale: 'es', - contexts: [ - { - id: 'System', - label: 'System', - values: [ - { key: 'original 1', value: 'original 1' }, - { key: 'original 2', value: 'original 2' }, - { key: 'original 3', value: 'original 3' }, - ], - }, - ], + contexts: commonTranslationContexts, }, { _id: db.id(), locale: 'fr', - contexts: [ - { - id: 'System', - label: 'System', - values: [ - { key: 'original 1', value: 'original 1' }, - { key: 'original 2', value: 'original 2' }, - { key: 'original 3', value: 'original 3' }, - ], - }, - ], + contexts: commonTranslationContexts, }, ], }; diff --git a/app/api/csv/specs/test.csv b/app/api/csv/specs/test.csv index 93a6521abe..25506ae1ea 100644 --- a/app/api/csv/specs/test.csv +++ b/app/api/csv/specs/test.csv @@ -1,5 +1,5 @@ -Title , text label , numeric label, non configured, select_label, not defined type, geolocation_geolocation,auto id, additional tag(s) +Title , text label , numeric label, non configured, select_label, not defined type, geolocation_geolocation,auto id, additional tag(s), multi_select_label -title1, text value 1, 1977, ______________, thesauri1 , notType1 , 1|1,,tag1 -title2, text value 2, 2019, ______________, thesauri2 , notType2 , ,,tag2 -title3, text value 3, 2020, ______________, thesauri2 , notType3 , 0|0,,tag3 +title1, text value 1, 1977, ______________, thesauri1 , notType1 , 1|1,,tag1, multivalue1 +title2, text value 2, 2019, ______________, thesauri2 , notType2 , ,,tag2, multivalue2 +title3, text value 3, 2020, ______________, thesauri2 , notType3 , 0|0,,tag3, multivalue1|multivalue3 diff --git a/app/api/csv/typeParsers/select.ts b/app/api/csv/typeParsers/select.ts index bf914ebc56..1f8e3b295c 100644 --- a/app/api/csv/typeParsers/select.ts +++ b/app/api/csv/typeParsers/select.ts @@ -1,6 +1,6 @@ import thesauri from 'api/thesauri'; import { RawEntity } from 'api/csv/entityRow'; -import { ThesaurusValueSchema, ThesaurusSchema } from 'shared/types/thesaurusType'; +import { ThesaurusSchema } from 'shared/types/thesaurusType'; import { MetadataObjectSchema, PropertySchema } from 'shared/types/commonTypes'; import { ensure } from 'shared/tsUtils'; @@ -19,18 +19,18 @@ const select = async ( const currentThesauri = (await thesauri.getById(property.content)) || ({} as ThesaurusSchema); const thesauriValues = currentThesauri.values || []; - if (normalizeThesaurusLabel(entityToImport[ensure(property.name)]) === '') { + const propValue = entityToImport[ensure(property.name)]; + const normalizedPropValue = normalizeThesaurusLabel(propValue); + if (!normalizedPropValue) { return null; } - const thesauriMatching = (v: ThesaurusValueSchema) => - normalizeThesaurusLabel(v.label) === - normalizeThesaurusLabel(entityToImport[ensure(property.name)]); + const thesarusValue = thesauriValues.find( + tv => normalizeThesaurusLabel(tv.label) === normalizedPropValue + ); - const value = thesauriValues.find(thesauriMatching); - - if (value?.id) { - return [{ value: value.id, label: value.label }]; + if (thesarusValue?.id) { + return [{ value: thesarusValue.id, label: thesarusValue.label }]; } return null; From c5f6842d6fcc71742e6fb79c3fee683642c3e2e5 Mon Sep 17 00:00:00 2001 From: Laszlo Kecskes Date: Fri, 24 Sep 2021 12:42:51 +0200 Subject: [PATCH 35/52] removing eslint line --- app/api/csv/importEntity.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/app/api/csv/importEntity.ts b/app/api/csv/importEntity.ts index 4d6b45823d..ede65a6114 100644 --- a/app/api/csv/importEntity.ts +++ b/app/api/csv/importEntity.ts @@ -1,4 +1,3 @@ -/* eslint-disable max-statements */ import entities from 'api/entities'; import { search } from 'api/search'; import entitiesModel from 'api/entities/entitiesModel'; From 6e0e9032d988d5f0790c6269c51cad957e37b902 Mon Sep 17 00:00:00 2001 From: Laszlo Kecskes Date: Fri, 24 Sep 2021 12:54:18 +0200 Subject: [PATCH 36/52] changing thesauri database query to single --- app/api/csv/importEntity.ts | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/app/api/csv/importEntity.ts b/app/api/csv/importEntity.ts index ede65a6114..e651442517 100644 --- a/app/api/csv/importEntity.ts +++ b/app/api/csv/importEntity.ts @@ -80,6 +80,7 @@ const filterJSObject = (input: { [k: string]: any }, keys: string[]): { [k: stri return result; }; +// eslint-disable-next-line max-statements const arrangeThesauri = async ( file: ImportFile, template: TemplateSchema, @@ -110,11 +111,11 @@ const arrangeThesauri = async ( } } }); - const allRelatedThesauri = await Promise.all( - Array.from( + const allRelatedThesauri = await thesauri.get({ + $in: Array.from( new Set(thesauriRelatedProperties?.map(p => p.content?.toString()).filter(t => t)) - ).map(async id => thesauri.getById(id)) - ); + ), + }); allRelatedThesauri.forEach(t => { if (t) { const id = t._id.toString(); From 23c12cd672ef6c817f79da2025d2f30b7ad7b224 Mon Sep 17 00:00:00 2001 From: Laszlo Kecskes Date: Fri, 24 Sep 2021 13:30:49 +0200 Subject: [PATCH 37/52] changed error handling on arrangeThesauri --- app/api/csv/csvLoader.ts | 18 ++++++++++++++++-- app/api/csv/importEntity.ts | 24 +++++++++++++++++------- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/app/api/csv/csvLoader.ts b/app/api/csv/csvLoader.ts index c03659688e..a9aa3f272f 100644 --- a/app/api/csv/csvLoader.ts +++ b/app/api/csv/csvLoader.ts @@ -13,7 +13,12 @@ import { ensure } from 'shared/tsUtils'; import { ObjectId } from 'mongodb'; import csv, { CSVRow } from './csv'; import importFile from './importFile'; -import { arrangeThesauri, importEntity, translateEntity } from './importEntity'; +import { + arrangeThesauri, + ArrangeThesauriError, + importEntity, + translateEntity, +} from './importEntity'; import { extractEntity, toSafeName } from './entityRow'; export class CSVLoader extends EventEmitter { @@ -56,7 +61,16 @@ export class CSVLoader extends EventEmitter { (await settings.get()).languages ).map((l: LanguageSchema) => l.key); const { newNameGeneration = false } = await settings.get(); - await arrangeThesauri(file, template, availableLanguages, this); + try { + await arrangeThesauri(file, template, availableLanguages, this.stopOnError); + } catch (e) { + if (e instanceof ArrangeThesauriError) { + const _e: ArrangeThesauriError = e; + this.emit('loadError', _e.source, toSafeName(_e.row), _e.index); + } else { + throw e; + } + } await csv(await file.readStream(), this.stopOnError) .onRow(async (row: CSVRow) => { diff --git a/app/api/csv/importEntity.ts b/app/api/csv/importEntity.ts index e651442517..5a34da5af1 100644 --- a/app/api/csv/importEntity.ts +++ b/app/api/csv/importEntity.ts @@ -80,12 +80,25 @@ const filterJSObject = (input: { [k: string]: any }, keys: string[]): { [k: stri return result; }; +class ArrangeThesauriError extends Error { + source: Error; + row: CSVRow; + index: number; + + constructor(source: Error, row: CSVRow, index: number) { + super(source.message); + this.source = source; + this.row = row; + this.index = index; + } +} + // eslint-disable-next-line max-statements const arrangeThesauri = async ( file: ImportFile, template: TemplateSchema, languages?: string[], - errorContext?: any + stopOnError: boolean = true ) => { let nameToThesauriIdSelects: { [k: string]: string } = {}; let nameToThesauriIdMultiselects: { [k: string]: string } = {}; @@ -137,7 +150,7 @@ const arrangeThesauri = async ( thesauriIdToNormalizedNewValues.get(id).add(normalized); } } - await csv(await file.readStream(), errorContext?.stopOnError) + await csv(await file.readStream(), stopOnError) .onRow(async (row: CSVRow, index: number) => { if (index === 0) { const columnnames = Object.keys(row); @@ -161,10 +174,7 @@ const arrangeThesauri = async ( }); }) .onError(async (e: Error, row: CSVRow, index: number) => { - if (errorContext) { - errorContext._errors[index] = e; - errorContext.emit('loadError', e, toSafeName(row), index); - } + throw new ArrangeThesauriError(e, row, index); }) .read(); for (let i = 0; i < allRelatedThesauri.length; i += 1) { @@ -241,4 +251,4 @@ const translateEntity = async ( await search.indexEntities({ sharedId: entity.sharedId }, '+fullText'); }; -export { arrangeThesauri, importEntity, translateEntity }; +export { arrangeThesauri, importEntity, translateEntity, ArrangeThesauriError }; From d5f82984425f1f62e05a2f360b4723c856ee1203 Mon Sep 17 00:00:00 2001 From: Kevin Date: Fri, 24 Sep 2021 15:29:36 +0300 Subject: [PATCH 38/52] Added e2e --- e2e/helpers/createEntity.ts | 46 ++++++++++ e2e/helpers/createTemplate.ts | 15 ++++ e2e/suites/convert-entity-template.test.ts | 98 ++++++---------------- 3 files changed, 87 insertions(+), 72 deletions(-) create mode 100644 e2e/helpers/createEntity.ts create mode 100644 e2e/helpers/createTemplate.ts diff --git a/e2e/helpers/createEntity.ts b/e2e/helpers/createEntity.ts new file mode 100644 index 0000000000..a7c8e63216 --- /dev/null +++ b/e2e/helpers/createEntity.ts @@ -0,0 +1,46 @@ +import { host } from 'e2e/config'; +import { ElementHandle } from 'puppeteer'; + +interface FilesOptions { + pdf?: string; + supportingFile?: string; +} + +const uploadPDFToEntity = async (pdfName: string) => { + await expect(page).toUploadFile('#upload-button-input', `${__dirname}/test_files/${pdfName}`); +}; + +const uploadSupportingFileToEntity = async (fileName: string): Promise => { + await expect(page).toClick('button[type="button"].upload-button'); + const [fileChooser] = await Promise.all([ + page.waitForFileChooser(), + page.click('div.attachments-modal__dropzone > button'), + ]); + await fileChooser.accept([`${__dirname}/test_files/${fileName}`]); +}; + +export const createEntity = async (templateName: string, files: FilesOptions) => { + await page.goto(`${host}`); + await expect(page).toClick('button', { text: 'Create entity' }); + await expect(page).toFill('textarea[name="library.sidepanel.metadata.title"]', templateName); + let options: ElementHandle[] = []; + options = await page.$$('select.form-control > option'); + + // @ts-ignore + options.forEach(async (option: ElementHandle): void => { + const value = await option.evaluate(optionEl => ({ + text: optionEl.textContent, + value: optionEl.getAttribute('value') as string, + })); + if (value.text === templateName) await page.select('select.form-control', value.value); + }); + await expect(page).toMatchElement('button[form="metadataForm"]', { text: 'Save' }); + await expect(page).toClick('button[form="metadataForm"]', { text: 'Save' }); + await expect(page).toClick('span', { text: 'Entity created' }); + + if (files) { + if (files.pdf) await uploadPDFToEntity(files.pdf); + if (files.supportingFile) await uploadSupportingFileToEntity(files.supportingFile); + await expect(page).toClick('span', { text: 'Attachment uploaded' }); + } +}; diff --git a/e2e/helpers/createTemplate.ts b/e2e/helpers/createTemplate.ts new file mode 100644 index 0000000000..44b63cb93f --- /dev/null +++ b/e2e/helpers/createTemplate.ts @@ -0,0 +1,15 @@ +import { host } from 'e2e/config'; + +export const createTemplate = async (name: string) => { + await page.goto(`${host}/en/settings/account`); + await expect(page).toClick('span', { text: 'Templates' }); + await expect(page).toClick('span', { text: 'Add template' }); + await expect(page).toFill('input[name="template.data.name"]', name); + if (name === 'With image') { + const imageElement = await page.$$('ul.property-options-list > .list-group-item'); + const button = await imageElement[11].$$('button'); + await button[0].click(); + } + await expect(page).toClick('button[type="submit"]'); + await expect(page).toClick('span', { text: 'Saved successfully.' }); +}; diff --git a/e2e/suites/convert-entity-template.test.ts b/e2e/suites/convert-entity-template.test.ts index b90992d7f9..266bfe3335 100644 --- a/e2e/suites/convert-entity-template.test.ts +++ b/e2e/suites/convert-entity-template.test.ts @@ -1,9 +1,10 @@ /*global page*/ import { ElementHandle } from 'puppeteer'; +import { createEntity } from 'e2e/helpers/createEntity'; +import { createTemplate } from 'e2e/helpers/createTemplate'; import { adminLogin, logout } from '../helpers/login'; import proxyMock from '../helpers/proxyMock'; -import { host } from '../config'; import insertFixtures from '../helpers/insertFixtures'; import disableTransitions from '../helpers/disableTransitions'; @@ -14,88 +15,41 @@ const setupPreFlights = async (): Promise => { await disableTransitions(); }; -const createTemplate = async (name: string) => { - await page.goto(`${host}/en/settings/account`); - await expect(page).toClick('span', { text: 'Templates' }); - await expect(page).toClick('span', { text: 'Add template' }); - await expect(page).toFill('input[name="template.data.name"]', name); - if (name === 'With image') { - const imageElement = await page.$$('ul.property-options-list > .list-group-item'); - const button = await imageElement[11].$$('button'); - await button[0].click(); - } - await expect(page).toClick('button[type="submit"]'); - await expect(page).toClick('span', { text: 'Saved successfully.' }); -}; - -const createEntity = async (templateName: string) => { - await page.goto(`${host}`); - await expect(page).toClick('button', { text: 'Create entity' }); - await expect(page).toFill('textarea[name="library.sidepanel.metadata.title"]', templateName); - let options: ElementHandle[] = []; - await page.$$('select.form-control > option').then(selects => { - options = selects; - }); - - // @ts-ignore - options.forEach(async (option: ElementHandle): void => { - const value = await option.evaluate(optionEl => ({ - text: optionEl.textContent, - value: optionEl.getAttribute('value') as string, - })); - if (value.text === templateName) { - await page.select('select.form-control', value.value); - } - }); - await expect(page).toMatchElement('button[form="metadataForm"]', { text: 'Save' }); - await expect(page).toClick('button[form="metadataForm"]', { text: 'Save' }); - await expect(page).toClick('span', { text: 'Entity created' }); -}; - -const uploadPDFToEntity = async (entityName: string) => { - await page.goto(`${host}`); - await expect(page).toClick('span', { text: 'Restricted' }); - await expect(page).toClick('div.item-name > span', { text: entityName }); - await expect(page).toUploadFile('#upload-button-input', `${__dirname}/test_files/valid.pdf`); -}; - -const UploadSupportingFileToEntity = async (entityName: string): Promise => { - await page.goto(`${host}`); - await expect(page).toClick('span', { text: 'Restricted' }); - await expect(page).toClick('div.item-name > span', { text: entityName }); - await expect(page).toClick('button[type="button"].upload-button'); - const [fileChooser] = await Promise.all([ - page.waitForFileChooser(), - page.click('div.attachments-modal__dropzone > button'), - ]); - await fileChooser.accept([`${__dirname}/test_files/batman.jpg`]); -}; - -const convertEntityTemplate = async (entityName: string, targetTemplate: string): Promise => { - await page.goto(`${host}`); - await expect(page).toClick('span', { text: 'Restricted' }); - await expect(page).toClick('div.item-name > span', { text: entityName }); - await expect(page).toClick('div.sidepanel-footer > button[type="button"].edit-metadata'); -}; - const setupTest = async () => { await createTemplate('Without image'); await createTemplate('With image'); - await createEntity('Without image'); - await uploadPDFToEntity('Without image'); - await UploadSupportingFileToEntity('Without image'); + await createEntity('Without image', { pdf: 'valid.pdf', supportingFile: 'batman.jpg' }); }; describe('Convert entity template', () => { beforeAll(async () => { await setupPreFlights(); - - await convertEntityTemplate('Without image', 'With image'); }); - it('Should create new entity', async () => { + it('Should select image for image property from supporting files', async () => { await setupTest(); - await expect(page).toClick('a[(type = "button")].btn'); + await expect(page).toClick('a[type="button"]'); + await page.reload(); + await expect(page).toClick('button.edit-metadata'); + await page.waitFor(1000); + const options = await page.$$('select.form-control > option'); + + const optionsValues = await options.map(async (optionElement: ElementHandle) => { + console.log('option'); + const value = await optionElement.evaluate(optionEl => ({ + text: optionEl.textContent, + value: optionEl.getAttribute('value') as string, + })); + return value; + }); + + const optionValues: any[] = await Promise.all(optionsValues); + const value = optionValues.find(option => option.text === 'With image').value; + + await page.select('select.form-control', value); + await expect(page).toClick('span', { text: 'Select supporting file' }); + await page.waitFor(10000); + await expect(page).toMatchElement('div.media-grid-card-header > h5', { text: 'batman.jpg' }); }); afterAll(async () => { From c5f5b1ffed0a4390aabaf6257808406f999537a0 Mon Sep 17 00:00:00 2001 From: Laszlo Kecskes Date: Fri, 24 Sep 2021 14:33:17 +0200 Subject: [PATCH 39/52] refactored arrangeThesauri --- app/api/csv/arrangeThesauri.ts | 180 +++++++++++++++++++++++++ app/api/csv/csvLoader.ts | 8 +- app/api/csv/importEntity.ts | 133 +----------------- app/api/csv/specs/importEntity.spec.js | 2 +- 4 files changed, 185 insertions(+), 138 deletions(-) create mode 100644 app/api/csv/arrangeThesauri.ts diff --git a/app/api/csv/arrangeThesauri.ts b/app/api/csv/arrangeThesauri.ts new file mode 100644 index 0000000000..8842e33759 --- /dev/null +++ b/app/api/csv/arrangeThesauri.ts @@ -0,0 +1,180 @@ +import { ImportFile } from 'api/csv/importFile'; +import thesauri from 'api/thesauri'; +import { propertyTypes } from 'shared/propertyTypes'; +import { PropertySchema } from 'shared/types/commonTypes'; +import { TemplateSchema } from 'shared/types/templateType'; +import { ThesaurusSchema } from 'shared/types/thesaurusType'; + +import csv, { CSVRow } from './csv'; +import { splitMultiselectLabels } from './typeParsers/multiselect'; +import { normalizeThesaurusLabel } from './typeParsers/select'; + +const filterJSObject = (input: { [k: string]: any }, keys: string[]): { [k: string]: any } => { + const result: { [k: string]: any } = {}; + keys.forEach(k => { + if (input.hasOwnProperty(k)) { + result[k] = input[k]; + } + }); + return result; +}; + +class ArrangeThesauriError extends Error { + source: Error; + row: CSVRow; + index: number; + + constructor(source: Error, row: CSVRow, index: number) { + super(source.message); + this.source = source; + this.row = row; + this.index = index; + } +} + +const separateSelectAndMultiselectThesauri = ( + thesauriRelatedProperties: PropertySchema[] | undefined, + languages?: string[] +): [{ [k: string]: string }, { [k: string]: string }] => { + const nameToThesauriIdSelects: { [k: string]: string } = {}; + const nameToThesauriIdMultiselects: { [k: string]: string } = {}; + + thesauriRelatedProperties?.forEach(p => { + if (p.content && p.type) { + const thesarusID = p.content.toString(); + if (p.type === propertyTypes.select) { + nameToThesauriIdSelects[p.name] = thesarusID; + languages?.forEach(suffix => { + nameToThesauriIdSelects[`${p.name}__${suffix}`] = thesarusID; + }); + } else if (p.type === propertyTypes.multiselect) { + nameToThesauriIdMultiselects[p.name] = thesarusID; + languages?.forEach(suffix => { + nameToThesauriIdMultiselects[`${p.name}__${suffix}`] = thesarusID; + }); + } + } + }); + + return [nameToThesauriIdSelects, nameToThesauriIdMultiselects]; +}; + +type ThesauriValueData = { + thesauriIdToExistingValues: Map>; + thesauriIdToNewValues: Map>; + thesauriIdToNormalizedNewValues: Map>; +}; + +const setupIdValueMaps = (allRelatedThesauri: ThesaurusSchema[]): ThesauriValueData => { + const thesauriIdToExistingValues = new Map(); + const thesauriIdToNewValues = new Map(); + const thesauriIdToNormalizedNewValues = new Map(); + + allRelatedThesauri.forEach(t => { + if (t._id) { + const id = t._id.toString(); + thesauriIdToExistingValues.set( + id, + new Set(t.values?.map(v => normalizeThesaurusLabel(v.label))) + ); + thesauriIdToNewValues.set(id, new Set()); + thesauriIdToNormalizedNewValues.set(id, new Set()); + } + }); + + return { thesauriIdToExistingValues, thesauriIdToNewValues, thesauriIdToNormalizedNewValues }; +}; + +const handleLabels = ( + id: string, + original: string, + normalized: string | null, + thesauriValueData: ThesauriValueData +) => { + if ( + normalized && + !thesauriValueData.thesauriIdToExistingValues.get(id)?.has(normalized) && + !thesauriValueData.thesauriIdToNormalizedNewValues.get(id)?.has(normalized) + ) { + thesauriValueData.thesauriIdToNewValues.get(id)?.add(original); + thesauriValueData.thesauriIdToNormalizedNewValues.get(id)?.add(normalized); + } +}; + +const syncSaveThesauri = async ( + allRelatedThesauri: ThesaurusSchema[], + thesauriIdToNewValues: Map> +) => { + for (let i = 0; i < allRelatedThesauri.length; i += 1) { + const thesaurus = allRelatedThesauri[i]; + if (thesaurus?._id) { + const newValues: { label: string }[] = Array.from( + thesauriIdToNewValues.get(thesaurus._id.toString()) || [] + ).map(tval => ({ label: tval })); + if (newValues.length > 0) { + const thesaurusValues = thesaurus.values || []; + // eslint-disable-next-line no-await-in-loop + await thesauri.save({ + ...thesaurus, + values: thesaurusValues.concat(newValues), + }); + } + } + } +}; + +const arrangeThesauri = async ( + file: ImportFile, + template: TemplateSchema, + languages?: string[], + stopOnError: boolean = true +) => { + const thesauriRelatedProperties = template.properties?.filter(p => + ['select', 'multiselect'].includes(p.type) + ); + + let [ + nameToThesauriIdSelects, + nameToThesauriIdMultiselects, + ] = separateSelectAndMultiselectThesauri(thesauriRelatedProperties, languages); + + const allRelatedThesauri = await thesauri.get({ + $in: Array.from( + new Set(thesauriRelatedProperties?.map(p => p.content?.toString()).filter(t => t)) + ), + }); + + const thesauriValueData = setupIdValueMaps(allRelatedThesauri); + + await csv(await file.readStream(), stopOnError) + .onRow(async (row: CSVRow, index: number) => { + if (index === 0) { + const columnnames = Object.keys(row); + nameToThesauriIdSelects = filterJSObject(nameToThesauriIdSelects, columnnames); + nameToThesauriIdMultiselects = filterJSObject(nameToThesauriIdMultiselects, columnnames); + } + Object.entries(nameToThesauriIdSelects).forEach(([name, id]) => { + const label = row[name]; + if (label) { + const normalizedLabel = normalizeThesaurusLabel(label); + handleLabels(id, label, normalizedLabel, thesauriValueData); + } + }); + Object.entries(nameToThesauriIdMultiselects).forEach(([name, id]) => { + const labels = splitMultiselectLabels(row[name]); + if (labels) { + Object.entries(labels).forEach(([normalizedLabel, originalLabel]) => { + handleLabels(id, originalLabel, normalizedLabel, thesauriValueData); + }); + } + }); + }) + .onError(async (e: Error, row: CSVRow, index: number) => { + throw new ArrangeThesauriError(e, row, index); + }) + .read(); + + await syncSaveThesauri(allRelatedThesauri, thesauriValueData.thesauriIdToNewValues); +}; + +export { arrangeThesauri, ArrangeThesauriError }; diff --git a/app/api/csv/csvLoader.ts b/app/api/csv/csvLoader.ts index a9aa3f272f..20ba786191 100644 --- a/app/api/csv/csvLoader.ts +++ b/app/api/csv/csvLoader.ts @@ -11,14 +11,10 @@ import { ThesaurusSchema } from 'shared/types/thesaurusType'; import { ensure } from 'shared/tsUtils'; import { ObjectId } from 'mongodb'; +import { arrangeThesauri, ArrangeThesauriError } from './arrangeThesauri'; import csv, { CSVRow } from './csv'; import importFile from './importFile'; -import { - arrangeThesauri, - ArrangeThesauriError, - importEntity, - translateEntity, -} from './importEntity'; +import { importEntity, translateEntity } from './importEntity'; import { extractEntity, toSafeName } from './entityRow'; export class CSVLoader extends EventEmitter { diff --git a/app/api/csv/importEntity.ts b/app/api/csv/importEntity.ts index 5a34da5af1..3a058c3ca8 100644 --- a/app/api/csv/importEntity.ts +++ b/app/api/csv/importEntity.ts @@ -2,21 +2,17 @@ import entities from 'api/entities'; import { search } from 'api/search'; import entitiesModel from 'api/entities/entitiesModel'; import { processDocument } from 'api/files/processDocument'; -import { RawEntity, toSafeName } from 'api/csv/entityRow'; +import { RawEntity } from 'api/csv/entityRow'; import { TemplateSchema } from 'shared/types/templateType'; import { MetadataSchema, PropertySchema } from 'shared/types/commonTypes'; import { propertyTypes } from 'shared/propertyTypes'; import { ImportFile } from 'api/csv/importFile'; -import thesauri from 'api/thesauri'; import { EntitySchema } from 'shared/types/entityType'; import { ensure } from 'shared/tsUtils'; import { attachmentsPath, files } from 'api/files'; import { generateID } from 'shared/IDGenerator'; -import { normalizeThesaurusLabel } from './typeParsers/select'; -import { splitMultiselectLabels } from './typeParsers/multiselect'; import typeParsers from './typeParsers'; -import csv, { CSVRow } from './csv'; const parse = async (toImportEntity: RawEntity, prop: PropertySchema) => typeParsers[prop.type] @@ -70,131 +66,6 @@ type Options = { language: string; }; -const filterJSObject = (input: { [k: string]: any }, keys: string[]): { [k: string]: any } => { - const result: { [k: string]: any } = {}; - keys.forEach(k => { - if (input.hasOwnProperty(k)) { - result[k] = input[k]; - } - }); - return result; -}; - -class ArrangeThesauriError extends Error { - source: Error; - row: CSVRow; - index: number; - - constructor(source: Error, row: CSVRow, index: number) { - super(source.message); - this.source = source; - this.row = row; - this.index = index; - } -} - -// eslint-disable-next-line max-statements -const arrangeThesauri = async ( - file: ImportFile, - template: TemplateSchema, - languages?: string[], - stopOnError: boolean = true -) => { - let nameToThesauriIdSelects: { [k: string]: string } = {}; - let nameToThesauriIdMultiselects: { [k: string]: string } = {}; - const thesauriIdToExistingValues = new Map(); - const thesauriIdToNewValues: Map> = new Map(); - const thesauriIdToNormalizedNewValues = new Map(); - const thesauriRelatedProperties = template.properties?.filter(p => - ['select', 'multiselect'].includes(p.type) - ); - thesauriRelatedProperties?.forEach(p => { - if (p.content && p.type) { - const thesarusID = p.content.toString(); - if (p.type === propertyTypes.select) { - nameToThesauriIdSelects[p.name] = thesarusID; - languages?.forEach(suffix => { - nameToThesauriIdSelects[`${p.name}__${suffix}`] = thesarusID; - }); - } else if (p.type === propertyTypes.multiselect) { - nameToThesauriIdMultiselects[p.name] = thesarusID; - languages?.forEach(suffix => { - nameToThesauriIdMultiselects[`${p.name}__${suffix}`] = thesarusID; - }); - } - } - }); - const allRelatedThesauri = await thesauri.get({ - $in: Array.from( - new Set(thesauriRelatedProperties?.map(p => p.content?.toString()).filter(t => t)) - ), - }); - allRelatedThesauri.forEach(t => { - if (t) { - const id = t._id.toString(); - thesauriIdToExistingValues.set( - id, - new Set(t.values?.map(v => normalizeThesaurusLabel(v.label))) - ); - thesauriIdToNewValues.set(id, new Set()); - thesauriIdToNormalizedNewValues.set(id, new Set()); - } - }); - function handleLabels(id: string, original: string, normalized: string | null) { - if ( - normalized && - !thesauriIdToExistingValues.get(id).has(normalized) && - !thesauriIdToNormalizedNewValues.get(id).has(normalized) - ) { - thesauriIdToNewValues.get(id)?.add(original); - thesauriIdToNormalizedNewValues.get(id).add(normalized); - } - } - await csv(await file.readStream(), stopOnError) - .onRow(async (row: CSVRow, index: number) => { - if (index === 0) { - const columnnames = Object.keys(row); - nameToThesauriIdSelects = filterJSObject(nameToThesauriIdSelects, columnnames); - nameToThesauriIdMultiselects = filterJSObject(nameToThesauriIdMultiselects, columnnames); - } - Object.entries(nameToThesauriIdSelects).forEach(([name, id]) => { - const label = row[name]; - if (label) { - const normalizedLabel = normalizeThesaurusLabel(label); - handleLabels(id, label, normalizedLabel); - } - }); - Object.entries(nameToThesauriIdMultiselects).forEach(([name, id]) => { - const labels = splitMultiselectLabels(row[name]); - if (labels) { - Object.entries(labels).forEach(([normalizedLabel, originalLabel]) => { - handleLabels(id, originalLabel, normalizedLabel); - }); - } - }); - }) - .onError(async (e: Error, row: CSVRow, index: number) => { - throw new ArrangeThesauriError(e, row, index); - }) - .read(); - for (let i = 0; i < allRelatedThesauri.length; i += 1) { - const thesaurus = allRelatedThesauri[i]; - if (thesaurus !== null) { - const newValues: { label: string }[] = Array.from( - thesauriIdToNewValues.get(thesaurus._id.toString()) || [] - ).map(tval => ({ label: tval })); - if (newValues.length > 0) { - const thesaurusValues = thesaurus.values || []; - // eslint-disable-next-line no-await-in-loop - await thesauri.save({ - ...thesaurus, - values: thesaurusValues.concat(newValues), - }); - } - } - } -}; - const importEntity = async ( toImportEntity: RawEntity, template: TemplateSchema, @@ -251,4 +122,4 @@ const translateEntity = async ( await search.indexEntities({ sharedId: entity.sharedId }, '+fullText'); }; -export { arrangeThesauri, importEntity, translateEntity, ArrangeThesauriError }; +export { importEntity, translateEntity }; diff --git a/app/api/csv/specs/importEntity.spec.js b/app/api/csv/specs/importEntity.spec.js index d735a54f10..40c25ac6a5 100644 --- a/app/api/csv/specs/importEntity.spec.js +++ b/app/api/csv/specs/importEntity.spec.js @@ -6,7 +6,7 @@ import settings from 'api/settings'; import { getFixturesFactory } from 'api/utils/fixturesFactory'; import db from 'api/utils/testing_db'; -import { arrangeThesauri } from '../importEntity'; +import { arrangeThesauri } from '../arrangeThesauri'; import importFile from '../importFile'; import { mockCsvFileReadStream } from './helpers'; From 8293a6cc79a09e86a18bcf66ef2ae0f2ff0d8f2f Mon Sep 17 00:00:00 2001 From: Kevin Date: Fri, 24 Sep 2021 15:39:04 +0300 Subject: [PATCH 40/52] Fixed errors in e2e --- e2e/helpers/createEntity.ts | 6 +++--- e2e/helpers/createTemplate.ts | 2 +- e2e/suites/convert-entity-template.test.ts | 14 ++++++++------ 3 files changed, 12 insertions(+), 10 deletions(-) diff --git a/e2e/helpers/createEntity.ts b/e2e/helpers/createEntity.ts index a7c8e63216..ff9f0bc20f 100644 --- a/e2e/helpers/createEntity.ts +++ b/e2e/helpers/createEntity.ts @@ -1,4 +1,4 @@ -import { host } from 'e2e/config'; +import { host } from '../config'; import { ElementHandle } from 'puppeteer'; interface FilesOptions { @@ -7,7 +7,7 @@ interface FilesOptions { } const uploadPDFToEntity = async (pdfName: string) => { - await expect(page).toUploadFile('#upload-button-input', `${__dirname}/test_files/${pdfName}`); + await expect(page).toUploadFile('#upload-button-input', pdfName); }; const uploadSupportingFileToEntity = async (fileName: string): Promise => { @@ -16,7 +16,7 @@ const uploadSupportingFileToEntity = async (fileName: string): Promise => page.waitForFileChooser(), page.click('div.attachments-modal__dropzone > button'), ]); - await fileChooser.accept([`${__dirname}/test_files/${fileName}`]); + await fileChooser.accept([fileName]); }; export const createEntity = async (templateName: string, files: FilesOptions) => { diff --git a/e2e/helpers/createTemplate.ts b/e2e/helpers/createTemplate.ts index 44b63cb93f..5b3616ed38 100644 --- a/e2e/helpers/createTemplate.ts +++ b/e2e/helpers/createTemplate.ts @@ -1,4 +1,4 @@ -import { host } from 'e2e/config'; +import { host } from '../config'; export const createTemplate = async (name: string) => { await page.goto(`${host}/en/settings/account`); diff --git a/e2e/suites/convert-entity-template.test.ts b/e2e/suites/convert-entity-template.test.ts index 266bfe3335..2c0996bc26 100644 --- a/e2e/suites/convert-entity-template.test.ts +++ b/e2e/suites/convert-entity-template.test.ts @@ -1,8 +1,8 @@ /*global page*/ import { ElementHandle } from 'puppeteer'; -import { createEntity } from 'e2e/helpers/createEntity'; -import { createTemplate } from 'e2e/helpers/createTemplate'; +import { createEntity } from '../helpers/createEntity'; +import { createTemplate } from '../helpers/createTemplate'; import { adminLogin, logout } from '../helpers/login'; import proxyMock from '../helpers/proxyMock'; import insertFixtures from '../helpers/insertFixtures'; @@ -18,7 +18,10 @@ const setupPreFlights = async (): Promise => { const setupTest = async () => { await createTemplate('Without image'); await createTemplate('With image'); - await createEntity('Without image', { pdf: 'valid.pdf', supportingFile: 'batman.jpg' }); + await createEntity('Without image', { + pdf: `${__dirname}/test_files/valid.pdf`, + supportingFile: `${__dirname}/test_files/batman.jpg`, + }); }; describe('Convert entity template', () => { @@ -31,11 +34,11 @@ describe('Convert entity template', () => { await expect(page).toClick('a[type="button"]'); await page.reload(); await expect(page).toClick('button.edit-metadata'); - await page.waitFor(1000); + await expect(page).toMatchElement('select.form-control > option'); + // await page.waitFor(1000); const options = await page.$$('select.form-control > option'); const optionsValues = await options.map(async (optionElement: ElementHandle) => { - console.log('option'); const value = await optionElement.evaluate(optionEl => ({ text: optionEl.textContent, value: optionEl.getAttribute('value') as string, @@ -48,7 +51,6 @@ describe('Convert entity template', () => { await page.select('select.form-control', value); await expect(page).toClick('span', { text: 'Select supporting file' }); - await page.waitFor(10000); await expect(page).toMatchElement('div.media-grid-card-header > h5', { text: 'batman.jpg' }); }); From 861f214299efbb3582c13ea5967c499166cb9457 Mon Sep 17 00:00:00 2001 From: Kevin Date: Fri, 24 Sep 2021 16:01:12 +0300 Subject: [PATCH 41/52] removed comment --- e2e/suites/convert-entity-template.test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/e2e/suites/convert-entity-template.test.ts b/e2e/suites/convert-entity-template.test.ts index 2c0996bc26..a5f7ab874d 100644 --- a/e2e/suites/convert-entity-template.test.ts +++ b/e2e/suites/convert-entity-template.test.ts @@ -35,7 +35,6 @@ describe('Convert entity template', () => { await page.reload(); await expect(page).toClick('button.edit-metadata'); await expect(page).toMatchElement('select.form-control > option'); - // await page.waitFor(1000); const options = await page.$$('select.form-control > option'); const optionsValues = await options.map(async (optionElement: ElementHandle) => { From ef998dab9201fd7e0b5228769a2a0d7dd40e6dae Mon Sep 17 00:00:00 2001 From: Laszlo Kecskes Date: Fri, 24 Sep 2021 15:23:15 +0200 Subject: [PATCH 42/52] updated tests --- app/api/csv/csvLoader.ts | 1 - app/api/csv/specs/csvLoader.spec.js | 16 --- ...ntity.spec.js => csvLoaderSelects.spec.js} | 121 +++++++++++++----- 3 files changed, 90 insertions(+), 48 deletions(-) rename app/api/csv/specs/{importEntity.spec.js => csvLoaderSelects.spec.js} (59%) diff --git a/app/api/csv/csvLoader.ts b/app/api/csv/csvLoader.ts index 20ba786191..6fadac6fca 100644 --- a/app/api/csv/csvLoader.ts +++ b/app/api/csv/csvLoader.ts @@ -1,4 +1,3 @@ -/* eslint-disable max-statements */ import { EventEmitter } from 'events'; import templates from 'api/templates'; diff --git a/app/api/csv/specs/csvLoader.spec.js b/app/api/csv/specs/csvLoader.spec.js index 7a800e2410..10243947ca 100644 --- a/app/api/csv/specs/csvLoader.spec.js +++ b/app/api/csv/specs/csvLoader.spec.js @@ -169,22 +169,6 @@ describe('csvLoader', () => { expect(textValues.length).toEqual(0); }); - it('should arrange translations for selects and multiselects', async () => { - const trs = await translations.get(); - trs.forEach(tr => { - expect(tr.contexts.find(c => c.label === 'thesauri1').values).toMatchObject({ - thesauri1: 'thesauri1', - thesauri2: 'thesauri2', - }); - expect(tr.contexts.find(c => c.label === 'multi_select_thesaurus').values).toMatchObject({ - multi_select_thesaurus: 'multi_select_thesaurus', - multivalue1: 'multivalue1', - multivalue2: 'multivalue2', - multivalue3: 'multivalue3', - }); - }); - }); - describe('metadata parsing', () => { it('should parse metadata properties by type using typeParsers', () => { const textValues = imported.map(i => i.metadata.text_label[0].value); diff --git a/app/api/csv/specs/importEntity.spec.js b/app/api/csv/specs/csvLoaderSelects.spec.js similarity index 59% rename from app/api/csv/specs/importEntity.spec.js rename to app/api/csv/specs/csvLoaderSelects.spec.js index 40c25ac6a5..89725502a2 100644 --- a/app/api/csv/specs/importEntity.spec.js +++ b/app/api/csv/specs/csvLoaderSelects.spec.js @@ -1,17 +1,45 @@ import path from 'path'; -import templates from 'api/templates'; +import translations from 'api/i18n/translations'; import thesauri from 'api/thesauri'; -import settings from 'api/settings'; import { getFixturesFactory } from 'api/utils/fixturesFactory'; import db from 'api/utils/testing_db'; -import { arrangeThesauri } from '../arrangeThesauri'; -import importFile from '../importFile'; -import { mockCsvFileReadStream } from './helpers'; +import { CSVLoader } from '../csvLoader'; const fixtureFactory = getFixturesFactory(); +const commonTranslationContexts = (id1, id2) => [ + { + id: 'System', + label: 'System', + values: [ + { key: 'original 1', value: 'original 1' }, + { key: 'original 2', value: 'original 2' }, + { key: 'original 3', value: 'original 3' }, + ], + }, + { + id: id1.toString(), + label: 'select_thesaurus', + values: [ + { key: 'select_thesaurus', value: 'select_thesaurus' }, + { key: 'A', value: 'A' }, + ], + type: 'Dictionary', + }, + { + id: id2.toString(), + label: 'multiselect_thesaurus', + values: [ + { key: 'multiselect_thesaurus', value: 'multiselect_thesaurus' }, + { key: 'A', value: 'A' }, + { key: 'B', value: 'B' }, + ], + type: 'Dictionary', + }, +]; + const fixtures = { dictionaries: [ fixtureFactory.thesauri('select_thesaurus', ['A']), @@ -48,12 +76,30 @@ const fixtures = { ], }, ], + translations: [ + { + _id: db.id(), + locale: 'en', + contexts: commonTranslationContexts( + fixtureFactory.id('select_thesaurus'), + fixtureFactory.id('multiselect_thesaurus') + ), + }, + { + _id: db.id(), + locale: 'es', + contexts: commonTranslationContexts( + fixtureFactory.id('select_thesaurus'), + fixtureFactory.id('multiselect_thesaurus') + ), + }, + ], }; -describe('arrangeThesauri', () => { - let file; +const loader = new CSVLoader(); + +describe('loader', () => { let fileSpy; - let template; let selectThesaurus; let selectLabels; let selectLabelsSet; @@ -63,10 +109,10 @@ describe('arrangeThesauri', () => { beforeAll(async () => { await db.clearAllAndLoad(fixtures); - template = await templates.getById(fixtureFactory.id('template')); - file = importFile(path.join(__dirname, '/arrangeThesauriTest.csv')); - const languages = (await settings.get()).languages.map(l => l.key); - await arrangeThesauri(file, template, languages); + await loader.load( + path.join(__dirname, '/arrangeThesauriTest.csv'), + fixtureFactory.id('template') + ); selectThesaurus = await thesauri.getById(fixtureFactory.id('select_thesaurus')); selectLabels = selectThesaurus.values.map(tv => tv.label); selectLabelsSet = new Set(selectLabels); @@ -79,25 +125,6 @@ describe('arrangeThesauri', () => { fileSpy.mockRestore(); }); - it('should not fail on templates with no select or multiselect fields', async () => { - const noselTemplate = templates.getById(fixtureFactory.id('no_selects_template')); - const csv = `title,unrelated_text -first,first -second,second`; - const readStreamMock = mockCsvFileReadStream(csv); - await arrangeThesauri(importFile('mockedFile'), noselTemplate); - readStreamMock.mockRestore(); - }); - - it('should not fail if the select or multiselect fields are missing from the csv', async () => { - const csv = `title,unrelated_property -first,first -second,second`; - const readStreamMock = mockCsvFileReadStream(csv); - await arrangeThesauri(importFile('mockedFile'), template); - readStreamMock.mockRestore(); - }); - it('should create values in thesauri', async () => { expect(selectLabels).toEqual(['A', 'B', 'Bes', 'C', 'Ces', 'd', 'des', 'Aes']); expect(multiselectLabels).toEqual([ @@ -138,4 +165,36 @@ second,second`; expect(selectLabels.length).toBe(selectLabelsSet.size); expect(multiselectLabels.length).toBe(multiselectLabelsSet.size); }); + + it('should arrange translations for selects and multiselects', async () => { + const trs = await translations.get(); + trs.forEach(tr => { + expect(tr.contexts.find(c => c.label === 'select_thesaurus').values).toMatchObject({ + A: 'A', + Aes: 'Aes', + B: 'B', + Bes: 'Bes', + C: 'C', + Ces: 'Ces', + d: 'd', + des: 'des', + select_thesaurus: 'select_thesaurus', + }); + expect(tr.contexts.find(c => c.label === 'multiselect_thesaurus').values).toMatchObject({ + A: 'A', + Aes: 'Aes', + B: 'B', + Bes: 'Bes', + D: 'D', + Des: 'Des', + E: 'E', + Ees: 'Ees', + c: 'c', + ces: 'ces', + g: 'g', + ges: 'ges', + multiselect_thesaurus: 'multiselect_thesaurus', + }); + }); + }); }); From b42630d4460833063570a458b4a7f289fa929c3e Mon Sep 17 00:00:00 2001 From: Laszlo Kecskes Date: Fri, 24 Sep 2021 17:05:39 +0200 Subject: [PATCH 43/52] changing a test description --- app/api/csv/specs/csvLoaderSelects.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/api/csv/specs/csvLoaderSelects.spec.js b/app/api/csv/specs/csvLoaderSelects.spec.js index 89725502a2..f7629fb247 100644 --- a/app/api/csv/specs/csvLoaderSelects.spec.js +++ b/app/api/csv/specs/csvLoaderSelects.spec.js @@ -166,7 +166,7 @@ describe('loader', () => { expect(multiselectLabels.length).toBe(multiselectLabelsSet.size); }); - it('should arrange translations for selects and multiselects', async () => { + it('should check that the thesauri saving saves all contexts properly', async () => { const trs = await translations.get(); trs.forEach(tr => { expect(tr.contexts.find(c => c.label === 'select_thesaurus').values).toMatchObject({ From 1cb517a0f2ed280527ec79480ea42dea1c47b783 Mon Sep 17 00:00:00 2001 From: Kevin Date: Mon, 27 Sep 2021 15:30:01 +0300 Subject: [PATCH 44/52] Dissabled transitions when reloading --- e2e/suites/convert-entity-template.test.ts | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/e2e/suites/convert-entity-template.test.ts b/e2e/suites/convert-entity-template.test.ts index a5f7ab874d..c6c39ccaaf 100644 --- a/e2e/suites/convert-entity-template.test.ts +++ b/e2e/suites/convert-entity-template.test.ts @@ -12,7 +12,6 @@ const setupPreFlights = async (): Promise => { await insertFixtures(); await proxyMock(); await adminLogin(); - await disableTransitions(); }; const setupTest = async () => { @@ -27,13 +26,16 @@ const setupTest = async () => { describe('Convert entity template', () => { beforeAll(async () => { await setupPreFlights(); + await setupTest(); + await disableTransitions(); }); it('Should select image for image property from supporting files', async () => { - await setupTest(); await expect(page).toClick('a[type="button"]'); - await page.reload(); - await expect(page).toClick('button.edit-metadata'); + + await expect(page).toClick('.metadata-sidepanel button.edit-metadata', { + text: 'Edit', + }); await expect(page).toMatchElement('select.form-control > option'); const options = await page.$$('select.form-control > option'); @@ -46,8 +48,9 @@ describe('Convert entity template', () => { }); const optionValues: any[] = await Promise.all(optionsValues); - const value = optionValues.find(option => option.text === 'With image').value; + const { value } = optionValues.find(option => option.text === 'With image'); + await expect(page).toMatchElement('select.form-control'); await page.select('select.form-control', value); await expect(page).toClick('span', { text: 'Select supporting file' }); await expect(page).toMatchElement('div.media-grid-card-header > h5', { text: 'batman.jpg' }); From 4cae86286f29161f6ff94e6a0673de8bdf72b769 Mon Sep 17 00:00:00 2001 From: Laszlo Kecskes Date: Tue, 28 Sep 2021 11:40:31 +0200 Subject: [PATCH 45/52] removed select-multiselect differentiation --- app/api/csv/arrangeThesauri.ts | 73 ++++++++++------------------------ 1 file changed, 21 insertions(+), 52 deletions(-) diff --git a/app/api/csv/arrangeThesauri.ts b/app/api/csv/arrangeThesauri.ts index 8842e33759..c91c8d356e 100644 --- a/app/api/csv/arrangeThesauri.ts +++ b/app/api/csv/arrangeThesauri.ts @@ -1,6 +1,5 @@ import { ImportFile } from 'api/csv/importFile'; import thesauri from 'api/thesauri'; -import { propertyTypes } from 'shared/propertyTypes'; import { PropertySchema } from 'shared/types/commonTypes'; import { TemplateSchema } from 'shared/types/templateType'; import { ThesaurusSchema } from 'shared/types/thesaurusType'; @@ -32,31 +31,23 @@ class ArrangeThesauriError extends Error { } } -const separateSelectAndMultiselectThesauri = ( +const createNameToIdMap = ( thesauriRelatedProperties: PropertySchema[] | undefined, languages?: string[] -): [{ [k: string]: string }, { [k: string]: string }] => { - const nameToThesauriIdSelects: { [k: string]: string } = {}; - const nameToThesauriIdMultiselects: { [k: string]: string } = {}; +): { [k: string]: string } => { + const nameToThesauriId: { [k: string]: string } = {}; thesauriRelatedProperties?.forEach(p => { if (p.content && p.type) { const thesarusID = p.content.toString(); - if (p.type === propertyTypes.select) { - nameToThesauriIdSelects[p.name] = thesarusID; - languages?.forEach(suffix => { - nameToThesauriIdSelects[`${p.name}__${suffix}`] = thesarusID; - }); - } else if (p.type === propertyTypes.multiselect) { - nameToThesauriIdMultiselects[p.name] = thesarusID; - languages?.forEach(suffix => { - nameToThesauriIdMultiselects[`${p.name}__${suffix}`] = thesarusID; - }); - } + nameToThesauriId[p.name] = thesarusID; + languages?.forEach(suffix => { + nameToThesauriId[`${p.name}__${suffix}`] = thesarusID; + }); } }); - return [nameToThesauriIdSelects, nameToThesauriIdMultiselects]; + return nameToThesauriId; }; type ThesauriValueData = { @@ -85,22 +76,6 @@ const setupIdValueMaps = (allRelatedThesauri: ThesaurusSchema[]): ThesauriValueD return { thesauriIdToExistingValues, thesauriIdToNewValues, thesauriIdToNormalizedNewValues }; }; -const handleLabels = ( - id: string, - original: string, - normalized: string | null, - thesauriValueData: ThesauriValueData -) => { - if ( - normalized && - !thesauriValueData.thesauriIdToExistingValues.get(id)?.has(normalized) && - !thesauriValueData.thesauriIdToNormalizedNewValues.get(id)?.has(normalized) - ) { - thesauriValueData.thesauriIdToNewValues.get(id)?.add(original); - thesauriValueData.thesauriIdToNormalizedNewValues.get(id)?.add(normalized); - } -}; - const syncSaveThesauri = async ( allRelatedThesauri: ThesaurusSchema[], thesauriIdToNewValues: Map> @@ -133,10 +108,7 @@ const arrangeThesauri = async ( ['select', 'multiselect'].includes(p.type) ); - let [ - nameToThesauriIdSelects, - nameToThesauriIdMultiselects, - ] = separateSelectAndMultiselectThesauri(thesauriRelatedProperties, languages); + let nameToThesauriId = createNameToIdMap(thesauriRelatedProperties, languages); const allRelatedThesauri = await thesauri.get({ $in: Array.from( @@ -150,23 +122,20 @@ const arrangeThesauri = async ( .onRow(async (row: CSVRow, index: number) => { if (index === 0) { const columnnames = Object.keys(row); - nameToThesauriIdSelects = filterJSObject(nameToThesauriIdSelects, columnnames); - nameToThesauriIdMultiselects = filterJSObject(nameToThesauriIdMultiselects, columnnames); + nameToThesauriId = filterJSObject(nameToThesauriId, columnnames); } - Object.entries(nameToThesauriIdSelects).forEach(([name, id]) => { - const label = row[name]; - if (label) { - const normalizedLabel = normalizeThesaurusLabel(label); - handleLabels(id, label, normalizedLabel, thesauriValueData); - } - }); - Object.entries(nameToThesauriIdMultiselects).forEach(([name, id]) => { + Object.entries(nameToThesauriId).forEach(([name, id]) => { const labels = splitMultiselectLabels(row[name]); - if (labels) { - Object.entries(labels).forEach(([normalizedLabel, originalLabel]) => { - handleLabels(id, originalLabel, normalizedLabel, thesauriValueData); - }); - } + Object.entries(labels).forEach(([normalizedLabel, originalLabel]) => { + if ( + normalizedLabel && + !thesauriValueData.thesauriIdToExistingValues.get(id)?.has(normalizedLabel) && + !thesauriValueData.thesauriIdToNormalizedNewValues.get(id)?.has(normalizedLabel) + ) { + thesauriValueData.thesauriIdToNewValues.get(id)?.add(originalLabel); + thesauriValueData.thesauriIdToNormalizedNewValues.get(id)?.add(normalizedLabel); + } + }); }); }) .onError(async (e: Error, row: CSVRow, index: number) => { From c9a8e7d43f3970b0406c98ae8849746ad78d3071 Mon Sep 17 00:00:00 2001 From: Kevin Date: Tue, 28 Sep 2021 13:19:15 +0300 Subject: [PATCH 46/52] Changedtest description --- e2e/suites/convert-entity-template.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/suites/convert-entity-template.test.ts b/e2e/suites/convert-entity-template.test.ts index c6c39ccaaf..810feb1824 100644 --- a/e2e/suites/convert-entity-template.test.ts +++ b/e2e/suites/convert-entity-template.test.ts @@ -23,7 +23,7 @@ const setupTest = async () => { }); }; -describe('Convert entity template', () => { +describe('Image is rendered when switching entity template', () => { beforeAll(async () => { await setupPreFlights(); await setupTest(); From ab7986ae3e74ae5fea51d0d0690af976458c0796 Mon Sep 17 00:00:00 2001 From: Laszlo Kecskes Date: Tue, 28 Sep 2021 15:31:30 +0200 Subject: [PATCH 47/52] removed error catch, added typing, refactored functions --- app/api/csv/arrangeThesauri.ts | 59 ++++++++++++++----------------- app/api/csv/csvLoader.ts | 13 ++----- app/api/csv/typeParsers/select.ts | 5 +-- 3 files changed, 29 insertions(+), 48 deletions(-) diff --git a/app/api/csv/arrangeThesauri.ts b/app/api/csv/arrangeThesauri.ts index c91c8d356e..9dcf1504a1 100644 --- a/app/api/csv/arrangeThesauri.ts +++ b/app/api/csv/arrangeThesauri.ts @@ -1,4 +1,5 @@ import { ImportFile } from 'api/csv/importFile'; +import { WithId } from 'api/odm'; import thesauri from 'api/thesauri'; import { PropertySchema } from 'shared/types/commonTypes'; import { TemplateSchema } from 'shared/types/templateType'; @@ -56,45 +57,42 @@ type ThesauriValueData = { thesauriIdToNormalizedNewValues: Map>; }; -const setupIdValueMaps = (allRelatedThesauri: ThesaurusSchema[]): ThesauriValueData => { +const setupIdValueMaps = (allRelatedThesauri: WithId[]): ThesauriValueData => { const thesauriIdToExistingValues = new Map(); const thesauriIdToNewValues = new Map(); const thesauriIdToNormalizedNewValues = new Map(); allRelatedThesauri.forEach(t => { - if (t._id) { - const id = t._id.toString(); - thesauriIdToExistingValues.set( - id, - new Set(t.values?.map(v => normalizeThesaurusLabel(v.label))) - ); - thesauriIdToNewValues.set(id, new Set()); - thesauriIdToNormalizedNewValues.set(id, new Set()); - } + const id = t._id.toString(); + thesauriIdToExistingValues.set( + id, + new Set(t.values?.map(v => normalizeThesaurusLabel(v.label))) + ); + thesauriIdToNewValues.set(id, new Set()); + thesauriIdToNormalizedNewValues.set(id, new Set()); }); return { thesauriIdToExistingValues, thesauriIdToNewValues, thesauriIdToNormalizedNewValues }; }; const syncSaveThesauri = async ( - allRelatedThesauri: ThesaurusSchema[], + allRelatedThesauri: WithId[], thesauriIdToNewValues: Map> ) => { - for (let i = 0; i < allRelatedThesauri.length; i += 1) { + const thesauriWithNewValues = allRelatedThesauri.filter( + t => (thesauriIdToNewValues.get(t._id.toString()) || new Set()).size > 0 + ); + for (let i = 0; i < thesauriWithNewValues.length; i += 1) { const thesaurus = allRelatedThesauri[i]; - if (thesaurus?._id) { - const newValues: { label: string }[] = Array.from( - thesauriIdToNewValues.get(thesaurus._id.toString()) || [] - ).map(tval => ({ label: tval })); - if (newValues.length > 0) { - const thesaurusValues = thesaurus.values || []; - // eslint-disable-next-line no-await-in-loop - await thesauri.save({ - ...thesaurus, - values: thesaurusValues.concat(newValues), - }); - } - } + const newValues = Array.from( + thesauriIdToNewValues.get(thesaurus._id.toString()) || [] + ).map(tval => ({ label: tval })); + const thesaurusValues = thesaurus.values || []; + // eslint-disable-next-line no-await-in-loop + await thesauri.save({ + ...thesaurus, + values: thesaurusValues.concat(newValues), + }); } }; @@ -108,7 +106,7 @@ const arrangeThesauri = async ( ['select', 'multiselect'].includes(p.type) ); - let nameToThesauriId = createNameToIdMap(thesauriRelatedProperties, languages); + const nameToThesauriId = createNameToIdMap(thesauriRelatedProperties, languages); const allRelatedThesauri = await thesauri.get({ $in: Array.from( @@ -119,16 +117,11 @@ const arrangeThesauri = async ( const thesauriValueData = setupIdValueMaps(allRelatedThesauri); await csv(await file.readStream(), stopOnError) - .onRow(async (row: CSVRow, index: number) => { - if (index === 0) { - const columnnames = Object.keys(row); - nameToThesauriId = filterJSObject(nameToThesauriId, columnnames); - } - Object.entries(nameToThesauriId).forEach(([name, id]) => { + .onRow(async (row: CSVRow) => { + Object.entries(filterJSObject(nameToThesauriId, Object.keys(row))).forEach(([name, id]) => { const labels = splitMultiselectLabels(row[name]); Object.entries(labels).forEach(([normalizedLabel, originalLabel]) => { if ( - normalizedLabel && !thesauriValueData.thesauriIdToExistingValues.get(id)?.has(normalizedLabel) && !thesauriValueData.thesauriIdToNormalizedNewValues.get(id)?.has(normalizedLabel) ) { diff --git a/app/api/csv/csvLoader.ts b/app/api/csv/csvLoader.ts index 6fadac6fca..b05d01d932 100644 --- a/app/api/csv/csvLoader.ts +++ b/app/api/csv/csvLoader.ts @@ -10,7 +10,7 @@ import { ThesaurusSchema } from 'shared/types/thesaurusType'; import { ensure } from 'shared/tsUtils'; import { ObjectId } from 'mongodb'; -import { arrangeThesauri, ArrangeThesauriError } from './arrangeThesauri'; +import { arrangeThesauri } from './arrangeThesauri'; import csv, { CSVRow } from './csv'; import importFile from './importFile'; import { importEntity, translateEntity } from './importEntity'; @@ -56,16 +56,7 @@ export class CSVLoader extends EventEmitter { (await settings.get()).languages ).map((l: LanguageSchema) => l.key); const { newNameGeneration = false } = await settings.get(); - try { - await arrangeThesauri(file, template, availableLanguages, this.stopOnError); - } catch (e) { - if (e instanceof ArrangeThesauriError) { - const _e: ArrangeThesauriError = e; - this.emit('loadError', _e.source, toSafeName(_e.row), _e.index); - } else { - throw e; - } - } + await arrangeThesauri(file, template, availableLanguages); await csv(await file.readStream(), this.stopOnError) .onRow(async (row: CSVRow) => { diff --git a/app/api/csv/typeParsers/select.ts b/app/api/csv/typeParsers/select.ts index 1f8e3b295c..73c6a28f4e 100644 --- a/app/api/csv/typeParsers/select.ts +++ b/app/api/csv/typeParsers/select.ts @@ -4,10 +4,7 @@ import { ThesaurusSchema } from 'shared/types/thesaurusType'; import { MetadataObjectSchema, PropertySchema } from 'shared/types/commonTypes'; import { ensure } from 'shared/tsUtils'; -export function normalizeThesaurusLabel(label?: string | null): string | null { - if (label === undefined || label === null) { - return null; - } +export function normalizeThesaurusLabel(label: string): string | null { const trimmed = label.trim().toLowerCase(); return trimmed.length > 0 ? trimmed : null; } From a9e7f7202f4b1473f637a9b65c1e3b8c70ca2306 Mon Sep 17 00:00:00 2001 From: mfacar Date: Tue, 28 Sep 2021 20:07:55 -0500 Subject: [PATCH 48/52] removes session-id at proxy calling --- app/api/files/jsRoutes.js | 1 + app/api/files/specs/jsRoutes.spec.js | 25 +++++++++++++++++++++---- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/app/api/files/jsRoutes.js b/app/api/files/jsRoutes.js index a4fb4e5eef..8569095064 100644 --- a/app/api/files/jsRoutes.js +++ b/app/api/files/jsRoutes.js @@ -127,6 +127,7 @@ const routes = app => { }, proxyReqOptDecorator(proxyReqOpts) { const { tenant, ...headers } = proxyReqOpts.headers; + headers.cookie = headers.cookie ? headers.cookie.replace(/connect\.sid=.*/, '') : null; return { ...proxyReqOpts, headers: { ...headers }, diff --git a/app/api/files/specs/jsRoutes.spec.js b/app/api/files/specs/jsRoutes.spec.js index d3a48420ca..5f8eaf48a3 100644 --- a/app/api/files/specs/jsRoutes.spec.js +++ b/app/api/files/specs/jsRoutes.spec.js @@ -156,6 +156,10 @@ describe('upload routes', () => { beforeEach(() => { app = express(); uploadRoutes(app); + remoteApp = express(); + remoteApp.post('/api/public', (_req, res) => { + res.json(_req.headers); + }); }); afterEach(async () => { @@ -163,20 +167,33 @@ describe('upload routes', () => { }); it('should return the captcha and store it', done => { - remoteApp = express(); - remoteApp.post('/api/public', (_req, res) => { - res.json(_req.headers); + remoteServer = remoteApp.listen(54321, async () => { + const response = await request(app) + .post('/api/remotepublic') + .send({ title: 'Title' }) + .set('tenant', 'tenant') + .expect(200); + + const headersOnRemote = JSON.parse(response.text); + expect(headersOnRemote.tenant).not.toBeDefined(); + done(); }); + }); + it('should remove the session from the cookie', done => { remoteServer = remoteApp.listen(54321, async () => { const response = await request(app) .post('/api/remotepublic') .send({ title: 'Title' }) + .set( + 'cookie', + 'locale=en; SL_G_WPT_TO=en; connect.sid=s%3AnK04AiZIYyWOjO_p.kFF17AeJhqKr207n95pV8' + ) .set('tenant', 'tenant') .expect(200); const headersOnRemote = JSON.parse(response.text); - expect(headersOnRemote.tenant).not.toBeDefined(); + expect(headersOnRemote.cookie).toEqual('locale=en; SL_G_WPT_TO=en;'); done(); }); }); From 8078c240e7faa5a5a807fb4c85d8b8a46fb9dc5f Mon Sep 17 00:00:00 2001 From: Kevin Date: Wed, 29 Sep 2021 19:49:25 +0300 Subject: [PATCH 49/52] Removed unnecessary steps --- e2e/suites/convert-entity-template.test.ts | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/e2e/suites/convert-entity-template.test.ts b/e2e/suites/convert-entity-template.test.ts index 810feb1824..322a74d14c 100644 --- a/e2e/suites/convert-entity-template.test.ts +++ b/e2e/suites/convert-entity-template.test.ts @@ -1,6 +1,6 @@ /*global page*/ -import { ElementHandle } from 'puppeteer'; +// import { ElementHandle } from 'puppeteer'; import { createEntity } from '../helpers/createEntity'; import { createTemplate } from '../helpers/createTemplate'; import { adminLogin, logout } from '../helpers/login'; @@ -15,9 +15,8 @@ const setupPreFlights = async (): Promise => { }; const setupTest = async () => { - await createTemplate('Without image'); await createTemplate('With image'); - await createEntity('Without image', { + await createEntity('With image', { pdf: `${__dirname}/test_files/valid.pdf`, supportingFile: `${__dirname}/test_files/batman.jpg`, }); @@ -36,22 +35,6 @@ describe('Image is rendered when switching entity template', () => { await expect(page).toClick('.metadata-sidepanel button.edit-metadata', { text: 'Edit', }); - await expect(page).toMatchElement('select.form-control > option'); - const options = await page.$$('select.form-control > option'); - - const optionsValues = await options.map(async (optionElement: ElementHandle) => { - const value = await optionElement.evaluate(optionEl => ({ - text: optionEl.textContent, - value: optionEl.getAttribute('value') as string, - })); - return value; - }); - - const optionValues: any[] = await Promise.all(optionsValues); - const { value } = optionValues.find(option => option.text === 'With image'); - - await expect(page).toMatchElement('select.form-control'); - await page.select('select.form-control', value); await expect(page).toClick('span', { text: 'Select supporting file' }); await expect(page).toMatchElement('div.media-grid-card-header > h5', { text: 'batman.jpg' }); }); From 796009e3a42ff7409115cd2b2645f24ebf42ad17 Mon Sep 17 00:00:00 2001 From: mfacar Date: Wed, 29 Sep 2021 13:02:38 -0500 Subject: [PATCH 50/52] removing cookie before proxy --- app/api/files/jsRoutes.js | 2 +- app/api/files/specs/jsRoutes.spec.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/app/api/files/jsRoutes.js b/app/api/files/jsRoutes.js index 8569095064..9de64726f6 100644 --- a/app/api/files/jsRoutes.js +++ b/app/api/files/jsRoutes.js @@ -127,7 +127,7 @@ const routes = app => { }, proxyReqOptDecorator(proxyReqOpts) { const { tenant, ...headers } = proxyReqOpts.headers; - headers.cookie = headers.cookie ? headers.cookie.replace(/connect\.sid=.*/, '') : null; + delete headers.cookie; return { ...proxyReqOpts, headers: { ...headers }, diff --git a/app/api/files/specs/jsRoutes.spec.js b/app/api/files/specs/jsRoutes.spec.js index 5f8eaf48a3..0d714e4e04 100644 --- a/app/api/files/specs/jsRoutes.spec.js +++ b/app/api/files/specs/jsRoutes.spec.js @@ -193,7 +193,7 @@ describe('upload routes', () => { .expect(200); const headersOnRemote = JSON.parse(response.text); - expect(headersOnRemote.cookie).toEqual('locale=en; SL_G_WPT_TO=en;'); + expect(headersOnRemote.cookie).toBeUndefined(); done(); }); }); From 7d73c5db3da0c8628b22b210ac6f3b86bfc1136d Mon Sep 17 00:00:00 2001 From: Kevin Date: Wed, 29 Sep 2021 21:14:43 +0300 Subject: [PATCH 51/52] Removed comments and updated test description --- e2e/suites/convert-entity-template.test.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/e2e/suites/convert-entity-template.test.ts b/e2e/suites/convert-entity-template.test.ts index 322a74d14c..f7768f5c56 100644 --- a/e2e/suites/convert-entity-template.test.ts +++ b/e2e/suites/convert-entity-template.test.ts @@ -1,6 +1,3 @@ -/*global page*/ - -// import { ElementHandle } from 'puppeteer'; import { createEntity } from '../helpers/createEntity'; import { createTemplate } from '../helpers/createTemplate'; import { adminLogin, logout } from '../helpers/login'; @@ -22,7 +19,7 @@ const setupTest = async () => { }); }; -describe('Image is rendered when switching entity template', () => { +describe('Image is rendered when editing an entity in document view', () => { beforeAll(async () => { await setupPreFlights(); await setupTest(); From d1f70fd05b9bd229d360458c9c992742249d2371 Mon Sep 17 00:00:00 2001 From: mfacar Date: Thu, 30 Sep 2021 14:36:12 -0500 Subject: [PATCH 52/52] same approach that tenant for delete the cookie --- app/api/files/jsRoutes.js | 3 +-- app/api/files/specs/jsRoutes.spec.js | 23 +++++------------------ 2 files changed, 6 insertions(+), 20 deletions(-) diff --git a/app/api/files/jsRoutes.js b/app/api/files/jsRoutes.js index 9de64726f6..ab848363a5 100644 --- a/app/api/files/jsRoutes.js +++ b/app/api/files/jsRoutes.js @@ -126,8 +126,7 @@ const routes = app => { return '/api/public'; }, proxyReqOptDecorator(proxyReqOpts) { - const { tenant, ...headers } = proxyReqOpts.headers; - delete headers.cookie; + const { tenant, cookie, ...headers } = proxyReqOpts.headers; return { ...proxyReqOpts, headers: { ...headers }, diff --git a/app/api/files/specs/jsRoutes.spec.js b/app/api/files/specs/jsRoutes.spec.js index 0d714e4e04..18e1138359 100644 --- a/app/api/files/specs/jsRoutes.spec.js +++ b/app/api/files/specs/jsRoutes.spec.js @@ -156,31 +156,17 @@ describe('upload routes', () => { beforeEach(() => { app = express(); uploadRoutes(app); - remoteApp = express(); - remoteApp.post('/api/public', (_req, res) => { - res.json(_req.headers); - }); }); afterEach(async () => { await remoteServer.close(); }); - it('should return the captcha and store it', done => { - remoteServer = remoteApp.listen(54321, async () => { - const response = await request(app) - .post('/api/remotepublic') - .send({ title: 'Title' }) - .set('tenant', 'tenant') - .expect(200); - - const headersOnRemote = JSON.parse(response.text); - expect(headersOnRemote.tenant).not.toBeDefined(); - done(); + it('should remove the tenant and cookie from headers', done => { + remoteApp = express(); + remoteApp.post('/api/public', (_req, res) => { + res.json(_req.headers); }); - }); - - it('should remove the session from the cookie', done => { remoteServer = remoteApp.listen(54321, async () => { const response = await request(app) .post('/api/remotepublic') @@ -193,6 +179,7 @@ describe('upload routes', () => { .expect(200); const headersOnRemote = JSON.parse(response.text); + expect(headersOnRemote.tenant).toBeUndefined(); expect(headersOnRemote.cookie).toBeUndefined(); done(); });