3242 slow csv import with thesaurus creation

app/api/csv/csvLoader.ts

@@ -1,3 +1,4 @@
+/* eslint-disable max-statements */
 import { EventEmitter } from 'events';
 
 import templates from 'api/templates';
@@ -12,7 +13,7 @@ import { ensure } from 'shared/tsUtils';
 import { ObjectId } from 'mongodb';
 import csv, { CSVRow } from './csv';
 import importFile from './importFile';
-import { importEntity, translateEntity } from './importEntity';
+import { arrangeThesauri, importEntity, translateEntity } from './importEntity';
 import { extractEntity, toSafeName } from './entityRow';
 
 export class CSVLoader extends EventEmitter {
@@ -55,6 +56,7 @@ export class CSVLoader extends EventEmitter {
       (await settings.get()).languages
     ).map((l: LanguageSchema) => l.key);
     const { newNameGeneration = false } = await settings.get();
+    await arrangeThesauri(file, template, availableLanguages, this);
 
     await csv(await file.readStream(), this.stopOnError)
       .onRow(async (row: CSVRow) => {
@@ -64,7 +66,6 @@ export class CSVLoader extends EventEmitter {
           options.language,
           newNameGeneration
         );
-
         if (rawEntity) {
           const entity = await importEntity(rawEntity, template, file, options);
           await translateEntity(entity, rawTranslations, template, file);

app/api/csv/importEntity.ts

@@ -1,18 +1,23 @@
+/* eslint-disable max-statements */
 import entities from 'api/entities';
 import { search } from 'api/search';
 import entitiesModel from 'api/entities/entitiesModel';
 import { processDocument } from 'api/files/processDocument';
-import { RawEntity } from 'api/csv/entityRow';
+import { RawEntity, toSafeName } from 'api/csv/entityRow';
 import { TemplateSchema } from 'shared/types/templateType';
 import { MetadataSchema, PropertySchema } from 'shared/types/commonTypes';
+import { propertyTypes } from 'shared/propertyTypes';
 import { ImportFile } from 'api/csv/importFile';
+import thesauri from 'api/thesauri';
 import { EntitySchema } from 'shared/types/entityType';
 import { ensure } from 'shared/tsUtils';
-
 import { attachmentsPath, files } from 'api/files';
-import { propertyTypes } from 'shared/propertyTypes';
 import { generateID } from 'shared/IDGenerator';
+
+import { normalizeThesaurusLabel } from './typeParsers/select';
+import { splitMultiselectLabels } from './typeParsers/multiselect';
 import typeParsers from './typeParsers';
+import csv, { CSVRow } from './csv';
 
 const parse = async (toImportEntity: RawEntity, prop: PropertySchema) =>
   typeParsers[prop.type]
@@ -66,6 +71,120 @@ type Options = {
   language: string;
 };
 
+const filterJSObject = (input: { [k: string]: any }, keys: string[]): { [k: string]: any } => {
+  const result: { [k: string]: any } = {};
+  keys.forEach(k => {
+    if (input.hasOwnProperty(k)) {
+      result[k] = input[k];
+    }
+  });
+  return result;
+};
+
+const arrangeThesauri = async (
+  file: ImportFile,
+  template: TemplateSchema,
+  languages?: string[],
+  errorContext?: any
+) => {
+  let nameToThesauriIdSelects: { [k: string]: string } = {};
+  let nameToThesauriIdMultiselects: { [k: string]: string } = {};
+  const thesauriIdToExistingValues = new Map();
+  const thesauriIdToNewValues: Map<string, Set<string>> = new Map();
+  const thesauriIdToNormalizedNewValues = new Map();
+  const thesauriRelatedProperties = template.properties?.filter(p =>
+    ['select', 'multiselect'].includes(p.type)
+  );
+  thesauriRelatedProperties?.forEach(p => {
+    if (p.content && p.type) {
+      const thesarusID = p.content.toString();
+      if (p.type === propertyTypes.select) {
+        nameToThesauriIdSelects[p.name] = thesarusID;
+        languages?.forEach(suffix => {
+          nameToThesauriIdSelects[`${p.name}__${suffix}`] = thesarusID;
+        });
+      } else if (p.type === propertyTypes.multiselect) {
+        nameToThesauriIdMultiselects[p.name] = thesarusID;
+        languages?.forEach(suffix => {
+          nameToThesauriIdMultiselects[`${p.name}__${suffix}`] = thesarusID;
+        });
+      }
+    }
+  });
+  const allRelatedThesauri = await Promise.all(
+    Array.from(
+      new Set(thesauriRelatedProperties?.map(p => p.content?.toString()).filter(t => t))
+    ).map(async id => thesauri.getById(id))
+  );
+  allRelatedThesauri.forEach(t => {
+    if (t) {
+      const id = t._id.toString();
+      thesauriIdToExistingValues.set(
+        id,
+        new Set(t.values?.map(v => normalizeThesaurusLabel(v.label)))
+      );
+      thesauriIdToNewValues.set(id, new Set());
+      thesauriIdToNormalizedNewValues.set(id, new Set());
+    }
+  });
+  function handleLabels(id: string, original: string, normalized: string | null) {
+    if (
+      normalized &&
+      !thesauriIdToExistingValues.get(id).has(normalized) &&
+      !thesauriIdToNormalizedNewValues.get(id).has(normalized)
+    ) {
+      thesauriIdToNewValues.get(id)?.add(original);
+      thesauriIdToNormalizedNewValues.get(id).add(normalized);
+    }
+  }
+  await csv(await file.readStream(), errorContext?.stopOnError)
+    .onRow(async (row: CSVRow, index: number) => {
+      if (index === 0) {
+        const columnnames = Object.keys(row);
+        nameToThesauriIdSelects = filterJSObject(nameToThesauriIdSelects, columnnames);
+        nameToThesauriIdMultiselects = filterJSObject(nameToThesauriIdMultiselects, columnnames);
+      }
+      Object.entries(nameToThesauriIdSelects).forEach(([name, id]) => {
+        const label = row[name];
+        if (label) {
+          const normalizedLabel = normalizeThesaurusLabel(label);
+          handleLabels(id, label, normalizedLabel);
+        }
+      });
+      Object.entries(nameToThesauriIdMultiselects).forEach(([name, id]) => {
+        const labels = splitMultiselectLabels(row[name]);
+        if (labels) {
+          Object.entries(labels).forEach(([normalizedLabel, originalLabel]) => {
+            handleLabels(id, originalLabel, normalizedLabel);
+          });
+        }
+      });
+    })
+    .onError(async (e: Error, row: CSVRow, index: number) => {
+      if (errorContext) {
+        errorContext._errors[index] = e;
+        errorContext.emit('loadError', e, toSafeName(row), index);
+      }
+    })
+    .read();
+  for (let i = 0; i < allRelatedThesauri.length; i += 1) {
+    const thesaurus = allRelatedThesauri[i];
+    if (thesaurus !== null) {
+      const newValues: { label: string }[] = Array.from(
+        thesauriIdToNewValues.get(thesaurus._id.toString()) || []
+      ).map(tval => ({ label: tval }));
+      if (newValues.length > 0) {
+        const thesaurusValues = thesaurus.values || [];
+        // eslint-disable-next-line no-await-in-loop
+        await thesauri.save({
+          ...thesaurus,
+          values: thesaurusValues.concat(newValues),
+        });
+      }
+    }
+  }
+};
+
 const importEntity = async (
   toImportEntity: RawEntity,
   template: TemplateSchema,
@@ -122,4 +241,4 @@ const translateEntity = async (
   await search.indexEntities({ sharedId: entity.sharedId }, '+fullText');
 };
 
-export { importEntity, translateEntity };
+export { arrangeThesauri, importEntity, translateEntity };

app/api/csv/importFile.ts

@@ -1,6 +1,5 @@
 import fs from 'fs';
 import path from 'path';
-import { Readable } from 'stream';
 
 import { generateFileName, fileFromReadStream, uploadsPath } from 'api/files/filesystem';
 import { createError } from 'api/utils';
@@ -17,16 +16,13 @@ const extractFromZip = async (zipPath: string, fileName: string) => {
 };
 
 export class ImportFile {
-  filePath: string | Readable;
+  filePath: string;
 
-  constructor(filePath: string | Readable) {
+  constructor(filePath: string) {
     this.filePath = filePath;
   }
 
   async readStream(fileName = 'import.csv') {
-    if (this.filePath instanceof Readable) {
-      return this.filePath;
-    }
     if (path.extname(this.filePath) === '.zip') {
       return extractFromZip(this.filePath, fileName);
     }
@@ -46,6 +42,6 @@ export class ImportFile {
   }
 }
 
-const importFile = (filePath: string | Readable) => new ImportFile(filePath);
+const importFile = (filePath: string) => new ImportFile(filePath);
 
 export default importFile;

app/api/csv/specs/__snapshots__/importFile.spec.ts.snap

@@ -1,11 +1,11 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
 exports[`importFile readStream should return a readable stream for the csv file 1`] = `
-"Title , text label  , numeric label, non configured, select label, not defined type, geolocation_geolocation,auto id, additional tag(s)
+"Title , text label  , numeric label, non configured, select_label, not defined type, geolocation_geolocation,auto id, additional tag(s), multi_select_label
 
-title1, text value 1, 1977, ______________, thesauri1   , notType1                 , 1|1,,tag1
-title2, text value 2, 2019, ______________, thesauri2   , notType2                 ,    ,,tag2
-title3, text value 3, 2020, ______________, thesauri2   , notType3                 , 0|0,,tag3
+title1, text value 1, 1977, ______________, thesauri1   , notType1                 , 1|1,,tag1, multivalue1
+title2, text value 2, 2019, ______________, thesauri2   , notType2                 ,    ,,tag2, multivalue2
+title3, text value 3, 2020, ______________, thesauri2   , notType3                 , 0|0,,tag3, multivalue1|multivalue3
 "
 `;
 

app/api/csv/specs/arrangeThesauriTest.csv

@@ -0,0 +1,17 @@
+title,unrelated_property,select_property__en, select_property__es,multiselect_property__en, multiselect_property__es
+select_1,unrelated_text,B,Bes,A,Aes
+select_2,unrelated_text,C,Ces,A,Aes
+select_3,unrelated_text,b,bes,A,Aes
+select_4,unrelated_text,B,Bes,A,Aes
+select_5,unrelated_text,d,des,A,Aes
+select_6,unrelated_text,D,Des,A,Aes
+select_7,unrelated_text,  b,bes,A,Aes
+select_8,unrelated_text, , ,A,Aes
+select_8,unrelated_text,      ,     ,A,Aes
+multiselect_1,unrelated_text,A,Aes,B,Bes
+multiselect_2,unrelated_text,A,Aes,c,ces
+multiselect_3,unrelated_text,A,Aes,A|b,Aes|bes
+multiselect_4,unrelated_text,A,Aes,a|B|C,aes|Bes|Ces
+multiselect_5,unrelated_text,A,Aes,  a|   b | ,  aes|   bes | 
+multiselect_6,unrelated_text,A,Aes, |   | , |   | 
+multiselect_7, unrelated_text,A,Aes,A|B|C|D|  |E| e| g  ,Aes|Bes|Ces|Des|  |Ees| ees| ges