3242 slow csv import with thesaurus creation

app/api/csv/arrangeThesauri.ts

@@ -0,0 +1,180 @@
+import { ImportFile } from 'api/csv/importFile';
+import thesauri from 'api/thesauri';
+import { propertyTypes } from 'shared/propertyTypes';
+import { PropertySchema } from 'shared/types/commonTypes';
+import { TemplateSchema } from 'shared/types/templateType';
+import { ThesaurusSchema } from 'shared/types/thesaurusType';
+
+import csv, { CSVRow } from './csv';
+import { splitMultiselectLabels } from './typeParsers/multiselect';
+import { normalizeThesaurusLabel } from './typeParsers/select';
+
+const filterJSObject = (input: { [k: string]: any }, keys: string[]): { [k: string]: any } => {
+  const result: { [k: string]: any } = {};
+  keys.forEach(k => {
+    if (input.hasOwnProperty(k)) {
+      result[k] = input[k];
+    }
+  });
+  return result;
+};
+
+class ArrangeThesauriError extends Error {
+  source: Error;
+  row: CSVRow;
+  index: number;
+
+  constructor(source: Error, row: CSVRow, index: number) {
+    super(source.message);
+    this.source = source;
+    this.row = row;
+    this.index = index;
+  }
+}
+
+const separateSelectAndMultiselectThesauri = (
+  thesauriRelatedProperties: PropertySchema[] | undefined,
+  languages?: string[]
+): [{ [k: string]: string }, { [k: string]: string }] => {
+  const nameToThesauriIdSelects: { [k: string]: string } = {};
+  const nameToThesauriIdMultiselects: { [k: string]: string } = {};
+
+  thesauriRelatedProperties?.forEach(p => {
+    if (p.content && p.type) {
+      const thesarusID = p.content.toString();
+      if (p.type === propertyTypes.select) {
+        nameToThesauriIdSelects[p.name] = thesarusID;
+        languages?.forEach(suffix => {
+          nameToThesauriIdSelects[`${p.name}__${suffix}`] = thesarusID;
+        });
+      } else if (p.type === propertyTypes.multiselect) {
+        nameToThesauriIdMultiselects[p.name] = thesarusID;
+        languages?.forEach(suffix => {
+          nameToThesauriIdMultiselects[`${p.name}__${suffix}`] = thesarusID;
+        });
+      }
+    }
+  });
+
+  return [nameToThesauriIdSelects, nameToThesauriIdMultiselects];
+};
+
+type ThesauriValueData = {
+  thesauriIdToExistingValues: Map<string, Set<string>>;
+  thesauriIdToNewValues: Map<string, Set<string>>;
+  thesauriIdToNormalizedNewValues: Map<string, Set<string>>;
+};
+
+const setupIdValueMaps = (allRelatedThesauri: ThesaurusSchema[]): ThesauriValueData => {
+  const thesauriIdToExistingValues = new Map();
+  const thesauriIdToNewValues = new Map();
+  const thesauriIdToNormalizedNewValues = new Map();
+
+  allRelatedThesauri.forEach(t => {
+    if (t._id) {
+      const id = t._id.toString();
+      thesauriIdToExistingValues.set(
+        id,
+        new Set(t.values?.map(v => normalizeThesaurusLabel(v.label)))
+      );
+      thesauriIdToNewValues.set(id, new Set());
+      thesauriIdToNormalizedNewValues.set(id, new Set());
+    }
+  });
+
+  return { thesauriIdToExistingValues, thesauriIdToNewValues, thesauriIdToNormalizedNewValues };
+};
+
+const handleLabels = (
+  id: string,
+  original: string,
+  normalized: string | null,
+  thesauriValueData: ThesauriValueData
+) => {
+  if (
+    normalized &&
+    !thesauriValueData.thesauriIdToExistingValues.get(id)?.has(normalized) &&
+    !thesauriValueData.thesauriIdToNormalizedNewValues.get(id)?.has(normalized)
+  ) {
+    thesauriValueData.thesauriIdToNewValues.get(id)?.add(original);
+    thesauriValueData.thesauriIdToNormalizedNewValues.get(id)?.add(normalized);
+  }
+};
+
+const syncSaveThesauri = async (
+  allRelatedThesauri: ThesaurusSchema[],
+  thesauriIdToNewValues: Map<string, Set<string>>
+) => {
+  for (let i = 0; i < allRelatedThesauri.length; i += 1) {
+    const thesaurus = allRelatedThesauri[i];
+    if (thesaurus?._id) {
+      const newValues: { label: string }[] = Array.from(
+        thesauriIdToNewValues.get(thesaurus._id.toString()) || []
+      ).map(tval => ({ label: tval }));
+      if (newValues.length > 0) {
+        const thesaurusValues = thesaurus.values || [];
+        // eslint-disable-next-line no-await-in-loop
+        await thesauri.save({
+          ...thesaurus,
+          values: thesaurusValues.concat(newValues),
+        });
+      }
+    }
+  }
+};
+
+const arrangeThesauri = async (
+  file: ImportFile,
+  template: TemplateSchema,
+  languages?: string[],
+  stopOnError: boolean = true
+) => {
+  const thesauriRelatedProperties = template.properties?.filter(p =>
+    ['select', 'multiselect'].includes(p.type)
+  );
+
+  let [
+    nameToThesauriIdSelects,
+    nameToThesauriIdMultiselects,
+  ] = separateSelectAndMultiselectThesauri(thesauriRelatedProperties, languages);
+
+  const allRelatedThesauri = await thesauri.get({
+    $in: Array.from(
+      new Set(thesauriRelatedProperties?.map(p => p.content?.toString()).filter(t => t))
+    ),
+  });
+
+  const thesauriValueData = setupIdValueMaps(allRelatedThesauri);
+
+  await csv(await file.readStream(), stopOnError)
+    .onRow(async (row: CSVRow, index: number) => {
+      if (index === 0) {
+        const columnnames = Object.keys(row);
+        nameToThesauriIdSelects = filterJSObject(nameToThesauriIdSelects, columnnames);
+        nameToThesauriIdMultiselects = filterJSObject(nameToThesauriIdMultiselects, columnnames);
+      }
+      Object.entries(nameToThesauriIdSelects).forEach(([name, id]) => {
+        const label = row[name];
+        if (label) {
+          const normalizedLabel = normalizeThesaurusLabel(label);
+          handleLabels(id, label, normalizedLabel, thesauriValueData);
+        }
+      });
+      Object.entries(nameToThesauriIdMultiselects).forEach(([name, id]) => {
+        const labels = splitMultiselectLabels(row[name]);
+        if (labels) {
+          Object.entries(labels).forEach(([normalizedLabel, originalLabel]) => {
+            handleLabels(id, originalLabel, normalizedLabel, thesauriValueData);
+          });
+        }
+      });
+    })
+    .onError(async (e: Error, row: CSVRow, index: number) => {
+      throw new ArrangeThesauriError(e, row, index);
+    })
+    .read();
+
+  await syncSaveThesauri(allRelatedThesauri, thesauriValueData.thesauriIdToNewValues);
+};
+
+export { arrangeThesauri, ArrangeThesauriError };

app/api/csv/csvLoader.ts

@@ -1,4 +1,3 @@
-/* eslint-disable max-statements */
 import { EventEmitter } from 'events';
 
 import templates from 'api/templates';
@@ -11,9 +10,10 @@ import { ThesaurusSchema } from 'shared/types/thesaurusType';
 
 import { ensure } from 'shared/tsUtils';
 import { ObjectId } from 'mongodb';
+import { arrangeThesauri, ArrangeThesauriError } from './arrangeThesauri';
 import csv, { CSVRow } from './csv';
 import importFile from './importFile';
-import { arrangeThesauri, importEntity, translateEntity } from './importEntity';
+import { importEntity, translateEntity } from './importEntity';
 import { extractEntity, toSafeName } from './entityRow';
 
 export class CSVLoader extends EventEmitter {
@@ -56,7 +56,16 @@ export class CSVLoader extends EventEmitter {
       (await settings.get()).languages
     ).map((l: LanguageSchema) => l.key);
     const { newNameGeneration = false } = await settings.get();
-    await arrangeThesauri(file, template, availableLanguages, this);
+    try {
+      await arrangeThesauri(file, template, availableLanguages, this.stopOnError);
+    } catch (e) {
+      if (e instanceof ArrangeThesauriError) {
+        const _e: ArrangeThesauriError = e;
+        this.emit('loadError', _e.source, toSafeName(_e.row), _e.index);
+      } else {
+        throw e;
+      }
+    }
 
     await csv(await file.readStream(), this.stopOnError)
       .onRow(async (row: CSVRow) => {

app/api/csv/importEntity.ts

@@ -1,23 +1,18 @@
-/* eslint-disable max-statements */
 import entities from 'api/entities';
 import { search } from 'api/search';
 import entitiesModel from 'api/entities/entitiesModel';
 import { processDocument } from 'api/files/processDocument';
-import { RawEntity, toSafeName } from 'api/csv/entityRow';
+import { RawEntity } from 'api/csv/entityRow';
 import { TemplateSchema } from 'shared/types/templateType';
 import { MetadataSchema, PropertySchema } from 'shared/types/commonTypes';
 import { propertyTypes } from 'shared/propertyTypes';
 import { ImportFile } from 'api/csv/importFile';
-import thesauri from 'api/thesauri';
 import { EntitySchema } from 'shared/types/entityType';
 import { ensure } from 'shared/tsUtils';
 import { attachmentsPath, files } from 'api/files';
 import { generateID } from 'shared/IDGenerator';
 
-import { normalizeThesaurusLabel } from './typeParsers/select';
-import { splitMultiselectLabels } from './typeParsers/multiselect';
 import typeParsers from './typeParsers';
-import csv, { CSVRow } from './csv';
 
 const parse = async (toImportEntity: RawEntity, prop: PropertySchema) =>
   typeParsers[prop.type]
@@ -71,120 +66,6 @@ type Options = {
   language: string;
 };
 
-const filterJSObject = (input: { [k: string]: any }, keys: string[]): { [k: string]: any } => {
-  const result: { [k: string]: any } = {};
-  keys.forEach(k => {
-    if (input.hasOwnProperty(k)) {
-      result[k] = input[k];
-    }
-  });
-  return result;
-};
-
-const arrangeThesauri = async (
-  file: ImportFile,
-  template: TemplateSchema,
-  languages?: string[],
-  errorContext?: any
-) => {
-  let nameToThesauriIdSelects: { [k: string]: string } = {};
-  let nameToThesauriIdMultiselects: { [k: string]: string } = {};
-  const thesauriIdToExistingValues = new Map();
-  const thesauriIdToNewValues: Map<string, Set<string>> = new Map();
-  const thesauriIdToNormalizedNewValues = new Map();
-  const thesauriRelatedProperties = template.properties?.filter(p =>
-    ['select', 'multiselect'].includes(p.type)
-  );
-  thesauriRelatedProperties?.forEach(p => {
-    if (p.content && p.type) {
-      const thesarusID = p.content.toString();
-      if (p.type === propertyTypes.select) {
-        nameToThesauriIdSelects[p.name] = thesarusID;
-        languages?.forEach(suffix => {
-          nameToThesauriIdSelects[`${p.name}__${suffix}`] = thesarusID;
-        });
-      } else if (p.type === propertyTypes.multiselect) {
-        nameToThesauriIdMultiselects[p.name] = thesarusID;
-        languages?.forEach(suffix => {
-          nameToThesauriIdMultiselects[`${p.name}__${suffix}`] = thesarusID;
-        });
-      }
-    }
-  });
-  const allRelatedThesauri = await Promise.all(
-    Array.from(
-      new Set(thesauriRelatedProperties?.map(p => p.content?.toString()).filter(t => t))
-    ).map(async id => thesauri.getById(id))
-  );
-  allRelatedThesauri.forEach(t => {
-    if (t) {
-      const id = t._id.toString();
-      thesauriIdToExistingValues.set(
-        id,
-        new Set(t.values?.map(v => normalizeThesaurusLabel(v.label)))
-      );
-      thesauriIdToNewValues.set(id, new Set());
-      thesauriIdToNormalizedNewValues.set(id, new Set());
-    }
-  });
-  function handleLabels(id: string, original: string, normalized: string | null) {
-    if (
-      normalized &&
-      !thesauriIdToExistingValues.get(id).has(normalized) &&
-      !thesauriIdToNormalizedNewValues.get(id).has(normalized)
-    ) {
-      thesauriIdToNewValues.get(id)?.add(original);
-      thesauriIdToNormalizedNewValues.get(id).add(normalized);
-    }
-  }
-  await csv(await file.readStream(), errorContext?.stopOnError)
-    .onRow(async (row: CSVRow, index: number) => {
-      if (index === 0) {
-        const columnnames = Object.keys(row);
-        nameToThesauriIdSelects = filterJSObject(nameToThesauriIdSelects, columnnames);
-        nameToThesauriIdMultiselects = filterJSObject(nameToThesauriIdMultiselects, columnnames);
-      }
-      Object.entries(nameToThesauriIdSelects).forEach(([name, id]) => {
-        const label = row[name];
-        if (label) {
-          const normalizedLabel = normalizeThesaurusLabel(label);
-          handleLabels(id, label, normalizedLabel);
-        }
-      });
-      Object.entries(nameToThesauriIdMultiselects).forEach(([name, id]) => {
-        const labels = splitMultiselectLabels(row[name]);
-        if (labels) {
-          Object.entries(labels).forEach(([normalizedLabel, originalLabel]) => {
-            handleLabels(id, originalLabel, normalizedLabel);
-          });
-        }
-      });
-    })
-    .onError(async (e: Error, row: CSVRow, index: number) => {
-      if (errorContext) {
-        errorContext._errors[index] = e;
-        errorContext.emit('loadError', e, toSafeName(row), index);
-      }
-    })
-    .read();
-  for (let i = 0; i < allRelatedThesauri.length; i += 1) {
-    const thesaurus = allRelatedThesauri[i];
-    if (thesaurus !== null) {
-      const newValues: { label: string }[] = Array.from(
-        thesauriIdToNewValues.get(thesaurus._id.toString()) || []
-      ).map(tval => ({ label: tval }));
-      if (newValues.length > 0) {
-        const thesaurusValues = thesaurus.values || [];
-        // eslint-disable-next-line no-await-in-loop
-        await thesauri.save({
-          ...thesaurus,
-          values: thesaurusValues.concat(newValues),
-        });
-      }
-    }
-  }
-};
-
 const importEntity = async (
   toImportEntity: RawEntity,
   template: TemplateSchema,
@@ -241,4 +122,4 @@ const translateEntity = async (
   await search.indexEntities({ sharedId: entity.sharedId }, '+fullText');
 };
 
-export { arrangeThesauri, importEntity, translateEntity };
+export { importEntity, translateEntity };

app/api/csv/specs/csvLoader.spec.js

@@ -169,22 +169,6 @@ describe('csvLoader', () => {
       expect(textValues.length).toEqual(0);
     });
 
-    it('should arrange translations for selects and multiselects', async () => {
-      const trs = await translations.get();
-      trs.forEach(tr => {
-        expect(tr.contexts.find(c => c.label === 'thesauri1').values).toMatchObject({
-          thesauri1: 'thesauri1',
-          thesauri2: 'thesauri2',
-        });
-        expect(tr.contexts.find(c => c.label === 'multi_select_thesaurus').values).toMatchObject({
-          multi_select_thesaurus: 'multi_select_thesaurus',
-          multivalue1: 'multivalue1',
-          multivalue2: 'multivalue2',
-          multivalue3: 'multivalue3',
-        });
-      });
-    });
-
     describe('metadata parsing', () => {
       it('should parse metadata properties by type using typeParsers', () => {
         const textValues = imported.map(i => i.metadata.text_label[0].value);