Fixed suggestions document language issues

app/api/services/informationextraction/InformationExtraction.ts

@@ -186,9 +186,19 @@ class InformationExtraction {
         if (!entity) {
           return Promise.resolve();
         }
+
+        const [segmentation] = await SegmentationModel.get({
+          xmlname: rawSuggestion.xml_file_name,
+        });
+
+        if (!segmentation) {
+          return Promise.resolve();
+        }
+
         const [currentSuggestion] = await IXSuggestionsModel.get({
           entityId: entity.sharedId,
           propertyName: rawSuggestion.property_name,
+          fileId: segmentation.fileID,
         });
 
         let status: 'ready' | 'failed' = 'ready';
@@ -207,9 +217,6 @@ class InformationExtraction {
 
         const suggestion: IXSuggestionType = {
           ...currentSuggestion,
-          entityId: entity.sharedId!,
-          language: entity.language!,
-          propertyName: rawSuggestion.property_name,
           suggestedValue,
           segment: rawSuggestion.segment_text,
           status,
@@ -228,13 +235,13 @@ class InformationExtraction {
     const [existingSuggestions] = await IXSuggestionsModel.get({
       entityId: entity.sharedId,
       propertyName,
-      language: entity.language,
+      fileId: file._id,
     });
     const suggestion: IXSuggestionType = {
       ...existingSuggestions,
       entityId: entity.sharedId!,
       fileId: file._id,
-      language: entity.language!,
+      language: languages.get(file.language, 'ISO639_1') || 'other',
       propertyName,
       status: 'processing',
       date: new Date().getTime(),

app/api/services/informationextraction/specs/InformationExtraction.spec.ts

@@ -3,7 +3,7 @@ import { testingEnvironment } from 'api/utils/testingEnvironment';
 import { testingTenants } from 'api/utils/testingTenants';
 import { IXSuggestionsModel } from 'api/suggestions/IXSuggestionsModel';
 import { fs } from 'api/files';
-import { fixtures } from './fixtures';
+import { factory, fixtures } from './fixtures';
 import { InformationExtraction } from '../InformationExtraction';
 import { ExternalDummyService } from '../../tasksmanager/specs/ExternalDummyService';
 
@@ -43,6 +43,24 @@ describe('InformationExtraction', () => {
     await testingEnvironment.tearDown();
   });
 
+  const saveSuggestionProcess = async (
+    id: string,
+    entity: string,
+    language: string,
+    property: string
+  ) => {
+    await informationExtraction.saveSuggestionProcess(
+      {
+        _id: factory.id(id),
+        entity,
+        language,
+        segmentation: {},
+        extractedMetadata: [],
+      },
+      property
+    );
+  };
+
   describe('trainModel', () => {
     it('should send xmls', async () => {
       await informationExtraction.trainModel('property1');
@@ -203,6 +221,8 @@ describe('InformationExtraction', () => {
         },
       ]);
 
+      await saveSuggestionProcess('F3', 'A3', 'eng', 'property1');
+      await saveSuggestionProcess('F1', 'A1', 'eng', 'property1');
       await informationExtraction.processResults({
         params: { property_name: 'property1' },
         tenant: 'tenant1',
@@ -229,6 +249,61 @@ describe('InformationExtraction', () => {
       );
     });
 
+    it('should save different language suggestions for the same entity', async () => {
+      IXExternalService.setResults([
+        {
+          tenant: 'tenant1',
+          property_name: 'property1',
+          xml_file_name: 'documentA.xml',
+          text: 'text_in_other_language',
+          segment_text: 'segmented_text_in_other_language',
+        },
+        {
+          tenant: 'tenant1',
+          property_name: 'property1',
+          xml_file_name: 'documentD.xml',
+          text: 'text_in_eng_language',
+          segment_text: 'segmented_text_in_eng_language',
+        },
+      ]);
+
+      await saveSuggestionProcess('F1', 'A1', 'other', 'property1');
+      await saveSuggestionProcess('F4', 'A1', 'eng', 'property1');
+
+      await informationExtraction.processResults({
+        params: { property_name: 'property1' },
+        tenant: 'tenant1',
+        task: 'suggestions',
+        success: true,
+        data_url: 'http://localhost:1234/suggestions_results',
+      });
+
+      const suggestions = await IXSuggestionsModel.get({
+        status: 'ready',
+        propertyName: 'property1',
+      });
+
+      expect(suggestions.length).toBe(2);
+
+      expect(suggestions.find(s => s.language === 'other')).toEqual(
+        expect.objectContaining({
+          language: 'other',
+          propertyName: 'property1',
+          status: 'ready',
+          suggestedValue: 'text_in_other_language',
+        })
+      );
+
+      expect(suggestions.find(s => s.language === 'en')).toEqual(
+        expect.objectContaining({
+          language: 'en',
+          propertyName: 'property1',
+          status: 'ready',
+          suggestedValue: 'text_in_eng_language',
+        })
+      );
+    });
+
     it('should store failed suggestions', async () => {
       IXExternalService.setResults([
         {
@@ -319,6 +394,8 @@ describe('InformationExtraction', () => {
         },
       ]);
 
+      await saveSuggestionProcess('F1', 'A1', 'eng', 'property4');
+
       await informationExtraction.processResults({
         params: { property_name: 'property1' },
         tenant: 'tenant1',
@@ -359,6 +436,8 @@ describe('InformationExtraction', () => {
         },
       ]);
 
+      await saveSuggestionProcess('F5', 'A5', 'eng', 'property1');
+
       await informationExtraction.processResults({
         params: { property_name: 'property1' },
         tenant: 'tenant1',

app/api/services/informationextraction/specs/fixtures.ts

@@ -86,7 +86,7 @@ const fixtures: DBFixture = {
         },
       },
     ]),
-    factory.file('F4', 'A4', 'document', fixturesPdfNameD, 'eng', '', [
+    factory.file('F4', 'A1', 'document', fixturesPdfNameD, 'eng', '', [
       {
         name: 'property2',
         selection: {
@@ -162,6 +162,7 @@ const fixtures: DBFixture = {
   ],
   ixsuggestions: [
     {
+      fileId: factory.id('F1'),
       entityId: 'A1',
       language: 'en',
       propertyName: 'property1',

app/api/suggestions/IXSuggestionsModel.ts

@@ -12,7 +12,8 @@ const mongoSchema = new mongoose.Schema(props, {
   strict: false,
 });
 
-mongoSchema.index({ propertyName: 'text' });
+// @ts-ignore
+mongoSchema.index({ propertyName: 'text' }, { language_override: '_text' });
 
 const IXSuggestionsModel = instanceModel<IXSuggestionType>('ixsuggestions', mongoSchema);