Add more suggestions states

app/api/migrations/migrations/75-add_system_key_translations/index.js

@@ -0,0 +1,65 @@
+//eslint-disable-next-line node/no-restricted-import
+import * as fs from 'fs';
+
+import csv from 'api/csv/csv';
+
+/*
+This migration is meant to be repeatable.
+After copy pasting:
+  - change the contents of system_keys.csv to the new keyset
+  - change the file location in the readCsvToSystemKeys call
+  - change the tests, if necessary
+*/
+
+// eslint-disable-next-line max-statements
+async function readCsvToSystemKeys(db, filename) {
+  const fstream = fs.createReadStream(filename);
+  const rows = await csv(fstream).read();
+  fstream.close();
+  const translations = await db.collection('translations').find().toArray();
+  const locales = translations.map(tr => tr.locale);
+
+  const locToSystemContext = {};
+  translations.forEach(tr => {
+    locToSystemContext[tr.locale] = tr.contexts.find(c => c.id === 'System');
+  });
+  const locToKeys = {};
+  Object.entries(locToSystemContext).forEach(([loc, context]) => {
+    locToKeys[loc] = new Set(context.values.map(v => v.key));
+  });
+
+  rows.forEach(row => {
+    const { key, optionalValue } = row;
+
+    locales.forEach(loc => {
+      if (!locToKeys[loc].has(key)) {
+        const newValue = optionalValue || key;
+        locToSystemContext[loc].values.push({ key, value: newValue });
+        locToKeys[loc].add(key);
+      }
+    });
+  });
+
+  await Promise.all(
+    translations.map(tr => db.collection('translations').replaceOne({ _id: tr._id }, tr))
+  );
+}
+
+export default {
+  delta: 75,
+
+  reindex: false,
+
+  name: 'add_system_key_translations',
+
+  description: 'Adding missing translations for system keys, through importing from a csv file.',
+
+  async up(db) {
+    process.stdout.write(`${this.name}...\r\n`);
+
+    await readCsvToSystemKeys(
+      db,
+      'app/api/migrations/migrations/75-add_system_key_translations/system_keys.csv'
+    );
+  },
+};

app/api/migrations/migrations/75-add_system_key_translations/specs/75-add_system_key_translations.spec.js

@@ -0,0 +1,88 @@
+import { testingDB } from 'api/utils/testing_db';
+import migration from '../index.js';
+import { fixtures, templateId, defaultTemplateName, defaultTemplateTitle } from './fixtures.js';
+
+const locales = ['en', 'es', 'hu'];
+const newKeyValues = [
+  {
+    key: 'Match / Label',
+    value: 'Match / Label',
+  },
+  {
+    key: 'Mismatch / Label',
+    value: 'Mismatch / Label',
+  },
+  {
+    key: 'Match / Value',
+    value: 'Match / Value',
+  },
+  {
+    key: 'Mismatch / Value',
+    value: 'Mismatch / Value',
+  },
+  {
+    key: 'Obsolete',
+    value: 'Obsolete',
+  },
+  {
+    key: 'Empty / Label',
+    value: 'Empty / Label',
+  },
+  {
+    key: 'Empty / Value',
+    value: 'Empty / Value',
+  },
+];
+const alreadyInAllContexts = {
+  key: 'Duplicated label',
+  en: 'Duplicated label',
+  es: 'Nombre duplicado',
+  hu: 'Ismétlődő címke',
+};
+
+describe('migration add_system_key_translations', () => {
+  beforeEach(async () => {
+    spyOn(process.stdout, 'write');
+    await testingDB.clearAllAndLoad(fixtures);
+  });
+
+  afterAll(async () => {
+    await testingDB.disconnect();
+  });
+
+  it('should have a delta number', () => {
+    expect(migration.delta).toBe(75);
+  });
+
+  it('should append new keys, leave existing keys intact.', async () => {
+    await migration.up(testingDB.mongodb);
+
+    const allTranslations = await testingDB.mongodb.collection('translations').find().toArray();
+    function testKeyValue(key, value, locale, contextId) {
+      expect(
+        allTranslations
+          .find(tr => tr.locale === locale)
+          .contexts.find(c => c.id === contextId)
+          .values.find(v => v.key === key).value
+      ).toBe(value);
+    }
+
+    newKeyValues.forEach(({ key, value }) => {
+      locales.forEach(loc => {
+        testKeyValue(key, value, loc, 'System');
+      });
+    });
+    locales.forEach(loc => {
+      testKeyValue(alreadyInAllContexts.key, alreadyInAllContexts[loc], loc, 'System');
+    });
+    locales.forEach(loc => {
+      expect(
+        allTranslations
+          .find(tr => tr.locale === loc)
+          .contexts.find(c => c.id === templateId.toString()).values
+      ).toHaveLength(2);
+      testKeyValue(defaultTemplateName, defaultTemplateName, loc, templateId.toString());
+      testKeyValue(defaultTemplateTitle, defaultTemplateTitle, loc, templateId.toString());
+    });
+  });
+});

app/api/migrations/migrations/75-add_system_key_translations/specs/fixtures.js

@@ -0,0 +1,90 @@
+import db from 'api/utils/testing_db';
+
+const templateId = db.id();
+const defaultTemplateName = 'default template';
+const defaultTemplateTitle = 'Title';
+
+//contexts
+const commonContext = {
+  id: 'System',
+  label: 'User Interface',
+  type: 'Uwazi UI',
+  values: [
+    {
+      key: 'existing-key-in-system',
+      value: 'existing-key-in-system',
+    },
+  ],
+};
+const templateContext = {
+  id: templateId.toString(),
+  label: defaultTemplateName,
+  type: 'Entity',
+  values: [
+    {
+      key: defaultTemplateName,
+      value: defaultTemplateName,
+    },
+    {
+      key: defaultTemplateTitle,
+      value: defaultTemplateTitle,
+    },
+  ],
+};
+
+const fixtures = {
+  templates: [
+    //default template name - correct
+    {
+      _id: templateId,
+      name: defaultTemplateName,
+      commonProperties: [{ name: 'title', label: defaultTemplateTitle, type: 'text' }],
+      properties: [],
+    },
+  ],
+  translations: [
+    {
+      _id: db.id(),
+      locale: 'es',
+      contexts: [
+        {
+          ...commonContext,
+          values: commonContext.values.concat([
+            { key: 'Drag properties here', value: 'Arrastra propiedades aquí' },
+            { key: 'Duplicated label', value: 'Nombre duplicado' },
+          ]),
+        },
+        templateContext,
+      ],
+    },
+    {
+      _id: db.id(),
+      locale: 'en',
+      contexts: [
+        {
+          ...commonContext,
+          values: commonContext.values.concat([
+            { key: 'Priority sorting', value: 'Priority sort' },
+            { key: 'Duplicated label', value: 'Duplicated label' },
+          ]),
+        },
+        templateContext,
+      ],
+    },
+    {
+      _id: db.id(),
+      locale: 'hu',
+      contexts: [
+        {
+          ...commonContext,
+          values: commonContext.values.concat([
+            { key: 'Duplicated label', value: 'Ismétlődő címke' },
+          ]),
+        },
+        templateContext,
+      ],
+    },
+  ],
+};
+
+export { fixtures, templateId, defaultTemplateName, defaultTemplateTitle };

app/api/migrations/migrations/75-add_system_key_translations/system_keys.csv

@@ -0,0 +1,18 @@
+key, optionalValue
+"Match / Label"
+"Mismatch / Label"
+"Match / Value"
+"Mismatch / Value"
+"Obsolete"
+"Empty / Label"
+"Empty / Value"
+"State Legend"
+"Search Tips"
+labelMatchDesc,"It has a current value and a text selection matching with the modelsuggested value and selection. It will be used as a training sample."
+labelMismatchDesc, "It has a current value and text selection but they do not match the model prediction. Accepting the suggestion will replace the current value and text selection with the suggested ones becoming a "match / labeled". If the labeled data is correct and the suggestion is wrong no action is needed. It will be used as a training sample."
+labelEmptyDesc,"Accepting is not available since there is no suggestion.If the value is correct and the suggestion is wrong no action is needed. It will be used as a training sample."
+valueMatchDesc,"It has a current value that matches the suggestion, but it doesn't have a text selection. Accepting will keep the value as it is and enrich it with the suggested text selection becoming a "match / label" that can be used as a training sample."
+valueMismatchDesc, "Accepting the suggestion will replace the current value and text selection with the suggested ones becoming a "match / label" that will be used as a training sample. If the current value is correct, you can still click to fill the text selection so it becomes a "mismatch / label" that will be used as a training sample."
+valueEmptyDesc,"Accepting is not available since there is no suggestion. If the value is correct, you can still click to fill the text selection so it becomes a "mismatch / label" that will be used as a training sample."
+obsoleteDesc,"A new model is training and processing suggestions. This suggestion was created by a previous model so no actions are possible until the new suggestion is received."
+emptyDesc,"Both the current value and the suggestion are empty. You can click to fill the value and text selection so it becomes a "empty / label" that will be used as a training sample."

app/api/services/informationextraction/InformationExtraction.ts

@@ -1,3 +1,5 @@
+/* eslint-disable max-lines */
+/* eslint-disable max-statements */
 import path from 'path';
 import urljoin from 'url-join';
 import _ from 'lodash';
@@ -21,12 +23,12 @@ import { ObjectIdSchema, PropertySchema } from 'shared/types/commonTypes';
 import { TemplateSchema } from 'shared/types/templateType';
 import { IXSuggestionType } from 'shared/types/suggestionType';
 import { FileType } from 'shared/types/fileType';
-import { IXModelsModel } from './IXModelsModel';
 import {
   FileWithAggregation,
   getFilesForTraining,
   getFilesForSuggestions,
 } from 'api/services/informationextraction/getFiles';
+import { IXModelsModel } from './IXModelsModel';
 
 type RawSuggestion = {
   tenant: string;
@@ -35,7 +37,13 @@ type RawSuggestion = {
   xml_file_name: string;
   text: string;
   segment_text: string;
-  page_number: number;
+  segments_boxes: {
+    top: number;
+    left: number;
+    width: number;
+    height: number;
+    page_number: number;
+  }[];
   /* eslint-enable camelcase */
 };
 
@@ -222,7 +230,11 @@ class InformationExtraction {
           status,
           error,
           date: new Date().getTime(),
-          page: rawSuggestion.page_number,
+          selectionRectangles: rawSuggestion.segments_boxes.map((box: any) => {
+            const rect = { ...box, page: box.page_number.toString() };
+            delete rect.page_number;
+            return rect;
+          }),
         };
         return IXSuggestionsModel.save(suggestion);
       })