Move dateToSeconds function behind an endpoint

app/api/entities/routes.js

@@ -6,6 +6,7 @@ import { saveEntity } from 'api/entities/entitySavingManager';
 import entities from './entities';
 import templates from '../templates/templates';
 import thesauri from '../thesauri/thesauri';
+import date from '../utils/date';
 import needsAuthorization from '../auth/authMiddleware';
 import { parseQuery, validation } from '../utils';
 
@@ -22,7 +23,47 @@ async function updateThesauriWithEntity(entity, req) {
   req.sockets.emitToCurrentTenant('thesauriChange', templateTransformed);
 }
 
+function coerceValues(value, type, locale) {
+  let dateSeconds = '';
+  switch (type) {
+    case 'date':
+      dateSeconds = date.dateToSeconds(value, locale);
+      if (Number.isNaN(dateSeconds)) {
+        return { success: false };
+      }
+      return { success: true, value: dateSeconds };
+    default:
+      throw Error('Unsupported type');
+  }
+}
+
 export default app => {
+  app.post(
+    '/api/entities/coerce_value',
+    needsAuthorization(['admin']),
+    validation.validateRequest({
+      type: 'object',
+      properties: {
+        body: {
+          type: 'object',
+          properties: {
+            value: { type: 'string' },
+            type: { type: 'string' },
+            locale: { type: 'string' },
+          },
+        },
+      },
+    }),
+    async (req, res, next) => {
+      const { value, type, locale } = req.body;
+      try {
+        const coerced = coerceValues(value, type, locale);
+        return res.json(coerced);
+      } catch (e) {
+        return next(e);
+      }
+    }
+  );
   app.post(
     '/api/entities',
     needsAuthorization(['admin', 'editor', 'collaborator']),

app/api/migrations/migrations/90-add_system_key_translations/index.js

@@ -0,0 +1,57 @@
+/*
+This migration is meant to be repeatable.
+After copy pasting:
+  - change the contents of system_keys.csv to the new keyset
+  - change the file location in the readCsvToSystemKeys call
+  - change the tests, if necessary
+*/
+
+async function insertSystemKeys(db, newKeys) {
+  const translations = await db.collection('translations').find().toArray();
+  const locales = translations.map(tr => tr.locale);
+
+  const locToSystemContext = {};
+  translations.forEach(tr => {
+    locToSystemContext[tr.locale] = tr.contexts.find(c => c.id === 'System');
+  });
+  const locToKeys = {};
+  Object.entries(locToSystemContext).forEach(([loc, context]) => {
+    locToKeys[loc] = new Set(context.values.map(v => v.key));
+  });
+
+  newKeys.forEach(row => {
+    const { key, value: optionalValue } = row;
+
+    locales.forEach(loc => {
+      if (!locToKeys[loc].has(key)) {
+        const newValue = optionalValue || key;
+        locToSystemContext[loc].values.push({ key, value: newValue });
+        locToKeys[loc].add(key);
+      }
+    });
+  });
+
+  await Promise.all(
+    translations.map(tr => db.collection('translations').replaceOne({ _id: tr._id }, tr))
+  );
+}
+
+export default {
+  delta: 90,
+
+  reindex: false,
+
+  name: 'add_system_key_translations',
+
+  description: 'Adding missing translations for system keys.',
+
+  async up(db) {
+    process.stdout.write(`${this.name}...\r\n`);
+    const systemKeys = [
+      {
+        key: 'Value cannot be transformed to date',
+      },
+    ];
+    await insertSystemKeys(db, systemKeys);
+  },
+};

app/api/migrations/migrations/90-add_system_key_translations/specs/90-add_system_key_translations.spec.js

@@ -0,0 +1,64 @@
+import { testingDB } from 'api/utils/testing_db';
+import migration from '../index.js';
+import { fixtures, templateId, defaultTemplateName, defaultTemplateTitle } from './fixtures.js';
+
+const locales = ['en', 'es', 'hu'];
+const newKeyValues = [
+  {
+    key: 'Value cannot be transformed to date',
+    value: 'Value cannot be transformed to date',
+  },
+];
+const alreadyInAllContexts = {
+  key: 'Duplicated label',
+  en: 'Duplicated label',
+  es: 'Nombre duplicado',
+  hu: 'Ismétlődő címke',
+};
+
+describe('migration add_system_key_translations', () => {
+  beforeEach(async () => {
+    spyOn(process.stdout, 'write');
+    await testingDB.setupFixturesAndContext(fixtures);
+  });
+
+  afterAll(async () => {
+    await testingDB.disconnect();
+  });
+
+  it('should have a delta number', () => {
+    expect(migration.delta).toBe(90);
+  });
+
+  it('should append new keys, leave existing keys intact.', async () => {
+    await migration.up(testingDB.mongodb);
+
+    const allTranslations = await testingDB.mongodb.collection('translations').find().toArray();
+    function testKeyValue(key, value, locale, contextId) {
+      expect(
+        allTranslations
+          .find(tr => tr.locale === locale)
+          .contexts.find(c => c.id === contextId)
+          .values.find(v => v.key === key).value
+      ).toBe(value);
+    }
+
+    newKeyValues.forEach(({ key, value }) => {
+      locales.forEach(loc => {
+        testKeyValue(key, value, loc, 'System');
+      });
+    });
+    locales.forEach(loc => {
+      testKeyValue(alreadyInAllContexts.key, alreadyInAllContexts[loc], loc, 'System');
+    });
+    locales.forEach(loc => {
+      expect(
+        allTranslations
+          .find(tr => tr.locale === loc)
+          .contexts.find(c => c.id === templateId.toString()).values
+      ).toHaveLength(2);
+      testKeyValue(defaultTemplateName, defaultTemplateName, loc, templateId.toString());
+      testKeyValue(defaultTemplateTitle, defaultTemplateTitle, loc, templateId.toString());
+    });
+  });
+});

app/api/migrations/migrations/90-add_system_key_translations/specs/fixtures.js

@@ -0,0 +1,90 @@
+import db from 'api/utils/testing_db';
+
+const templateId = db.id();
+const defaultTemplateName = 'default template';
+const defaultTemplateTitle = 'Title';
+
+//contexts
+const commonContext = {
+  id: 'System',
+  label: 'User Interface',
+  type: 'Uwazi UI',
+  values: [
+    {
+      key: 'existing-key-in-system',
+      value: 'existing-key-in-system',
+    },
+  ],
+};
+const templateContext = {
+  id: templateId.toString(),
+  label: defaultTemplateName,
+  type: 'Entity',
+  values: [
+    {
+      key: defaultTemplateName,
+      value: defaultTemplateName,
+    },
+    {
+      key: defaultTemplateTitle,
+      value: defaultTemplateTitle,
+    },
+  ],
+};
+
+const fixtures = {
+  templates: [
+    //default template name - correct
+    {
+      _id: templateId,
+      name: defaultTemplateName,
+      commonProperties: [{ name: 'title', label: defaultTemplateTitle, type: 'text' }],
+      properties: [],
+    },
+  ],
+  translations: [
+    {
+      _id: db.id(),
+      locale: 'es',
+      contexts: [
+        {
+          ...commonContext,
+          values: commonContext.values.concat([
+            { key: 'Drag properties here', value: 'Arrastra propiedades aquí' },
+            { key: 'Duplicated label', value: 'Nombre duplicado' },
+          ]),
+        },
+        templateContext,
+      ],
+    },
+    {
+      _id: db.id(),
+      locale: 'en',
+      contexts: [
+        {
+          ...commonContext,
+          values: commonContext.values.concat([
+            { key: 'Priority sorting', value: 'Priority sort' },
+            { key: 'Duplicated label', value: 'Duplicated label' },
+          ]),
+        },
+        templateContext,
+      ],
+    },
+    {
+      _id: db.id(),
+      locale: 'hu',
+      contexts: [
+        {
+          ...commonContext,
+          values: commonContext.values.concat([
+            { key: 'Duplicated label', value: 'Ismétlődő címke' },
+          ]),
+        },
+        templateContext,
+      ],
+    },
+  ],
+};
+
+export { fixtures, templateId, defaultTemplateName, defaultTemplateTitle };

app/api/services/informationextraction/InformationExtraction.ts

@@ -22,7 +22,7 @@ import { EntitySchema } from 'shared/types/entityType';
 import { ObjectIdSchema, PropertySchema } from 'shared/types/commonTypes';
 import { IXSuggestionType } from 'shared/types/suggestionType';
 import { FileType } from 'shared/types/fileType';
-import { dateToSeconds } from 'shared/dataUtils';
+import date from 'api/utils/date';
 import {
   FileWithAggregation,
   getFilesForTraining,
@@ -173,13 +173,17 @@ class InformationExtraction {
     return this._getEntityFromFile(file);
   };
 
-  coerceSuggestionValue = (suggestion: RawSuggestion, property?: PropertySchema) => {
+  coerceSuggestionValue = (
+    suggestion: RawSuggestion,
+    property?: PropertySchema,
+    language?: string
+  ) => {
     const suggestedValue = suggestion.text.trim();
     switch (property?.type) {
       case 'numeric':
         return parseFloat(suggestedValue) || null;
       case 'date':
-        return dateToSeconds(suggestedValue);
+        return date.dateToSeconds(suggestedValue, language);
       default:
         return suggestedValue;
     }
@@ -219,7 +223,11 @@ class InformationExtraction {
         );
         const property = allProps.find(p => p.name === rawSuggestion.property_name);
 
-        const suggestedValue = this.coerceSuggestionValue(rawSuggestion, property);
+        const suggestedValue = this.coerceSuggestionValue(
+          rawSuggestion,
+          property,
+          currentSuggestion?.language || entity.language
+        );
 
         if (suggestedValue === null) {
           status = 'failed';

app/api/utils/date.js

@@ -1,4 +1,6 @@
 import moment from 'moment';
+// @ts-ignore
+import parser from 'any-date-parser';
 
 export default {
   currentUTC() {
@@ -23,4 +25,14 @@ export default {
     newDate.setFullYear(newDate.getFullYear() + yearsToAdd);
     return newDate;
   },
+
+  dateToSeconds(value, locale) {
+    const parsedValue = value.normalize('NFD').replace(/[\u0300-\u036f]/g, '');
+    let getDate = parser.fromString(parsedValue, locale);
+    if (getDate.invalid) {
+      getDate = Date.parse(`${parsedValue} GMT`);
+    }
+    const formattedDate = getDate / 1000;
+    return formattedDate;
+  },
 };

app/api/utils/specs/date.spec.js

@@ -32,4 +32,30 @@ describe('date helper', () => {
     const newDate = date.addYearsToCurrentDate(1);
     expect(newDate).toEqual(nextYear);
   });
+  describe('dateToSeconds', () => {
+    it.each`
+      stringDate                     | language     | expectedMilliseconds
+      ${'5-2-2020'}                  | ${'en-GB'}   | ${1580860800}
+      ${'5-2-2020'}                  | ${'en-US'}   | ${1588377600}
+      ${'2020-01-01'}                | ${undefined} | ${1577836800}
+      ${'17-05-2018'}                | ${undefined} | ${1526515200}
+      ${'17/05/2018'}                | ${undefined} | ${1526515200}
+      ${'11-21-1982'}                | ${undefined} | ${406684800}
+      ${'05-17-2018'}                | ${undefined} | ${1526515200}
+      ${'September 30, 1999'}        | ${undefined} | ${938649600}
+      ${'13 October 2012'}           | ${undefined} | ${1350086400}
+      ${'13th October 2012'}         | ${undefined} | ${1350086400}
+      ${'October 13th, 2012'}        | ${undefined} | ${1350086400}
+      ${'22 decembre 2012'}          | ${undefined} | ${1356134400}
+      ${'mercredi 22 décembre 2010'} | ${undefined} | ${1292976000}
+      ${'Marzo 14, 2006'}            | ${undefined} | ${1142294400}
+      ${'28 августа 2017'}           | ${'ru'}      | ${1503878400}
+      ${'августа 28, 2017'}          | ${'ru'}      | ${1503878400}
+    `(
+      'should convert $stringDate to $expectedMilliseconds',
+      ({ stringDate, language, expectedMilliseconds }) => {
+        expect(date.dateToSeconds(stringDate, language)).toBe(expectedMilliseconds);
+      }
+    );
+  });
 });

app/react/Entities/EntitiesAPI.js

@@ -2,6 +2,11 @@ import api from 'app/utils/api';
 import { RequestParams } from 'app/utils/RequestParams';
 
 export default {
+  coerceValue(requestParams) {
+    const url = 'entities/coerce_value';
+    return api.post(url, requestParams).then(response => response.json);
+  },
+
   get(requestParams = new RequestParams(), language) {
     const params = requestParams.add({
       include: