Merge remote-tracking branch 'origin/release'

app/api/entities/entities.js

@@ -496,7 +496,7 @@ export default {
     await process(0, totalRows);
   },
 
-  getWithoutDocuments(query, select, options = {}) {
+  async getWithoutDocuments(query, select, options = {}) {
     return model.getUnrestricted(query, select, options);
   },
 

app/api/entities/entitiesModel.ts

@@ -1,5 +1,5 @@
-import { instanceModelWithPermissions } from 'api/odm/ModelWithPermissions';
 import mongoose from 'mongoose';
+import { instanceModelWithPermissions } from 'api/odm/ModelWithPermissions';
 import { MetadataObjectSchema, PropertyValueSchema } from 'shared/types/commonTypes';
 import { EntitySchema } from 'shared/types/entityType';
 
@@ -31,6 +31,8 @@ const mongoSchema = new mongoose.Schema(
   { emitIndexErrors: true, minimize: false }
 );
 
+//mongodb types not updated yet for language_override?
+//@ts-ignore
 mongoSchema.index({ title: 'text' }, { language_override: 'mongoLanguage' });
 
 const Model = instanceModelWithPermissions<EntitySchema>('entities', mongoSchema);

app/api/migrations/migrations/19-rename-uploads-to-files/specs/19-rename-uploads-to-files.spec.js

@@ -43,14 +43,19 @@ describe('migration rename-uploads-to-files', () => {
   });
 
   describe('when files already exists', () => {
+    beforeEach(async () => {
+      const collections = await testingDB.mongodb.listCollections().toArray();
+      if (!collections.find(c => c.name === 'files')) {
+        await testingDB.mongodb.createCollection('files');
+      }
+    });
+
     it('should not fail', async () => {
-      await testingDB.mongodb.createCollection('files');
       await migration.up(testingDB.mongodb);
     });
 
     it('should not delete files when uploads does not exists (migration already ran)', async () => {
       await testingDB.mongodb.collection('uploads').drop();
-      await testingDB.mongodb.createCollection('files');
 
       await migration.up(testingDB.mongodb);
       expect((await testingDB.mongodb.listCollections({ name: 'files' }).toArray()).length).toBe(1);

app/api/migrations/migrations/54-add_system_key_translations/index.js

@@ -0,0 +1,63 @@
+import * as fs from 'fs';
+
+import csv from 'api/csv/csv';
+
+/*
+This migration is meant to be repeatable.
+After copy pasting:
+  - change the contents of system_keys.csv to the new keyset
+  - change the file location in the readCsvToSystemKeys call
+  - change the tests, if necessary
+*/
+
+// eslint-disable-next-line max-statements
+async function readCsvToSystemKeys(db, filename) {
+  const fstream = fs.createReadStream(filename);
+  const rows = await csv(fstream).read();
+  fstream.close();
+  const translations = await db
+    .collection('translations')
+    .find()
+    .toArray();
+  const locales = translations.map(tr => tr.locale);
+
+  const locToSystemContext = {};
+  translations.forEach(tr => {
+    locToSystemContext[tr.locale] = tr.contexts.find(c => c.id === 'System');
+  });
+  const locToKeys = {};
+  Object.entries(locToSystemContext).forEach(([loc, context]) => {
+    locToKeys[loc] = new Set(context.values.map(v => v.key));
+  });
+
+  rows.forEach(row => {
+    const { key, optionalValue } = row;
+
+    locales.forEach(loc => {
+      if (!locToKeys[loc].has(key)) {
+        const newValue = optionalValue || key;
+        locToSystemContext[loc].values.push({ key, value: newValue });
+        locToKeys[loc].add(key);
+      }
+    });
+  });
+
+  await translations.forEach(tr => db.collection('translations').replaceOne({ _id: tr._id }, tr));
+}
+
+export default {
+  delta: 54,
+
+  name: 'add_system_key_translations',
+
+  description: 'Adding missing translations for system keys, through importing from a csv file.',
+
+  async up(db) {
+    process.stdout.write(`${this.name}...\r\n`);
+
+    await readCsvToSystemKeys(
+      db,
+      'app/api/migrations/migrations/54-add_system_key_translations/system_keys.csv'
+    );
+  },
+};

app/api/migrations/migrations/54-add_system_key_translations/specs/54-add_system_key_translations.spec.js

@@ -0,0 +1,83 @@
+import testingDB from 'api/utils/testing_db';
+import migration from '../index.js';
+import fixtures, { templateId, defaultTemplateName, defaultTemplateTitle } from './fixtures.js';
+
+const locales = ['en', 'es', 'hu'];
+const newKeyValues = [
+  {
+    key: 'Satellite',
+    value: 'Satellite',
+  },
+  {
+    key: 'Satellite View',
+    value: 'Satellite View',
+  },
+  {
+    key: 'Street',
+    value: 'Street',
+  },
+  {
+    key: 'Street View',
+    value: 'Street View',
+  },
+  {
+    key: 'Switch Map Style',
+    value: 'Switch Map Style',
+  },
+];
+const alreadyInAllContexts = {
+  key: 'Duplicated label',
+  en: 'Duplicated label',
+  es: 'Nombre duplicado',
+  hu: 'Ismétlődő címke',
+};
+
+describe('migration add_system_key_translations', () => {
+  beforeEach(async () => {
+    spyOn(process.stdout, 'write');
+    await testingDB.clearAllAndLoad(fixtures);
+  });
+
+  afterAll(async () => {
+    await testingDB.disconnect();
+  });
+
+  it('should have a delta number', () => {
+    expect(migration.delta).toBe(54);
+  });
+
+  it('should append new keys, leave existing keys intact.', async () => {
+    await migration.up(testingDB.mongodb);
+
+    const allTranslations = await testingDB.mongodb
+      .collection('translations')
+      .find()
+      .toArray();
+    function testKeyValue(key, value, locale, contextId) {
+      expect(
+        allTranslations
+          .find(tr => tr.locale === locale)
+          .contexts.find(c => c.id === contextId)
+          .values.find(v => v.key === key).value
+      ).toBe(value);
+    }
+
+    newKeyValues.forEach(({ key, value }) => {
+      locales.forEach(loc => {
+        testKeyValue(key, value, loc, 'System');
+      });
+    });
+    locales.forEach(loc => {
+      testKeyValue(alreadyInAllContexts.key, alreadyInAllContexts[loc], loc, 'System');
+    });
+    locales.forEach(loc => {
+      expect(
+        allTranslations
+          .find(tr => tr.locale === loc)
+          .contexts.find(c => c.id === templateId.toString()).values
+      ).toHaveLength(2);
+      testKeyValue(defaultTemplateName, defaultTemplateName, loc, templateId.toString());
+      testKeyValue(defaultTemplateTitle, defaultTemplateTitle, loc, templateId.toString());
+    });
+  });
+});

app/api/migrations/migrations/54-add_system_key_translations/specs/fixtures.js

@@ -0,0 +1,88 @@
+import db from 'api/utils/testing_db';
+
+export const templateId = db.id();
+export const defaultTemplateName = 'default template';
+export const defaultTemplateTitle = 'Title';
+
+//contexts
+const commonContext = {
+  id: 'System',
+  label: 'User Interface',
+  type: 'Uwazi UI',
+  values: [
+    {
+      key: 'existing-key-in-system',
+      value: 'existing-key-in-system',
+    },
+  ],
+};
+const templateContext = {
+  id: templateId.toString(),
+  label: defaultTemplateName,
+  type: 'Entity',
+  values: [
+    {
+      key: defaultTemplateName,
+      value: defaultTemplateName,
+    },
+    {
+      key: defaultTemplateTitle,
+      value: defaultTemplateTitle,
+    },
+  ],
+};
+
+export default {
+  templates: [
+    //default template name - correct
+    {
+      _id: templateId,
+      name: defaultTemplateName,
+      commonProperties: [{ name: 'title', label: defaultTemplateTitle, type: 'text' }],
+      properties: [],
+    },
+  ],
+  translations: [
+    {
+      _id: db.id(),
+      locale: 'es',
+      contexts: [
+        {
+          ...commonContext,
+          values: commonContext.values.concat([
+            { key: 'Drag properties here', value: 'Arrastra propiedades aquí' },
+            { key: 'Duplicated label', value: 'Nombre duplicado' },
+          ]),
+        },
+        templateContext,
+      ],
+    },
+    {
+      _id: db.id(),
+      locale: 'en',
+      contexts: [
+        {
+          ...commonContext,
+          values: commonContext.values.concat([
+            { key: 'Priority sorting', value: 'Priority sort' },
+            { key: 'Duplicated label', value: 'Duplicated label' },
+          ]),
+        },
+        templateContext,
+      ],
+    },
+    {
+      _id: db.id(),
+      locale: 'hu',
+      contexts: [
+        {
+          ...commonContext,
+          values: commonContext.values.concat([
+            { key: 'Duplicated label', value: 'Ismétlődő címke' },
+          ]),
+        },
+        templateContext,
+      ],
+    },
+  ],
+};

app/api/migrations/migrations/54-add_system_key_translations/system_keys.csv

@@ -0,0 +1,6 @@
+key, optionalValue
+"Satellite"
+"Street"
+"Satellite View"
+"Street View"
+"Switch Map Style"

app/api/migrations/migrations/55-replace_dictionary_with_thesaurus/index.js

@@ -0,0 +1,31 @@
+export default {
+  delta: 55,
+
+  name: 'replace_dictionary_with_thesaurus',
+
+  description: 'Replaces dictionary translation types  with thesaurus in translations',
+
+  async up(db) {
+    const cursor = await db.collection('translations').find({});
+
+    // eslint-disable-next-line no-await-in-loop
+    while (await cursor.hasNext()) {
+      // eslint-disable-next-line no-await-in-loop
+      const translation = await cursor.next();
+      const newContexts = [];
+      translation.contexts.forEach(context => {
+        const newContext = context;
+        if (context.type === 'Dictionary') {
+          newContext.type = 'Thesaurus';
+        }
+        newContexts.push(newContext);
+      });
+
+      // eslint-disable-next-line no-await-in-loop
+      await db
+        .collection('translations')
+        .updateOne({ _id: translation._id }, { $set: { contexts: newContexts } });
+    }
+    process.stdout.write(`${this.name}...\r\n`);
+  },
+};

app/api/migrations/migrations/55-replace_dictionary_with_thesaurus/specs/55-replace_dictionary_with_thesaurus.spec.js

@@ -0,0 +1,40 @@
+import testingDB from 'api/utils/testing_db';
+import migration from '../index.js';
+import fixtures from './fixtures.js';
+
+describe('migration replace_dictionary_with_thesaurus', () => {
+  let translations;
+  let contexts = [];
+
+  beforeEach(async () => {
+    //spyOn(process.stdout, 'write');
+    await testingDB.clearAllAndLoad(fixtures);
+    await migration.up(testingDB.mongodb);
+    translations = await testingDB.mongodb
+      .collection('translations')
+      .find({})
+      .toArray();
+    translations.forEach(translation => {
+      contexts = contexts.concat(translation.contexts);
+    });
+  });
+
+  afterEach(() => {
+    contexts = [];
+  });
+
+  afterAll(async () => {
+    await testingDB.tearDown();
+  });
+
+  it('should have a delta number', () => {
+    expect(migration.delta).toBe(55);
+  });
+
+  it('should update the correct translations', async () => {
+    expect(contexts.filter(context => context.type === 'Thesaurus').length).toEqual(2);
+  });
+  it('should not update the translations that have no Dictionary context', async () => {
+    expect(contexts.filter(context => context.type !== 'Thesaurus').length).toEqual(2);
+  });
+});