Added Preserve integration main UI (#4760)

* Added main UI

* Added main UI

* Update creation of templates

* Added main UI

* Update creation of templates

* Put preserve behind a feature toggle

* Added some tests

* Added more tests

* Reimplemented to consider multiple users

* Updated tests

* Updated preserve UI to match data returned by preserve service

* Added a translations migration

* Added masterToken field in preserveSync fixtures

* Updated tests

app/api/api.js

@@ -40,4 +40,5 @@ export default (app, server) => {
   require('./usergroups/routes').default(app);
   require('./permissions/routes').permissionRoutes(app);
   require('./suggestions/routes').suggestionsRoutes(app);
+  require('./preserve/routes').PreserveRoutes(app);
 };

app/api/migrations/migrations/85-add_system_key_translations/index.js

@@ -0,0 +1,64 @@
+//eslint-disable-next-line node/no-restricted-import
+import * as fs from 'fs';
+
+import csv from 'api/csv/csv';
+
+/*
+This migration is meant to be repeatable.
+After copy pasting:
+  - change the contents of system_keys.csv to the new keyset
+  - change the file location in the readCsvToSystemKeys call
+  - change the tests, if necessary
+*/
+
+// eslint-disable-next-line max-statements
+async function readCsvToSystemKeys(db, filename) {
+  const fstream = fs.createReadStream(filename);
+  const rows = await csv(fstream).read();
+  fstream.close();
+  const translations = await db.collection('translations').find().toArray();
+  const locales = translations.map(tr => tr.locale);
+
+  const locToSystemContext = {};
+  translations.forEach(tr => {
+    locToSystemContext[tr.locale] = tr.contexts.find(c => c.id === 'System');
+  });
+  const locToKeys = {};
+  Object.entries(locToSystemContext).forEach(([loc, context]) => {
+    locToKeys[loc] = new Set(context.values.map(v => v.key));
+  });
+
+  rows.forEach(row => {
+    const { key, optionalValue } = row;
+
+    locales.forEach(loc => {
+      if (!locToKeys[loc].has(key)) {
+        const newValue = optionalValue || key;
+        locToSystemContext[loc].values.push({ key, value: newValue });
+        locToKeys[loc].add(key);
+      }
+    });
+  });
+
+  await Promise.all(
+    translations.map(tr => db.collection('translations').replaceOne({ _id: tr._id }, tr))
+  );
+}
+
+export default {
+  delta: 85,
+  reindex: false,
+
+  name: 'add_system_key_translations',
+
+  description: 'Adding missing translations for system keys, through importing from a csv file.',
+
+  async up(db) {
+    process.stdout.write(`${this.name}...\r\n`);
+
+    await readCsvToSystemKeys(
+      db,
+      'app/api/migrations/migrations/85-add_system_key_translations/system_keys.csv'
+    );
+  },
+};

app/api/migrations/migrations/85-add_system_key_translations/specs/85-add_system_key_translations.spec.js

@@ -0,0 +1,85 @@
+import { testingDB } from 'api/utils/testing_db';
+import migration from '../index.js';
+import { fixtures, templateId, defaultTemplateName, defaultTemplateTitle } from './fixtures.js';
+
+const locales = ['en', 'es', 'hu'];
+const newKeyValues = [
+  { key: 'Preserve Extension', value: 'Preserve Extension' },
+  {
+    key: 'You have not connected an Uwazi instance, yet',
+    value: 'You have not connected an Uwazi instance, yet',
+  },
+  {
+    key: 'INSTALL the browser extension',
+    value: 'INSTALL the browser extension',
+  },
+  {
+    key: 'Preserve Setup Description',
+    value:
+      'If you know your Uwazi URL and TOKEN click the link below, and fill the required information.',
+  },
+  {
+    key: 'Install browser extension (dynamic link)',
+    value: 'Install browser extension (dynamic link)',
+  },
+  { key: 'Chrome extension store link', value: 'Chrome extension store link' },
+  { key: 'Firefox extension store link', value: 'Firefox extension store link' },
+  { key: 'Configuration', value: 'Configuration' },
+  { key: 'Extension Token', value: 'Extension Token' },
+  { key: 'Copy token', value: 'Copy token' },
+  { key: 'Request token', value: 'Request token' },
+  { key: 'Some information about the token', value: 'Some information about the token' },
+];
+const alreadyInAllContexts = {
+  key: 'Duplicated label',
+  en: 'Duplicated label',
+  es: 'Nombre duplicado',
+  hu: 'Ismétlődő címke',
+};
+
+describe('migration add_system_key_translations', () => {
+  beforeEach(async () => {
+    spyOn(process.stdout, 'write');
+    await testingDB.setupFixturesAndContext(fixtures);
+  });
+
+  afterAll(async () => {
+    await testingDB.disconnect();
+  });
+
+  it('should have a delta number', () => {
+    expect(migration.delta).toBe(85);
+  });
+
+  it('should append new keys, leave existing keys intact.', async () => {
+    await migration.up(testingDB.mongodb);
+
+    const allTranslations = await testingDB.mongodb.collection('translations').find().toArray();
+    function testKeyValue(key, value, locale, contextId) {
+      expect(
+        allTranslations
+          .find(tr => tr.locale === locale)
+          .contexts.find(c => c.id === contextId)
+          .values.find(v => v.key === key).value
+      ).toBe(value);
+    }
+
+    newKeyValues.forEach(({ key, value }) => {
+      locales.forEach(loc => {
+        testKeyValue(key, value, loc, 'System');
+      });
+    });
+    locales.forEach(loc => {
+      testKeyValue(alreadyInAllContexts.key, alreadyInAllContexts[loc], loc, 'System');
+    });
+    locales.forEach(loc => {
+      expect(
+        allTranslations
+          .find(tr => tr.locale === loc)
+          .contexts.find(c => c.id === templateId.toString()).values
+      ).toHaveLength(2);
+      testKeyValue(defaultTemplateName, defaultTemplateName, loc, templateId.toString());
+      testKeyValue(defaultTemplateTitle, defaultTemplateTitle, loc, templateId.toString());
+    });
+  });
+});

app/api/migrations/migrations/85-add_system_key_translations/specs/fixtures.js

@@ -0,0 +1,90 @@
+import db from 'api/utils/testing_db';
+
+const templateId = db.id();
+const defaultTemplateName = 'default template';
+const defaultTemplateTitle = 'Title';
+
+//contexts
+const commonContext = {
+  id: 'System',
+  label: 'User Interface',
+  type: 'Uwazi UI',
+  values: [
+    {
+      key: 'existing-key-in-system',
+      value: 'existing-key-in-system',
+    },
+  ],
+};
+const templateContext = {
+  id: templateId.toString(),
+  label: defaultTemplateName,
+  type: 'Entity',
+  values: [
+    {
+      key: defaultTemplateName,
+      value: defaultTemplateName,
+    },
+    {
+      key: defaultTemplateTitle,
+      value: defaultTemplateTitle,
+    },
+  ],
+};
+
+const fixtures = {
+  templates: [
+    //default template name - correct
+    {
+      _id: templateId,
+      name: defaultTemplateName,
+      commonProperties: [{ name: 'title', label: defaultTemplateTitle, type: 'text' }],
+      properties: [],
+    },
+  ],
+  translations: [
+    {
+      _id: db.id(),
+      locale: 'es',
+      contexts: [
+        {
+          ...commonContext,
+          values: commonContext.values.concat([
+            { key: 'Drag properties here', value: 'Arrastra propiedades aquí' },
+            { key: 'Duplicated label', value: 'Nombre duplicado' },
+          ]),
+        },
+        templateContext,
+      ],
+    },
+    {
+      _id: db.id(),
+      locale: 'en',
+      contexts: [
+        {
+          ...commonContext,
+          values: commonContext.values.concat([
+            { key: 'Priority sorting', value: 'Priority sort' },
+            { key: 'Duplicated label', value: 'Duplicated label' },
+          ]),
+        },
+        templateContext,
+      ],
+    },
+    {
+      _id: db.id(),
+      locale: 'hu',
+      contexts: [
+        {
+          ...commonContext,
+          values: commonContext.values.concat([
+            { key: 'Duplicated label', value: 'Ismétlődő címke' },
+          ]),
+        },
+        templateContext,
+      ],
+    },
+  ],
+};
+
+export { fixtures, templateId, defaultTemplateName, defaultTemplateTitle };

app/api/migrations/migrations/85-add_system_key_translations/system_keys.csv

@@ -0,0 +1,13 @@
+key,optionalValue,
+"Preserve Extension","Preserve Extension",
+"You have not connected an Uwazi instance, yet","You have not connected an Uwazi instance, yet",
+"INSTALL the browser extension","INSTALL the browser extension",
+"Preserve Setup Description","If you know your Uwazi URL and TOKEN click the link below, and fill the required information.",
+"Install browser extension (dynamic link)","Install browser extension (dynamic link)",
+"Chrome extension store link","Chrome extension store link",
+"Firefox extension store link","Firefox extension store link",
+"Configuration","Configuration",
+"Extension Token","Extension Token",
+"Copy token","Copy token",
+"Request token","Request token",
+"Some information about the token","Some information about the token",

app/api/preserve/preserve.ts

@@ -0,0 +1,98 @@
+import { WithId } from 'api/odm';
+import thesauri from 'api/thesauri';
+import settings from 'api/settings';
+import templates from 'api/templates';
+import request from 'shared/JSONRequest';
+import createError from 'api/utils/Error';
+import { ThesaurusSchema } from 'shared/types/thesaurusType';
+import { TemplateSchema } from 'shared/types/templateType';
+import { User } from 'api/users/usersModel';
+import { PreserveConfig } from 'shared/types/settingsType';
+import { ObjectIdSchema } from 'shared/types/commonTypes';
+
+export const Preserve = {
+  async setup(language: string, user: User) {
+    const currentSettings: any = await settings.get({});
+    const preserve: PreserveConfig | undefined = currentSettings?.features?.preserve;
+
+    if (!preserve) {
+      throw createError('Preserve configuration not found', 402);
+    }
+
+    let userConfig = preserve.config?.find(conf => conf.user?.toString() === user._id.toString());
+
+    if (userConfig) {
+      return userConfig;
+    }
+
+    userConfig = await this.createUserConfig(preserve, language, user);
+
+    await settings.save({
+      ...currentSettings,
+      features: {
+        ...currentSettings.features,
+        preserve: {
+          host: preserve.host,
+          masterToken: preserve.masterToken,
+          config: [...(preserve.config || []), userConfig],
+        },
+      },
+    });
+    return userConfig;
+  },
+
+  async createUserConfig(preserve: PreserveConfig, language: string, user: User) {
+    let templateId: ObjectIdSchema;
+    if (preserve.config?.length) {
+      templateId = preserve.config[0].template;
+    } else {
+      templateId = await (await this.createTemplate(language))!._id;
+    }
+    const token = await this.requestToken(preserve.host, {
+      Authorization: preserve.masterToken,
+    });
+
+    return {
+      template: templateId,
+      token,
+      user: user._id,
+    };
+  },
+
+  async requestToken(host: string, headers: { [key: string]: string }) {
+    const resp = await request.post(`${host}/api/tokens`, {}, headers);
+    return resp.json.data.token;
+  },
+
+  async createTemplate(language: string) {
+    const fetchedThesauri = await Preserve.createEmptyThesauri();
+    const toSave: TemplateSchema = {
+      name: 'Preserve',
+      commonProperties: [
+        { label: 'Title', name: 'title', type: 'text' },
+        { name: 'creationDate', label: 'Date added', type: 'date' },
+        { name: 'editDate', label: 'Date modified', type: 'date' },
+      ],
+      properties: [
+        { type: 'link', name: 'url', label: 'Url' },
+        {
+          type: 'select',
+          name: 'source',
+          label: 'Source',
+          content: fetchedThesauri._id.toString(),
+        },
+      ],
+    };
+    return templates.save(toSave, language);
+  },
+
+  async createEmptyThesauri(name?: string): Promise<WithId<ThesaurusSchema>> {
+    const internalName = name || 'Preserve';
+    const toSave = {
+      name: internalName,
+      values: [],
+    };
+    const createdThesauri = await thesauri.save(toSave);
+    return createdThesauri;
+  },
+};

app/api/preserve/routes.ts

@@ -0,0 +1,9 @@
+import { Application } from 'express';
+import { Preserve } from './preserve';
+
+export const PreserveRoutes = (app: Application) => {
+  app.post('/api/preserve/', async (req, res, _next) => {
+    const config = await Preserve.setup(req.language, req.user);
+    res.json({ ...config });
+  });
+};