Created script to re-save all entities.

app/api/entities/denormalize.ts

@@ -1,6 +1,6 @@
 /* eslint-disable max-lines */
 import { WithId } from 'api/odm';
-import translationsModel from 'api/i18n/translations';
+import translationsModel, { IndexedTranslations } from 'api/i18n/translations';
 import { search } from 'api/search';
 import templates from 'api/templates';
 import dictionariesModel from 'api/thesauri/dictionariesModel';
@@ -263,8 +263,8 @@ const denormalizeThesauriLabelInMetadata = async (
 const denormalizeSelectProperty = async (
   property: PropertySchema,
   values: MetadataObjectSchema[],
-  thesauriByKey: Record<string, ThesaurusSchema>,
-  translation: unknown
+  thesauriByKey?: Record<string, ThesaurusSchema>,
+  translation?: unknown
 ) => {
   const thesaurus = thesauriByKey
     ? thesauriByKey[property.content!]
@@ -382,7 +382,7 @@ const denormalizeProperty = async (
     translation,
     allTemplates,
   }: {
-    thesauriByKey: Record<string, ThesaurusSchema>;
+    thesauriByKey?: Record<string, ThesaurusSchema>;
     translation: unknown;
     allTemplates: TemplateSchema[];
   }
@@ -407,20 +407,20 @@ const denormalizeProperty = async (
 async function denormalizeMetadata(
   metadata: MetadataSchema,
   language: LanguageISO6391,
-  templateId: string,
-  thesauriByKey: Record<string, ThesaurusSchema>
+  template: TemplateSchema,
+  preloadedData: {
+    allTemplates?: TemplateSchema[];
+    translation?: IndexedTranslations;
+    thesauriByKey?: Record<string, ThesaurusSchema>;
+  } = {}
 ) {
-  if (!metadata) {
+  if (!metadata || !template) {
     return metadata;
   }
 
-  const translation = (await translationsModel.get({ locale: language }))[0];
-  const allTemplates = await templates.get();
-
-  const template = allTemplates.find(t => t._id.toString() === templateId);
-  if (!template) {
-    return metadata;
-  }
+  const allTemplates = preloadedData.allTemplates || (await templates.get());
+  const translation =
+    preloadedData.translation || (await translationsModel.get({ locale: language }))[0];
 
   const denormalizedProperties: {
     propertyName: string;
@@ -432,7 +432,7 @@ async function denormalizeMetadata(
         template.properties?.find(p => p.name === propertyName),
         metadata[propertyName],
         language,
-        { thesauriByKey, translation, allTemplates }
+        { thesauriByKey: preloadedData.thesauriByKey, translation, allTemplates }
       ),
     }))
   );

app/api/entities/entities.js

@@ -75,20 +75,17 @@ async function updateEntity(entity, _template, unrestricted = false) {
         delete toSave.permissions;
 
         if (entity.metadata) {
-          toSave.metadata = await denormalizeMetadata(
-            entity.metadata,
-            d.language,
-            template._id.toString(),
-            thesauriByKey
-          );
+          toSave.metadata = await denormalizeMetadata(entity.metadata, d.language, template, {
+            thesauriByKey,
+          });
         }
 
         if (entity.suggestedMetadata) {
           toSave.suggestedMetadata = await denormalizeMetadata(
             entity.suggestedMetadata,
             entity.language,
-            template._id.toString(),
-            thesauriByKey
+            template,
+            { thesauriByKey }
           );
         }
 
@@ -124,8 +121,8 @@ async function updateEntity(entity, _template, unrestricted = false) {
           toSave[metadataParent] = await denormalizeMetadata(
             toSave[metadataParent],
             toSave.language,
-            template._id.toString(),
-            thesauriByKey
+            template,
+            { thesauriByKey }
           );
         }
       }, Promise.resolve());
@@ -187,15 +184,15 @@ async function createEntity(doc, [currentLanguage, languages], sharedId, docTemp
       langDoc.metadata = await denormalizeMetadata(
         langDoc.metadata,
         langDoc.language,
-        langDoc.template.toString(),
-        thesauriByKey
+        docTemplate,
+        { thesauriByKey }
       );
 
       langDoc.suggestedMetadata = await denormalizeMetadata(
         langDoc.suggestedMetadata,
         langDoc.language,
-        langDoc.template.toString(),
-        thesauriByKey
+        docTemplate,
+        { thesauriByKey }
       );
 
       return model.save(langDoc);
@@ -464,15 +461,11 @@ export default {
       docTemplate = defaultTemplate;
     }
     const entity = this.sanitize(doc, docTemplate);
-    entity.metadata = await denormalizeMetadata(
-      entity.metadata,
-      entity.language,
-      entity.template.toString()
-    );
+    entity.metadata = await denormalizeMetadata(entity.metadata, entity.language, docTemplate);
     entity.suggestedMetadata = await denormalizeMetadata(
       entity.suggestedMetadata,
       entity.language,
-      entity.template.toString()
+      docTemplate
     );
     return entity;
   },

docker-compose.yml

@@ -24,14 +24,18 @@ services:
       - mongodata:/data/db
     ports:
       - '27017:27017'
+    ulimits:
+      nofile:
+        soft: "65536"
+        hard: "65536"
 
   mongoreplicaset_start_script:
     image: "mongo:5.0.27"
     restart: "no"
     container_name: mongoreplicaset_start_script
-    depends_on: 
+    depends_on:
       - mongo
-    entrypoint: [ "bash", "-c", "sleep 1 && mongo --host mongo:27017 --eval 'rs.initiate();cfg = rs.config(); cfg.members[0].host = \"localhost:27017\";rs.reconfig(cfg, {force:true})'"]   
+    entrypoint: [ "bash", "-c", "sleep 1 && mongo --host mongo:27017 --eval 'rs.initiate();cfg = rs.config(); cfg.members[0].host = \"localhost:27017\";rs.reconfig(cfg, {force:true})'"]
 
   redis:
     image: 'redis:5.0.14'

package.json

@@ -31,6 +31,7 @@
     "dev-worker": "node --no-experimental-fetch ./scripts/run.js ../app/worker.ts",
     "dev-queue": "node --no-experimental-fetch ./scripts/run.js ../app/queueWorker.ts",
     "new-files-healthcheck": "tsx ./scripts/scripts.v2/filesHealthCheck.ts",
+    "denormalize-all-entities": "tsx ./scripts/scripts.v2/denormalizeAllEntities.ts",
     "generateAutomaticTranslationConfig": "node --no-experimental-fetch ./scripts/run.js ../scripts/scripts.v2/generateAutomaticTranslationConfig.ts",
     "check-translations": "node --no-experimental-fetch ./scripts/checkTranslations.mjs",
     "update-translations-db": "node --no-experimental-fetch scripts/run.js ../scripts/updateTranslationsDB.js",

scripts/scripts.v2/denormalizeAllEntities.ts

@@ -0,0 +1,142 @@
+import { config } from 'api/config';
+import { DB } from 'api/odm';
+import { tenants } from 'api/tenants';
+
+import { denormalizeMetadata } from 'api/entities/denormalize';
+import entities from 'api/entities/entities';
+import entitiesModel from 'api/entities/entitiesModel';
+import translationsModel, { IndexedTranslations } from 'api/i18n/translations';
+import { permissionsContext } from 'api/permissions/permissionsContext';
+import { search } from 'api/search';
+import templates from 'api/templates';
+import { LanguageISO6391 } from 'shared/types/commonTypes';
+import { TemplateSchema } from 'shared/types/templateType';
+import { ThesaurusSchema } from 'shared/types/thesaurusType';
+import { inspect } from 'util';
+
+const { tenant, allTenants } = require('yargs')
+  .option('tenant', {
+    alias: 't',
+    type: 'string',
+    describe: 'Tenant to use',
+    default: 'default',
+  })
+  .option('allTenants', {
+    alias: 'a',
+    type: 'boolean',
+    describe: 'All tenants',
+    default: false,
+  }).argv;
+
+let dbAuth = {};
+
+if (process.env.DBUSER) {
+  dbAuth = {
+    auth: { authSource: 'admin' },
+    user: process.env.DBUSER,
+    pass: process.env.DBPASS,
+  };
+}
+
+async function handleTenant(tenantName: string) {
+  await tenants.run(async () => {
+    const start = process.hrtime();
+
+    permissionsContext.setCommandContext();
+
+    const templateRelatedThesauri: { [templateId: string]: Record<string, ThesaurusSchema> } = {};
+    const indexedTemplates = await (
+      await templates.get()
+    ).reduce<Promise<{ [k: string]: TemplateSchema }>>(async (prev, t) => {
+      let memo = await prev;
+      memo[t._id.toString()] = t;
+      templateRelatedThesauri[t._id.toString()] = await templates.getRelatedThesauri(t);
+      return memo;
+    }, Promise.resolve({}));
+
+    const translationsByLanguage = (
+      await translationsModel.get()
+    ).reduce<{ [language: string]: IndexedTranslations }>((memo, t) => {
+      if (!t.locale) {
+        throw new Error(`translation ${t._id} has no locale !`);
+      }
+      memo[t.locale] = t;
+      return memo;
+    }, {});
+
+    const allTemplates = Object.values(indexedTemplates);
+    const entityIds = await entities.getUnrestricted({}, '_id', {});
+
+    let entitiesProcessed = 0;
+    const errors: any[] = [];
+
+    await entityIds.reduce(async (prev, entity) => {
+      await prev;
+      const _id = entity._id;
+      const [entityToSave] = await entities.getUnrestricted({ _id });
+      try {
+        console.log(
+          JSON.stringify({ updating: `${entityToSave.title} | ${entityToSave.sharedId}` })
+        );
+
+        if (!entityToSave.template) {
+          throw new Error(`Entity ${entityToSave._id} has no template !`);
+        }
+
+        if (!entityToSave.language) {
+          throw new Error(`Entity ${entityToSave._id} has no language !`);
+        }
+
+        entityToSave.metadata = await denormalizeMetadata(
+          entityToSave.metadata || {},
+          entityToSave.language as LanguageISO6391,
+          indexedTemplates[entityToSave.template.toString()],
+          {
+            thesauriByKey: templateRelatedThesauri[entityToSave.template.toString()],
+            allTemplates,
+            translation: translationsByLanguage[entityToSave.language],
+          }
+        );
+        await entitiesModel.save(entityToSave);
+        entitiesProcessed += 1;
+      } catch (e) {
+        const error = { tenant, sharedId: _id, error: inspect(e) };
+        errors.push(error);
+      }
+      return Promise.resolve();
+    }, Promise.resolve());
+
+    await search.indexEntities({});
+
+    const [seconds, nanoseconds] = process.hrtime(start);
+    const elapsedTime = seconds + nanoseconds / 1e9;
+
+    console.log(
+      inspect({
+        logType: 'summary',
+        tenant: tenantName,
+        entitiesFetched: entityIds.length,
+        correctlyProcessed: entitiesProcessed,
+        notProcessed: errors.length,
+        elapsedTime: `${elapsedTime.toFixed(3)} s`,
+        errors: errors,
+      })
+    );
+  }, tenantName);
+}
+
+(async function run() {
+  await DB.connect(config.DBHOST, dbAuth);
+  await tenants.setupTenants();
+
+  if (!allTenants) {
+    await handleTenant(tenant);
+  } else {
+    await Object.keys(tenants.tenants).reduce(async (prev, tenantName) => {
+      await prev;
+      await handleTenant(tenantName);
+    }, Promise.resolve());
+  }
+  await tenants.model?.closeChangeStream();
+  await DB.disconnect();
+})();