fix(h5p-mongos3): library storage includes library.json in exports (#…

…1575) * fix(h5p-mongos3): library storage includes library.json in exports * test(h5p-mongos3): adapted tests
Lumieducation · Jul 10, 2021 · e6c4caa · e6c4caa
1 parent 826a2e5
commit e6c4caa
Show file tree

Hide file tree

Showing 5 changed files with 144 additions and 14 deletions.
diff --git a/packages/h5p-mongos3/src/MongoLibraryStorage.ts b/packages/h5p-mongos3/src/MongoLibraryStorage.ts
@@ -474,9 +474,23 @@ export default class MongoLibraryStorage implements ILibraryStorage {
         }
 
         if (!fileData) {
+            // The library.json file is in the MongoDB document as binary JSON,
+            // so we get it from there
+            if (file === 'library.json') {
+                const metadata = JSON.stringify(
+                    await this.getMetadata(library)
+                );
+                const readable = new ReadableStreamBuffer();
+                readable.put(metadata, 'utf-8');
+                readable.stop();
+                return readable;
+            }
             throw new H5pError(
                 'library-file-missing',
-                { ubername: LibraryName.toUberName(library), filename: file },
+                {
+                    ubername: LibraryName.toUberName(library),
+                    filename: file
+                },
                 404
             );
         } else {
@@ -613,7 +627,12 @@ export default class MongoLibraryStorage implements ILibraryStorage {
         try {
             result = await this.mongodb.findOne(
                 { _id: LibraryName.toUberName(library) },
-                { projection: { metadata: 1, additionalMetadata: 1 } }
+                {
+                    projection: {
+                        metadata: 1,
+                        additionalMetadata: 1
+                    }
+                }
             );
         } catch (error) {
             throw new H5pError(
@@ -725,7 +744,7 @@ export default class MongoLibraryStorage implements ILibraryStorage {
             );
         }
 
-        files = result.files.map((f) => f.filename);
+        files = result.files.map((f) => f.filename).concat(['library.json']);
         log.debug(`Found ${files.length} file(s) in MongoDB.`);
         return files;
     }
@@ -831,6 +850,52 @@ export default class MongoLibraryStorage implements ILibraryStorage {
         });
     }
 
+    /**
+     * Gets the the metadata of a library. In contrast to getLibrary this is
+     * only the metadata.
+     * @param library the library
+     * @returns the metadata about the locally installed library
+     */
+    private async getMetadata(
+        library: ILibraryName
+    ): Promise<ILibraryMetadata> {
+        if (!library) {
+            throw new Error('You must pass in a library name to getLibrary.');
+        }
+        let result;
+
+        try {
+            result = await this.mongodb.findOne(
+                { _id: LibraryName.toUberName(library) },
+                {
+                    projection: {
+                        metadata: 1
+                    }
+                }
+            );
+        } catch (error) {
+            console.log(error);
+            throw new H5pError(
+                'mongo-library-storage:error-getting-library-metadata',
+                { ubername: LibraryName.toUberName(library) }
+            );
+        }
+        if (!result) {
+            throw new H5pError(
+                'mongo-library-storage:library-not-found',
+                { ubername: LibraryName.toUberName(library) },
+                404
+            );
+        }
+        if (!result.metadata) {
+            throw new H5pError(
+                'mongo-library-storage:error-getting-library-metadata',
+                { ubername: LibraryName.toUberName(library) }
+            );
+        }
+        return result.metadata;
+    }
+
     private async readableToBuffer(readable: Readable): Promise<Buffer> {
         return new Promise((resolve) => {
             const chunks = [];

diff --git a/packages/h5p-mongos3/src/MongoS3LibraryStorage.ts b/packages/h5p-mongos3/src/MongoS3LibraryStorage.ts
@@ -5,7 +5,7 @@ import AWS from 'aws-sdk';
 import { Readable } from 'stream';
 import * as path from 'path';
 import { PromiseResult } from 'aws-sdk/lib/request';
-
+import { ReadableStreamBuffer } from 'stream-buffers';
 import {
     IAdditionalLibraryMetadata,
     IFileStats,
@@ -159,7 +159,9 @@ export default class MongoS3LibraryStorage implements ILibraryStorage {
                 }
             );
         }
-        const filesToDelete = await this.listFiles(library);
+        const filesToDelete = await this.listFiles(library, {
+            withMetadata: false
+        });
         // S3 batch deletes only work with 1000 files at a time, so we
         // might have to do this in several requests.
         try {
@@ -459,6 +461,15 @@ export default class MongoS3LibraryStorage implements ILibraryStorage {
     ): Promise<Readable> {
         validateFilename(file, this.options?.invalidCharactersRegexp);
 
+        // As the metadata is not S3, we need to get it from MongoDB.
+        if (file === 'library.json') {
+            const metadata = JSON.stringify(await this.getMetadata(library));
+            const readable = new ReadableStreamBuffer();
+            readable.put(metadata, 'utf-8');
+            readable.stop();
+            return readable;
+        }
+
         return this.s3
             .getObject({
                 Bucket: this.options.s3Bucket,
@@ -626,10 +637,15 @@ export default class MongoS3LibraryStorage implements ILibraryStorage {
 
     /**
      * Gets a list of all library files that exist for this library.
-     * @param library
+     * @param library the library name
+     * @param withMetadata true if the 'library.json' file should be included in
+     * the list
      * @returns all files that exist for the library
      */
-    public async listFiles(library: ILibraryName): Promise<string[]> {
+    public async listFiles(
+        library: ILibraryName,
+        options: { withMetadata?: boolean } = { withMetadata: true }
+    ): Promise<string[]> {
         const prefix = this.getS3Key(library, '');
         let files: string[] = [];
         try {
@@ -655,7 +671,7 @@ export default class MongoS3LibraryStorage implements ILibraryStorage {
             return [];
         }
         log.debug(`Found ${files.length} file(s) in S3.`);
-        return files;
+        return options?.withMetadata ? files.concat('library.json') : files;
     }
 
     /**
@@ -758,6 +774,52 @@ export default class MongoS3LibraryStorage implements ILibraryStorage {
         });
     }
 
+    /**
+     * Gets the the metadata of a library. In contrast to getLibrary this is
+     * only the metadata.
+     * @param library the library
+     * @returns the metadata about the locally installed library
+     */
+    private async getMetadata(
+        library: ILibraryName
+    ): Promise<ILibraryMetadata> {
+        if (!library) {
+            throw new Error('You must pass in a library name to getLibrary.');
+        }
+        let result;
+
+        try {
+            result = await this.mongodb.findOne(
+                { _id: LibraryName.toUberName(library) },
+                {
+                    projection: {
+                        metadata: 1
+                    }
+                }
+            );
+        } catch (error) {
+            console.log(error);
+            throw new H5pError(
+                'mongo-library-storage:error-getting-library-metadata',
+                { ubername: LibraryName.toUberName(library) }
+            );
+        }
+        if (!result) {
+            throw new H5pError(
+                'mongo-library-storage:library-not-found',
+                { ubername: LibraryName.toUberName(library) },
+                404
+            );
+        }
+        if (!result.metadata) {
+            throw new H5pError(
+                'mongo-library-storage:error-getting-library-metadata',
+                { ubername: LibraryName.toUberName(library) }
+            );
+        }
+        return result.metadata;
+    }
+
     private getS3Key(library: ILibraryName, filename: string): string {
         return `${LibraryName.toUberName(library)}/${filename}`;
     }

diff --git a/packages/h5p-mongos3/test/MongoLibraryStorage.test.ts b/packages/h5p-mongos3/test/MongoLibraryStorage.test.ts
@@ -452,7 +452,8 @@ describe('MongoS3LibraryStorage', () => {
         ).resolves.toMatchObject({ size: realStats.size });
         await expect(storage.listFiles(example1Name)).resolves.toMatchObject([
             'semantics.json',
-            'language/.en.json'
+            'language/.en.json',
+            'library.json'
         ]);
     });
 

diff --git a/packages/h5p-mongos3/test/MongoS3LibraryStorage.test.ts b/packages/h5p-mongos3/test/MongoS3LibraryStorage.test.ts
@@ -421,7 +421,8 @@ describe('MongoS3LibraryStorage', () => {
         ).resolves.toMatchObject({ size: realStats.size });
         await expect(storage.listFiles(example1Name)).resolves.toMatchObject([
             'language/.en.json',
-            'semantics.json'
+            'semantics.json',
+            'library.json'
         ]);
     });
 

diff --git a/scripts/mongo-s3-docker-compose.yml b/scripts/mongo-s3-docker-compose.yml
@@ -18,11 +18,12 @@ services:
         volumes:
             - minio_data:/data
         ports:
-            - '9000:9000'
-        command: ['server', '/data']
+            - 9000:9000
+            - 9001:9001
+        command: ['server', '--console-address', ':9001', '/data']
         environment:
-            MINIO_ACCESS_KEY: minioaccesskey
-            MINIO_SECRET_KEY: miniosecret
+            MINIO_ROOT_USER: minioaccesskey
+            MINIO_ROOT_PASSWORD: miniosecret
 
 volumes:
     mongodb_data: