From e1556b8aaad21fdeaf972ccea1762fc1241fa19b Mon Sep 17 00:00:00 2001 From: Daneryl Date: Fri, 5 Mar 2021 10:55:20 +0100 Subject: [PATCH] error handling on tocService --- .../toc_generation/specs/tocService.spec.ts | 156 ++++++++++++------ app/api/toc_generation/tocService.ts | 68 +++++--- 2 files changed, 152 insertions(+), 72 deletions(-) diff --git a/app/api/toc_generation/specs/tocService.spec.ts b/app/api/toc_generation/specs/tocService.spec.ts index 328a87edc7..a2398dd6e4 100644 --- a/app/api/toc_generation/specs/tocService.spec.ts +++ b/app/api/toc_generation/specs/tocService.spec.ts @@ -9,76 +9,134 @@ import { tocService } from '../tocService'; describe('tocService', () => { const service = tocService('url'); + let requestMock: jest.SpyInstance; + + beforeEach(async () => { + spyOn(errorLog, 'error'); + requestMock = jest.spyOn(request, 'uploadFile'); + requestMock.mockImplementation(async (url, filename) => { + expect(url).toBe('url'); + if (filename === 'pdf1.pdf') { + return Promise.resolve({ text: JSON.stringify([{ label: 'section1 pdf1' }]) }); + } + if (filename === 'pdf3.pdf') { + return Promise.resolve({ text: JSON.stringify([{ label: 'section1 pdf3' }]) }); + } + throw new Error(`this file is not supposed to be sent for toc generation ${filename}`); + }); + + const elasticIndex = 'toc.service.index'; + await testingDB.clearAllAndLoad(fixtures, elasticIndex); + await elasticTesting.resetIndex(); + await elasticTesting.refresh(); + }); + afterAll(async () => { await testingDB.disconnect(); }); - let requestMock: jest.SpyInstance; + it('should not fail when there is no more to process', async () => { + await service.processNext(); + await service.processNext(); + await expect(service.processNext()).resolves.not.toThrow(); + }); - describe('processNext', () => { - beforeEach(async () => { - spyOn(errorLog, 'error'); - requestMock = jest.spyOn(request, 'uploadFile'); - requestMock.mockImplementation(async (url, filename) => { - expect(url).toBe('url'); - if (filename === 'pdf1.pdf') { - return Promise.resolve([{ label: 'section1 pdf1' }]); - } - if (filename === 'pdf3.pdf') { - return Promise.resolve([{ label: 'section1 pdf3' }]); - } - throw new Error(`this file is not supposed to be sent for toc generation ${filename}`); - }); + it('should send the next pdfFile and save toc generated', async () => { + await service.processNext(); + await service.processNext(); + + let [fileProcessed] = await files.get({ filename: 'pdf1.pdf' }); + expect(fileProcessed.toc).toEqual([{ label: 'section1 pdf1' }]); + expect(fileProcessed.generatedToc).toEqual(true); + + [fileProcessed] = await files.get({ filename: 'pdf3.pdf' }); + expect(fileProcessed.toc).toEqual([{ label: 'section1 pdf3' }]); + expect(fileProcessed.generatedToc).toEqual(true); + }); + + it('should reindex the affected entities', async () => { + await service.processNext(); + await service.processNext(); + await elasticTesting.refresh(); + + const entitiesIndexed = await elasticTesting.getIndexedEntities(); + + expect(entitiesIndexed).toEqual([ + expect.objectContaining({ + title: 'pdf1entity', + generatedToc: true, + }), + expect.objectContaining({ + title: 'pdf3entity', + generatedToc: true, + }), + ]); + }); + + describe('error handling', () => { + it('should save a fake TOC when generated one is empty', async () => { + requestMock.mockImplementation(async () => Promise.resolve({ text: JSON.stringify([]) })); + await service.processNext(); + + const [fileProcessed] = await files.get({ filename: 'pdf1.pdf' }); + expect(fileProcessed.toc).toEqual([ + { + selectionRectangles: [{ top: 0, left: 0, width: 0, height: 0, page: '1' }], + label: 'ERROR: Toc was generated empty', + indentation: 0, + }, + ]); + expect(fileProcessed.generatedToc).toEqual(true); - const elasticIndex = 'toc.service.index'; - await testingDB.clearAllAndLoad(fixtures, elasticIndex); - await elasticTesting.resetIndex(); await elasticTesting.refresh(); + const entitiesIndexed = await elasticTesting.getIndexedEntities(); + expect(entitiesIndexed).toEqual([ + expect.objectContaining({ title: 'pdf1entity', generatedToc: true }), + ]); }); - it('should not fail when request fails', async () => { + it('should save a fake toc when there is an error', async () => { requestMock.mockImplementation(async () => { throw new Error('request error'); }); - await expect(service.processNext()).resolves.not.toThrow(); - }); - - it('should not fail when there is no more to process', async () => { - await service.processNext(); - await service.processNext(); - await expect(service.processNext()).resolves.not.toThrow(); - }); - - it('should send the next pdfFile and save toc generated', async () => { - await service.processNext(); await service.processNext(); - let [fileProcessed] = await files.get({ filename: 'pdf1.pdf' }); - expect(fileProcessed.toc).toEqual([{ label: 'section1 pdf1' }]); + const [fileProcessed] = await files.get({ filename: 'pdf1.pdf' }); + expect(fileProcessed.toc).toEqual([ + { + selectionRectangles: [{ top: 0, left: 0, width: 0, height: 0, page: '1' }], + label: 'ERROR: Toc generation throwed an error', + indentation: 0, + }, + { + selectionRectangles: [{ top: 0, left: 0, width: 0, height: 0, page: '1' }], + label: 'request error', + indentation: 0, + }, + ]); expect(fileProcessed.generatedToc).toEqual(true); - [fileProcessed] = await files.get({ filename: 'pdf3.pdf' }); - expect(fileProcessed.toc).toEqual([{ label: 'section1 pdf3' }]); - expect(fileProcessed.generatedToc).toEqual(true); + await elasticTesting.refresh(); + const entitiesIndexed = await elasticTesting.getIndexedEntities(); + expect(entitiesIndexed).toEqual([ + expect.objectContaining({ title: 'pdf1entity', generatedToc: true }), + ]); }); - it('should reindex the affected entities', async () => { - await service.processNext(); + it('should not save anything when the error is ECONNREFUSED', async () => { + requestMock.mockImplementation(async () => { + // eslint-disable-next-line no-throw-literal + throw { code: 'ECONNREFUSED' }; + }); await service.processNext(); - await elasticTesting.refresh(); - const entitiesIndexed = await elasticTesting.getIndexedEntities(); + const [fileProcessed] = await files.get({ filename: 'pdf1.pdf' }); + expect(fileProcessed.toc).not.toBeDefined(); + expect(fileProcessed.generatedToc).not.toBeDefined(); - expect(entitiesIndexed).toEqual([ - expect.objectContaining({ - title: 'pdf1entity', - generatedToc: true, - }), - expect.objectContaining({ - title: 'pdf3entity', - generatedToc: true, - }), - ]); + await elasticTesting.refresh(); + const entitiesIndexed = await elasticTesting.getIndexedEntities(); + expect(entitiesIndexed).toEqual([]); }); }); }); diff --git a/app/api/toc_generation/tocService.ts b/app/api/toc_generation/tocService.ts index e24de725ba..8831aaf1df 100644 --- a/app/api/toc_generation/tocService.ts +++ b/app/api/toc_generation/tocService.ts @@ -3,6 +3,44 @@ import { prettifyError } from 'api/utils/handleError'; import errorLog from 'api/log/errorLog'; import request from 'shared/JSONRequest'; import entities from 'api/entities'; +import { TocSchema } from 'shared/types/commonTypes'; +import { FileType } from 'shared/types/fileType'; + +const fakeTocEntry = (label: string): TocSchema => ({ + selectionRectangles: [{ top: 0, left: 0, width: 0, height: 0, page: '1' }], + indentation: 0, + label, +}); + +const saveToc = async (file: FileType, toc: TocSchema[]) => { + await files.save({ ...file, toc, generatedToc: true }); + const [entity] = await entities.get({ sharedId: file.entity }, {}); + await entities.save( + { + ...entity, + generatedToc: true, + }, + { user: {}, language: file.language }, + false + ); +}; + +const generateToc = async (url: string, file: FileType): Promise => { + const response = await request.uploadFile(url, file.filename, uploadsPath(file.filename)); + + let toc = JSON.parse(response.text); + if (!toc.length) { + toc = [fakeTocEntry('ERROR: Toc was generated empty')]; + } + return toc; +}; + +const handleError = async (e: { code?: string; message: string }, file: FileType) => { + if (e?.code !== 'ECONNREFUSED' && e?.code !== 'ECONNRESET') { + const toc = [fakeTocEntry('ERROR: Toc generation throwed an error'), fakeTocEntry(e.message)]; + await saveToc(file, toc); + } +}; const tocService = (serviceUrl: string) => ({ async processNext() { @@ -13,32 +51,16 @@ const tocService = (serviceUrl: string) => ({ filename: { $exists: true }, }, '', - { - sort: { _id: 1 }, - limit: 1, - } + { sort: { _id: 1 }, limit: 1 } ); - try { - if (nextFile) { - const toc = await request.uploadFile( - serviceUrl, - nextFile.filename, - uploadsPath(nextFile.filename) - ); - await files.save({ ...nextFile, toc, generatedToc: true }); - const [entity] = await entities.get({ sharedId: nextFile.entity }, {}); - await entities.save( - { - ...entity, - generatedToc: true, - }, - { user: {}, language: nextFile.language }, - false - ); + if (nextFile) { + try { + await saveToc(nextFile, await generateToc(serviceUrl, nextFile)); + } catch (e) { + await handleError(e, nextFile); + errorLog.error(prettifyError(e).prettyMessage); } - } catch (e) { - errorLog.error(prettifyError(e).prettyMessage); } }, });