Skip to content

Commit

Permalink
error handling on tocService
Browse files Browse the repository at this point in the history
  • Loading branch information
@daneryl
daneryl committed Mar 5, 2021
1 parent 41bd3b3 commit 9da8573
Show file tree
Hide file tree
Showing 2 changed files with 151 additions and 72 deletions.
156 changes: 107 additions & 49 deletions app/api/toc_generation/specs/tocService.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,76 +9,134 @@ import { tocService } from '../tocService';
describe('tocService', () => {
const service = tocService('url');

let requestMock: jest.SpyInstance;

beforeEach(async () => {
spyOn(errorLog, 'error');
requestMock = jest.spyOn(request, 'uploadFile');
requestMock.mockImplementation(async (url, filename) => {
expect(url).toBe('url');
if (filename === 'pdf1.pdf') {
return Promise.resolve({ text: JSON.stringify([{ label: 'section1 pdf1' }]) });
}
if (filename === 'pdf3.pdf') {
return Promise.resolve({ text: JSON.stringify([{ label: 'section1 pdf3' }]) });
}
throw new Error(`this file is not supposed to be sent for toc generation ${filename}`);
});

const elasticIndex = 'toc.service.index';
await testingDB.clearAllAndLoad(fixtures, elasticIndex);
await elasticTesting.resetIndex();
await elasticTesting.refresh();
});

afterAll(async () => {
await testingDB.disconnect();
});

let requestMock: jest.SpyInstance;
it('should not fail when there is no more to process', async () => {
await service.processNext();
await service.processNext();
await expect(service.processNext()).resolves.not.toThrow();
});

describe('processNext', () => {
beforeEach(async () => {
spyOn(errorLog, 'error');
requestMock = jest.spyOn(request, 'uploadFile');
requestMock.mockImplementation(async (url, filename) => {
expect(url).toBe('url');
if (filename === 'pdf1.pdf') {
return Promise.resolve([{ label: 'section1 pdf1' }]);
}
if (filename === 'pdf3.pdf') {
return Promise.resolve([{ label: 'section1 pdf3' }]);
}
throw new Error(`this file is not supposed to be sent for toc generation ${filename}`);
});
it('should send the next pdfFile and save toc generated', async () => {
await service.processNext();
await service.processNext();

let [fileProcessed] = await files.get({ filename: 'pdf1.pdf' });
expect(fileProcessed.toc).toEqual([{ label: 'section1 pdf1' }]);
expect(fileProcessed.generatedToc).toEqual(true);

[fileProcessed] = await files.get({ filename: 'pdf3.pdf' });
expect(fileProcessed.toc).toEqual([{ label: 'section1 pdf3' }]);
expect(fileProcessed.generatedToc).toEqual(true);
});

it('should reindex the affected entities', async () => {
await service.processNext();
await service.processNext();
await elasticTesting.refresh();

const entitiesIndexed = await elasticTesting.getIndexedEntities();

expect(entitiesIndexed).toEqual([
expect.objectContaining({
title: 'pdf1entity',
generatedToc: true,
}),
expect.objectContaining({
title: 'pdf3entity',
generatedToc: true,
}),
]);
});

describe('error handling', () => {
it('should save a fake TOC when generated one is empty', async () => {
requestMock.mockImplementation(async () => Promise.resolve({ text: JSON.stringify([]) }));
await service.processNext();

const [fileProcessed] = await files.get({ filename: 'pdf1.pdf' });
expect(fileProcessed.toc).toEqual([
{
selectionRectangles: [{ top: 0, left: 0, width: 0, height: 0, page: '1' }],
label: 'ERROR: Toc was generated empty',
indentation: 0,
},
]);
expect(fileProcessed.generatedToc).toEqual(true);

const elasticIndex = 'toc.service.index';
await testingDB.clearAllAndLoad(fixtures, elasticIndex);
await elasticTesting.resetIndex();
await elasticTesting.refresh();
const entitiesIndexed = await elasticTesting.getIndexedEntities();
expect(entitiesIndexed).toEqual([
expect.objectContaining({ title: 'pdf1entity', generatedToc: true }),
]);
});

it('should not fail when request fails', async () => {
it('should save a fake toc when there is an error', async () => {
requestMock.mockImplementation(async () => {
throw new Error('request error');
});
await expect(service.processNext()).resolves.not.toThrow();
});

it('should not fail when there is no more to process', async () => {
await service.processNext();
await service.processNext();
await expect(service.processNext()).resolves.not.toThrow();
});

it('should send the next pdfFile and save toc generated', async () => {
await service.processNext();
await service.processNext();

let [fileProcessed] = await files.get({ filename: 'pdf1.pdf' });
expect(fileProcessed.toc).toEqual([{ label: 'section1 pdf1' }]);
const [fileProcessed] = await files.get({ filename: 'pdf1.pdf' });
expect(fileProcessed.toc).toEqual([
{
selectionRectangles: [{ top: 0, left: 0, width: 0, height: 0, page: '1' }],
label: 'ERROR: Toc generation throwed an error',
indentation: 0,
},
{
selectionRectangles: [{ top: 0, left: 0, width: 0, height: 0, page: '1' }],
label: 'request error',
indentation: 0,
},
]);
expect(fileProcessed.generatedToc).toEqual(true);

[fileProcessed] = await files.get({ filename: 'pdf3.pdf' });
expect(fileProcessed.toc).toEqual([{ label: 'section1 pdf3' }]);
expect(fileProcessed.generatedToc).toEqual(true);
await elasticTesting.refresh();
const entitiesIndexed = await elasticTesting.getIndexedEntities();
expect(entitiesIndexed).toEqual([
expect.objectContaining({ title: 'pdf1entity', generatedToc: true }),
]);
});

it('should reindex the affected entities', async () => {
await service.processNext();
it('should not save anything when the error is ECONNREFUSED', async () => {
requestMock.mockImplementation(async () => {
// eslint-disable-next-line no-throw-literal
throw { code: 'ECONNREFUSED' };
});
await service.processNext();
await elasticTesting.refresh();

const entitiesIndexed = await elasticTesting.getIndexedEntities();
const [fileProcessed] = await files.get({ filename: 'pdf1.pdf' });
expect(fileProcessed.toc).not.toBeDefined();
expect(fileProcessed.generatedToc).not.toBeDefined();

expect(entitiesIndexed).toEqual([
expect.objectContaining({
title: 'pdf1entity',
generatedToc: true,
}),
expect.objectContaining({
title: 'pdf3entity',
generatedToc: true,
}),
]);
await elasticTesting.refresh();
const entitiesIndexed = await elasticTesting.getIndexedEntities();
expect(entitiesIndexed).toEqual([]);
});
});
});
67 changes: 44 additions & 23 deletions app/api/toc_generation/tocService.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,37 @@ import { prettifyError } from 'api/utils/handleError';
import errorLog from 'api/log/errorLog';
import request from 'shared/JSONRequest';
import entities from 'api/entities';
import { TocSchema } from 'shared/types/commonTypes';
import { FileType } from 'shared/types/fileType';

const fakeTocEntry = (label: string): TocSchema => ({
selectionRectangles: [{ top: 0, left: 0, width: 0, height: 0, page: '1' }],
indentation: 0,
label,
});

const saveToc = async (file: FileType, toc: TocSchema[]) => {
await files.save({ ...file, toc, generatedToc: true });
const [entity] = await entities.get({ sharedId: file.entity }, {});
await entities.save(
{
...entity,
generatedToc: true,
},
{ user: {}, language: file.language },
false
);
};

const generateToc = async (url: string, file: FileType): Promise<TocSchema[]> => {
const response = await request.uploadFile(url, file.filename, uploadsPath(file.filename));

let toc = JSON.parse(response.text);
if (!toc.length) {
toc = [fakeTocEntry('ERROR: Toc was generated empty')];
}
return toc;
};

const tocService = (serviceUrl: string) => ({
async processNext() {
Expand All @@ -13,32 +44,22 @@ const tocService = (serviceUrl: string) => ({
filename: { $exists: true },
},
'',
{
sort: { _id: 1 },
limit: 1,
}
{ sort: { _id: 1 }, limit: 1 }
);

try {
if (nextFile) {
const toc = await request.uploadFile(
serviceUrl,
nextFile.filename,
uploadsPath(nextFile.filename)
);
await files.save({ ...nextFile, toc, generatedToc: true });
const [entity] = await entities.get({ sharedId: nextFile.entity }, {});
await entities.save(
{
...entity,
generatedToc: true,
},
{ user: {}, language: nextFile.language },
false
);
if (nextFile) {
try {
await saveToc(nextFile, await generateToc(serviceUrl, nextFile));
} catch (e) {
if (e?.code !== 'ECONNREFUSED' && e?.code !== 'ECONNRESET') {
const toc = [
fakeTocEntry('ERROR: Toc generation throwed an error'),
fakeTocEntry(e.message),
];
await saveToc(nextFile, toc);
}
errorLog.error(prettifyError(e).prettyMessage);
}
} catch (e) {
errorLog.error(prettifyError(e).prettyMessage);
}
},
});
Expand Down

0 comments on commit 9da8573

Please sign in to comment.