Skip to content

Commit

Permalink
fix: Report app startup and DB migration errors to Sentry (#5127)
Browse files Browse the repository at this point in the history
  • Loading branch information
@netroy
netroy authored Jan 11, 2023
1 parent 3c109ff commit a573db2
Show file tree
Hide file tree
Showing 4 changed files with 74 additions and 79 deletions.
48 changes: 23 additions & 25 deletions packages/cli/src/commands/start.ts
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,5 @@
/* eslint-disable @typescript-eslint/no-non-null-assertion */
/* eslint-disable @typescript-eslint/await-thenable */
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
/* eslint-disable @typescript-eslint/explicit-module-boundary-types */
/* eslint-disable @typescript-eslint/unbound-method */
/* eslint-disable no-console */
/* eslint-disable @typescript-eslint/no-unsafe-call */
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
import path from 'path';
Expand All @@ -17,7 +13,7 @@ import replaceStream from 'replacestream';
import { promisify } from 'util';
import glob from 'fast-glob';

import { LoggerProxy, sleep } from 'n8n-workflow';
import { LoggerProxy, ErrorReporterProxy as ErrorReporter, sleep } from 'n8n-workflow';
import { createHash } from 'crypto';
import config from '@/config';

Expand Down Expand Up @@ -49,7 +45,20 @@ const open = require('open');
const pipeline = promisify(stream.pipeline);

let activeWorkflowRunner: ActiveWorkflowRunner.ActiveWorkflowRunner | undefined;
let processExitCode = 0;

const exitWithCrash = async (message: string, error: unknown) => {
ErrorReporter.error(new Error(message, { cause: error }), { level: 'fatal' });
await sleep(2000);
process.exit(1);
};

const exitSuccessFully = async () => {
try {
await CrashJournal.cleanup();
} finally {
process.exit();
}
};

export class Start extends Command {
static description = 'Starts n8n. Makes Web-UI available and starts active workflows';
Expand Down Expand Up @@ -101,24 +110,18 @@ export class Start extends Command {
static async stopProcess() {
getLogger().info('\nStopping n8n...');

const exit = () => {
CrashJournal.cleanup().finally(() => {
process.exit(processExitCode);
});
};

try {
// Stop with trying to activate workflows that could not be activated
activeWorkflowRunner?.removeAllQueuedWorkflowActivations();

const externalHooks = ExternalHooks();
await externalHooks.run('n8n.stop', []);

setTimeout(() => {
setTimeout(async () => {
// In case that something goes wrong with shutdown we
// kill after max. 30 seconds no matter what
console.log('process exited after 30s');
exit();
await exitSuccessFully();
}, 30000);

await InternalHooksManager.getInstance().onN8nStop();
Expand Down Expand Up @@ -159,10 +162,10 @@ export class Start extends Command {
//finally shut down Event Bus
await eventBus.close();
} catch (error) {
console.error('There was an error shutting down n8n.', error);
await exitWithCrash('There was an error shutting down n8n.', error);
}

exit();
await exitSuccessFully();
}

static async generateStaticAssets() {
Expand Down Expand Up @@ -236,12 +239,9 @@ export class Start extends Command {

try {
// Start directly with the init of the database to improve startup time
const startDbInitPromise = Db.init().catch((error: Error) => {
logger.error(`There was an error initializing DB: "${error.message}"`);

processExitCode = 1;
process.emit('exit', processExitCode);
});
const startDbInitPromise = Db.init().catch(async (error: Error) =>
exitWithCrash('There was an error initializing DB', error),
);

// Make sure the settings exist
const userSettings = await UserSettings.prepareUserSettings();
Expand Down Expand Up @@ -456,9 +456,7 @@ export class Start extends Command {
});
}
} catch (error) {
console.error('There was an error', error);
processExitCode = 1;
process.emit('exit', processExitCode);
await exitWithCrash('There was an error', error);
}
}
}
49 changes: 22 additions & 27 deletions packages/cli/src/commands/webhook.ts
View file
Original file line number Diff line number Diff line change
@@ -1,13 +1,8 @@
/* eslint-disable @typescript-eslint/no-unsafe-argument */
/* eslint-disable no-console */
/* eslint-disable @typescript-eslint/no-unsafe-call */
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
/* eslint-disable @typescript-eslint/unbound-method */
import { BinaryDataManager, UserSettings } from 'n8n-core';
import { Command, flags } from '@oclif/command';

import { LoggerProxy, sleep } from 'n8n-workflow';
import { LoggerProxy, ErrorReporterProxy as ErrorReporter, sleep } from 'n8n-workflow';
import config from '@/config';
import * as ActiveExecutions from '@/ActiveExecutions';
import { CredentialsOverwrites } from '@/CredentialsOverwrites';
Expand All @@ -22,7 +17,19 @@ import { getLogger } from '@/Logger';
import { initErrorHandling } from '@/ErrorReporting';
import * as CrashJournal from '@/CrashJournal';

let processExitCode = 0;
const exitWithCrash = async (message: string, error: unknown) => {
ErrorReporter.error(new Error(message, { cause: error }), { level: 'fatal' });
await sleep(2000);
process.exit(1);
};

const exitSuccessFully = async () => {
try {
await CrashJournal.cleanup();
} finally {
process.exit();
}
};

export class Webhook extends Command {
static description = 'Starts n8n webhook process. Intercepts only production URLs.';
Expand All @@ -42,20 +49,14 @@ export class Webhook extends Command {
static async stopProcess() {
LoggerProxy.info('\nStopping n8n...');

const exit = () => {
CrashJournal.cleanup().finally(() => {
process.exit(processExitCode);
});
};

try {
const externalHooks = ExternalHooks();
await externalHooks.run('n8n.stop', []);

setTimeout(() => {
setTimeout(async () => {
// In case that something goes wrong with shutdown we
// kill after max. 30 seconds no matter what
exit();
await exitSuccessFully();
}, 30000);

// Wait for active workflow executions to finish
Expand All @@ -74,10 +75,10 @@ export class Webhook extends Command {
executingWorkflows = activeExecutionsInstance.getActiveExecutions();
}
} catch (error) {
LoggerProxy.error('There was an error shutting down n8n.', error);
await exitWithCrash('There was an error shutting down n8n.', error);
}

exit();
await exitSuccessFully();
}

// eslint-disable-next-line @typescript-eslint/explicit-module-boundary-types
Expand Down Expand Up @@ -110,13 +111,9 @@ export class Webhook extends Command {

try {
// Start directly with the init of the database to improve startup time
const startDbInitPromise = Db.init().catch((error) => {
// eslint-disable-next-line @typescript-eslint/restrict-template-expressions, @typescript-eslint/no-unsafe-member-access
logger.error(`There was an error initializing DB: "${error.message}"`);

processExitCode = 1;
process.emit('exit', processExitCode);
});
const startDbInitPromise = Db.init().catch(async (error: Error) =>
exitWithCrash('There was an error initializing DB', error),
);

// Make sure the settings exist
// eslint-disable-next-line @typescript-eslint/no-unused-vars
Expand Down Expand Up @@ -151,9 +148,7 @@ export class Webhook extends Command {

console.info('Webhook listener waiting for requests.');
} catch (error) {
console.error('Exiting due to error.', error);
processExitCode = 1;
process.emit('exit', processExitCode);
await exitWithCrash('Exiting due to error.', error);
}
}
}
54 changes: 28 additions & 26 deletions packages/cli/src/commands/worker.ts
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,8 @@
/* eslint-disable @typescript-eslint/no-unsafe-argument */
/* eslint-disable no-console */
/* eslint-disable @typescript-eslint/no-shadow */
/* eslint-disable @typescript-eslint/explicit-module-boundary-types */
/* eslint-disable @typescript-eslint/unbound-method */
/* eslint-disable @typescript-eslint/no-unsafe-member-access */
/* eslint-disable @typescript-eslint/no-non-null-assertion */
/* eslint-disable @typescript-eslint/restrict-template-expressions */
/* eslint-disable @typescript-eslint/no-unused-vars */
import express from 'express';
import http from 'http';
import PCancelable from 'p-cancelable';
Expand All @@ -20,6 +16,7 @@ import {
IRun,
Workflow,
LoggerProxy,
ErrorReporterProxy as ErrorReporter,
sleep,
} from 'n8n-workflow';

Expand All @@ -29,7 +26,6 @@ import { CredentialsOverwrites } from '@/CredentialsOverwrites';
import { CredentialTypes } from '@/CredentialTypes';
import * as Db from '@/Db';
import { ExternalHooks } from '@/ExternalHooks';
import * as GenericHelpers from '@/GenericHelpers';
import { NodeTypes } from '@/NodeTypes';
import * as ResponseHelper from '@/ResponseHelper';
import * as WebhookHelpers from '@/WebhookHelpers';
Expand All @@ -41,9 +37,25 @@ import { PermissionChecker } from '@/UserManagement/PermissionChecker';

import config from '@/config';
import * as Queue from '@/Queue';
import * as CrashJournal from '@/CrashJournal';
import { getWorkflowOwner } from '@/UserManagement/UserManagementHelper';
import { generateFailedExecutionFromError } from '@/WorkflowHelpers';
import { N8N_VERSION } from '@/constants';
import { initErrorHandling } from '@/ErrorReporting';

const exitWithCrash = async (message: string, error: unknown) => {
ErrorReporter.error(new Error(message, { cause: error }), { level: 'fatal' });
await sleep(2000);
process.exit(1);
};

const exitSuccessFully = async () => {
try {
await CrashJournal.cleanup();
} finally {
process.exit();
}
};

export class Worker extends Command {
static description = '\nStarts a n8n worker';
Expand All @@ -64,9 +76,6 @@ export class Worker extends Command {

static jobQueue: Queue.JobQueue;

static processExitCode = 0;
// static activeExecutions = ActiveExecutions.getInstance();

/**
* Stop n8n in a graceful way.
* Make for example sure that all the webhooks from third party services
Expand All @@ -87,10 +96,10 @@ export class Worker extends Command {

const stopTime = new Date().getTime() + maxStopTime;

setTimeout(() => {
setTimeout(async () => {
// In case that something goes wrong with shutdown we
// kill after max. 30 seconds no matter what
process.exit(Worker.processExitCode);
await exitSuccessFully();
}, maxStopTime);

// Wait for active workflow executions to finish
Expand All @@ -108,10 +117,10 @@ export class Worker extends Command {
await sleep(500);
}
} catch (error) {
LoggerProxy.error('There was an error shutting down n8n.', error);
await exitWithCrash('There was an error shutting down n8n.', error);
}

process.exit(Worker.processExitCode);
await exitSuccessFully();
}

async runJob(job: Queue.Job, nodeTypes: INodeTypes): Promise<Queue.JobResponse> {
Expand Down Expand Up @@ -261,20 +270,18 @@ export class Worker extends Command {
process.once('SIGTERM', Worker.stopProcess);
process.once('SIGINT', Worker.stopProcess);

await initErrorHandling();
await CrashJournal.init();

// Wrap that the process does not close but we can still use async
await (async () => {
try {
const { flags } = this.parse(Worker);

// Start directly with the init of the database to improve startup time
const startDbInitPromise = Db.init().catch((error) => {
logger.error(`There was an error initializing DB: "${error.message}"`);

Worker.processExitCode = 1;
// @ts-ignore
process.emit('SIGINT');
process.exit(1);
});
const startDbInitPromise = Db.init().catch(async (error: Error) =>
exitWithCrash('There was an error initializing DB', error),
);

// Make sure the settings exist
await UserSettings.prepareUserSettings();
Expand Down Expand Up @@ -428,12 +435,7 @@ export class Worker extends Command {
});
}
} catch (error) {
logger.error(`Worker process cannot continue. "${error.message}"`);

Worker.processExitCode = 1;
// @ts-ignore
process.emit('SIGINT');
process.exit(1);
await exitWithCrash('Worker process cannot continue.', error);
}
})();
}
Expand Down
2 changes: 1 addition & 1 deletion packages/workflow/src/ErrorReporterProxy.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import type { Primitives } from './utils';
import * as Logger from './LoggerProxy';

export interface ReportingOptions {
level?: 'warning' | 'error';
level?: 'warning' | 'error' | 'fatal';
tags?: Record<string, Primitives>;
extra?: Record<string, unknown>;
}
Expand Down

0 comments on commit a573db2

Please sign in to comment.