Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

test: add pagerduty, check if test is running before attemmpting rerun #1624

Merged
merged 2 commits into from
Mar 2, 2021
Merged

test: add pagerduty, check if test is running before attemmpting rerun #1624

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0576431
test: add pagerduty, check if test is running before attemmpting rerun
Feb 9, 2021
eaf4915
test: wrap pagerduty await in try-catch, remove condition
Mar 1, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions test/cli-alert/package-lock.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions test/cli-alert/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
"license": "Apache-2.0",
"dependencies": {
"@octokit/rest": "^18.0.5",
"@pagerduty/pdjs": "^2.2.0",
"@slack/webhook": "^5.0.3",
"typescript": "^4.0.2"
},
Expand Down
79 changes: 56 additions & 23 deletions test/cli-alert/src/index.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,22 @@
import { Octokit } from '@octokit/rest';
import { IncomingWebhook } from '@slack/webhook';
import { IncomingWebhookDefaultArguments } from '@slack/webhook';

if (!process.env.USER_GITHUB_TOKEN || !process.env.SLACK_WEBHOOK_URL) {
console.error('Missing USER_GITHUB_TOKEN or SLACK_WEBHOOK_URL');
import { event } from '@pagerduty/pdjs';

if (
!process.env.USER_GITHUB_TOKEN ||
!process.env.SLACK_WEBHOOK_URL ||
!process.env.PD_ROUTING_KEY
) {
console.error(
'Missing USER_GITHUB_TOKEN, SLACK_WEBHOOK_URL or PD_ROUTING_KEY',
);
process.exit(1);
}

const GITHUB_TOKEN = process.env.USER_GITHUB_TOKEN;
const SLACK_WEBHOOK_URL = process.env.SLACK_WEBHOOK_URL;
const PD_ROUTING_KEY = process.env.PD_ROUTING_KEY;

const octokit = new Octokit({
auth: GITHUB_TOKEN,
Expand Down Expand Up @@ -54,7 +62,7 @@ async function discoverConsecutiveFailures(
process.exit(1);
} else if (
'failure' === firstJob.conclusion &&
firstJob?.conclusion === secondJob.conclusion
firstJob.conclusion === secondJob.conclusion
) {
console.log(`Found a job that failed 2 times in a row: ${jobName}`);
failedJobs.push(jobName);
Expand All @@ -64,6 +72,28 @@ async function discoverConsecutiveFailures(
return failedJobs;
}

async function sendPagerDuty() {
try {
const res = await event({
data: {
// eslint-disable-next-line @typescript-eslint/camelcase
routing_key: PD_ROUTING_KEY,
// eslint-disable-next-line @typescript-eslint/camelcase
event_action: 'trigger',
payload: {
summary: 'CLI Alert. Smoke tests failing',
source: 'Snyk CLI Smoke tests',
severity: 'warning',
},
},
});
console.log(res);
} catch (err) {
console.error(err);
process.exit(1);
}
}

async function sendSlackAlert(failedJobs: string[]) {
console.log('Jobs failed again. Sending Slack alert...');
const args: IncomingWebhookDefaultArguments = {
Expand Down Expand Up @@ -170,21 +200,6 @@ async function run() {

// Check the latest 2 smoke tests for tests that had the same job fail 2 times in a row.
const latestWorkflowRuns = workflowRuns.workflow_runs.slice(0, 2);

console.log('Checking smoke tests jobs...');
const failedWorkflows = await discoverConsecutiveFailures(
latestWorkflowRuns[0].id,
latestWorkflowRuns[1].id,
);

if (!failedWorkflows.length || failedWorkflows.length < 1) {
console.log(
'There were no 2 consecutive fails on a job. No need to alert.',
);
return;
}

console.log('Trying to re-run smoke test...');
const id = latestWorkflowRuns[0].id;

// Check current status of smoke test workflow and wait if it's still running
Expand All @@ -202,6 +217,21 @@ async function run() {
await waitForConclusion(id);
}

console.log('Checking smoke tests jobs...');
const failedWorkflows = await discoverConsecutiveFailures(
latestWorkflowRuns[0].id,
latestWorkflowRuns[1].id,
);

if (!failedWorkflows.length) {
console.log(
'There were no 2 consecutive fails on a job. No need to alert.',
);
return;
}

console.log('Trying to re-run smoke test...');

// After making sure smoke test isn't currently running - try to re-run
console.log(`Starting re-run of Smoke Test. ID number: ${id}...`);
await octokit.actions.reRunWorkflow({
Expand All @@ -216,10 +246,13 @@ async function run() {
const failedAgainJobs = await checkJobConclusion(id, failedWorkflows);
console.log('Re-run completed.');

// If run failed again, send Slack alert
failedAgainJobs.length > 0
? await sendSlackAlert(failedAgainJobs)
: console.log('Jobs succeeded after re-run. Do not alert.');
// If run failed again, send Slack alert and PagerDuty
if (failedAgainJobs.length > 0) {
await sendSlackAlert(failedAgainJobs);
await sendPagerDuty();
} else {
console.log('Jobs succeeded after re-run. Do not alert.');
}
} catch (error) {
console.error(error);
process.exit(1);
Expand Down