From 613ed657fb98d446f1afe57635b0c2032de9215a Mon Sep 17 00:00:00 2001 From: Niek Palm Date: Thu, 30 Sep 2021 21:48:22 +0200 Subject: [PATCH] fix: added more detailed logging for scaling up and down Only log number of runners after cleanup, removed diff calculaton --- .../lambdas/runners/src/scale-runners/gh-auth.test.ts | 2 +- .../lambdas/runners/src/scale-runners/scale-down.ts | 10 ++++++++-- .../lambdas/runners/src/scale-runners/scale-up.ts | 7 ++++--- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/modules/runners/lambdas/runners/src/scale-runners/gh-auth.test.ts b/modules/runners/lambdas/runners/src/scale-runners/gh-auth.test.ts index 86ab1b6c08..bd19f46e7c 100644 --- a/modules/runners/lambdas/runners/src/scale-runners/gh-auth.test.ts +++ b/modules/runners/lambdas/runners/src/scale-runners/gh-auth.test.ts @@ -1,4 +1,4 @@ -import { createOctoClient, createGithubAppAuth, createGithubInstallationAuth } from './gh-auth'; +import { createOctoClient, createGithubAppAuth } from './gh-auth'; import nock from 'nock'; import { createAppAuth } from '@octokit/auth-app'; diff --git a/modules/runners/lambdas/runners/src/scale-runners/scale-down.ts b/modules/runners/lambdas/runners/src/scale-runners/scale-down.ts index f96df1893b..d949791d30 100644 --- a/modules/runners/lambdas/runners/src/scale-runners/scale-down.ts +++ b/modules/runners/lambdas/runners/src/scale-runners/scale-down.ts @@ -117,6 +117,7 @@ async function evaluateAndRemoveRunners( for (const ownerTag of ownerTags) { const ec2RunnersFiltered = ec2Runners.filter((runner) => runner.owner === ownerTag); + console.debug(`Found: '${ec2RunnersFiltered.length}' active GitHub runners with owner tag: '${ownerTag}'`); for (const ec2Runner of ec2RunnersFiltered) { const ghRunners = await listGitHubRunners(ec2Runner); const ghRunner = ghRunners.find((runner) => runner.name === ec2Runner.instanceId); @@ -190,15 +191,20 @@ export async function scaleDown(): Promise { // list and sort runners, newest first. This ensure we keep the newest runners longer. const ec2Runners = await listAndSortRunners(environment); + const activeEc2RunnersCount = ec2Runners.length; + console.info(`Found: '${activeEc2RunnersCount}' active GitHub EC2 runner instances before clean-up.`); - if (ec2Runners.length === 0) { + if (activeEc2RunnersCount === 0) { console.debug(`No active runners found for environment: '${environment}'`); return; } const legacyRunners = filterLegacyRunners(ec2Runners); - console.log(JSON.stringify(legacyRunners)); + console.debug(JSON.stringify(legacyRunners)); const runners = filterRunners(ec2Runners); await evaluateAndRemoveRunners(runners, scaleDownConfigs); await evaluateAndRemoveRunners(legacyRunners, scaleDownConfigs); + + const activeEc2RunnersCountAfter = (await listAndSortRunners(environment)).length; + console.info(`Found: '${activeEc2RunnersCountAfter}' active GitHub EC2 runners instances after clean-up.`); } diff --git a/modules/runners/lambdas/runners/src/scale-runners/scale-up.ts b/modules/runners/lambdas/runners/src/scale-runners/scale-up.ts index 63a8f95111..62e1f8a09e 100644 --- a/modules/runners/lambdas/runners/src/scale-runners/scale-up.ts +++ b/modules/runners/lambdas/runners/src/scale-runners/scale-up.ts @@ -112,13 +112,14 @@ async function getJobStatus(githubInstallationClient: Octokit, payload: ActionRe export async function createRunnerLoop(runnerParameters: RunnerInputParameters): Promise { const launchTemplateNames = process.env.LAUNCH_TEMPLATE_NAME?.split(',') as string[]; let launched = false; - for (const launchTemplateName of launchTemplateNames) { - console.info(`Attempting to launch instance using ${launchTemplateName}.`); + for (let i = 0; i < launchTemplateNames.length; i++) { + console.info(`Attempt '${i}' to launch instance using ${launchTemplateNames[i]}.`); try { - await createRunner(runnerParameters, launchTemplateName); + await createRunner(runnerParameters, launchTemplateNames[i]); launched = true; break; } catch (error) { + console.debug(`Attempt '${i}' to launch instance using ${launchTemplateNames[i]} FAILED.`); console.error(error); } }