From c2a834fa324016a18227327c262203791478b394 Mon Sep 17 00:00:00 2001 From: Niek Palm Date: Fri, 25 Feb 2022 16:29:16 +0100 Subject: [PATCH] feat(runner): Add option to disable auto update (#1791) * feat: Add option to disable auto update * Add junit test * Add disable_update_runners to Terraform * Add disable_update_runners to Terraform * set default to false * review * Add option to disable auto update to the pool * revert example --- README.md | 5 +++-- main.tf | 1 + modules/runner-binaries-syncer/README.md | 2 +- modules/runners/README.md | 5 +++-- modules/runners/lambdas/runners/src/pool/pool.ts | 11 ++++++++++- .../runners/src/scale-runners/scale-up.test.ts | 7 +++++++ .../lambdas/runners/src/scale-runners/scale-up.ts | 6 +++++- modules/runners/pool.tf | 13 +++++++------ modules/runners/pool/main.tf | 13 +++++++------ modules/runners/pool/variables.tf | 5 +++-- modules/runners/scale-up.tf | 11 ++++++----- modules/runners/variables.tf | 6 ++++++ variables.tf | 6 ++++++ 13 files changed, 65 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index a6950239fb..9d59e2989c 100644 --- a/README.md +++ b/README.md @@ -400,9 +400,10 @@ In case the setup does not work as intended follow the trace of events: | [cloudwatch\_config](#input\_cloudwatch\_config) | (optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details. | `string` | `null` | no | | [create\_service\_linked\_role\_spot](#input\_create\_service\_linked\_role\_spot) | (optional) create the serviced linked role for spot instances that is required by the scale-up lambda. | `bool` | `false` | no | | [delay\_webhook\_event](#input\_delay\_webhook\_event) | The number of seconds the event accepted by the webhook is invisible on the queue before the scale up lambda will receive the event. | `number` | `30` | no | +| [disable\_runner\_autoupdate](#input\_disable\_runner\_autoupdate) | Disable the auto update of the github runner agent. Be-aware there is a grace period of 30 days, see also the [GitHub article](https://github.blog/changelog/2022-02-01-github-actions-self-hosted-runners-can-now-disable-automatic-updates/) | `bool` | `false` | no | | [enable\_cloudwatch\_agent](#input\_enable\_cloudwatch\_agent) | Enabling the cloudwatch agent on the ec2 runner instances, the runner contains default config. Configuration can be overridden via `cloudwatch_config`. | `bool` | `true` | no | | [enable\_ephemeral\_runners](#input\_enable\_ephemeral\_runners) | Enable ephemeral runners, runners will only be used once. | `bool` | `false` | no | -| [enable\_managed\_runner\_security\_group](#inputenable\_managed\_runner\_security\_group) | Enabling the default managed security group creation. Unmanaged security groups can be specified via `runner_additional_security_group_ids`. | `bool` | `true` | no | +| [enable\_managed\_runner\_security\_group](#input\_enable\_managed\_runner\_security\_group) | Enabling the default managed security group creation. Unmanaged security groups can be specified via `runner_additional_security_group_ids`. | `bool` | `true` | no | | [enable\_organization\_runners](#input\_enable\_organization\_runners) | Register runners to organization, instead of repo level | `bool` | `false` | no | | [enable\_ssm\_on\_runners](#input\_enable\_ssm\_on\_runners) | Enable to allow access the runner instances for debugging purposes via SSM. Note that this adds additional permissions to the runner instances. | `bool` | `false` | no | | [enabled\_userdata](#input\_enabled\_userdata) | Should the userdata script be enabled for the runner. Set this to false if you are using your own prebuilt AMI | `bool` | `true` | no | @@ -454,7 +455,7 @@ In case the setup does not work as intended follow the trace of events: | [runner\_iam\_role\_managed\_policy\_arns](#input\_runner\_iam\_role\_managed\_policy\_arns) | Attach AWS or customer-managed IAM policies (by ARN) to the runner IAM role | `list(string)` | `[]` | no | | [runner\_log\_files](#input\_runner\_log\_files) | (optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details. | 
list(object({
    log_group_name   = string
    prefix_log_group = bool
    file_path        = string
    log_stream_name  = string
  }))
| `null` | no | | [runner\_metadata\_options](#input\_runner\_metadata\_options) | Metadata options for the ec2 runner instances. | `map(any)` | 
{
  "http_endpoint": "enabled",
  "http_put_response_hop_limit": 1,
  "http_tokens": "optional"
}
| no | -| [runner\_os](#input\_runner\_os) | The Operating System to use for GitHub Actions Runners (linux,win) | `string` | `"linux"` | no | +| [runner\_os](#input\_runner\_os) | The EC2 Operating System type to use for action runner instances (linux,windows). | `string` | `"linux"` | no | | [runner\_run\_as](#input\_runner\_run\_as) | Run the GitHub actions agent as user. | `string` | `"ec2-user"` | no | | [runners\_lambda\_s3\_key](#input\_runners\_lambda\_s3\_key) | S3 key for runners lambda function. Required if using S3 bucket to specify lambdas. | `any` | `null` | no | | [runners\_lambda\_s3\_object\_version](#input\_runners\_lambda\_s3\_object\_version) | S3 object version for runners lambda function. Useful if S3 versioning is enabled on source bucket. | `any` | `null` | no | diff --git a/main.tf b/main.tf index c71d73aa79..aaf0a56dd9 100644 --- a/main.tf +++ b/main.tf @@ -107,6 +107,7 @@ module "runners" { github_app_parameters = local.github_app_parameters enable_organization_runners = var.enable_organization_runners enable_ephemeral_runners = var.enable_ephemeral_runners + disable_runner_autoupdate = var.disable_runner_autoupdate enable_managed_runner_security_group = var.enable_managed_runner_security_group scale_down_schedule_expression = var.scale_down_schedule_expression minimum_running_time_in_minutes = var.minimum_running_time_in_minutes diff --git a/modules/runner-binaries-syncer/README.md b/modules/runner-binaries-syncer/README.md index de7e67aa49..f67060d82e 100644 --- a/modules/runner-binaries-syncer/README.md +++ b/modules/runner-binaries-syncer/README.md @@ -92,7 +92,7 @@ No modules. | [role\_permissions\_boundary](#input\_role\_permissions\_boundary) | Permissions boundary that will be added to the created role for the lambda. | `string` | `null` | no | | [runner\_allow\_prerelease\_binaries](#input\_runner\_allow\_prerelease\_binaries) | Allow the runners to update to prerelease binaries. | `bool` | `false` | no | | [runner\_architecture](#input\_runner\_architecture) | The platform architecture of the runner instance\_type. | `string` | `"x64"` | no | -| [runner\_os](#input\_runner\_os) | The operating system for the runner instance (linux, win), defaults to 'linux' | `string` | `"linux"` | no | +| [runner\_os](#input\_runner\_os) | The EC2 Operating System type to use for action runner instances (linux,windows). | `string` | `"linux"` | no | | [server\_side\_encryption\_configuration](#input\_server\_side\_encryption\_configuration) | Map containing server-side encryption configuration. | `any` | `{}` | no | | [syncer\_lambda\_s3\_key](#input\_syncer\_lambda\_s3\_key) | S3 key for syncer lambda function. Required if using S3 bucket to specify lambdas. | `any` | `null` | no | | [syncer\_lambda\_s3\_object\_version](#input\_syncer\_lambda\_s3\_object\_version) | S3 object version for syncer lambda function. Useful if S3 versioning is enabled on source bucket. | `any` | `null` | no | diff --git a/modules/runners/README.md b/modules/runners/README.md index 9150f51c6d..d701010105 100644 --- a/modules/runners/README.md +++ b/modules/runners/README.md @@ -119,10 +119,11 @@ yarn run dist | [block\_device\_mappings](#input\_block\_device\_mappings) | The EC2 instance block device configuration. Takes the following keys: `device_name`, `delete_on_termination`, `volume_type`, `volume_size`, `encrypted`, `iops` | `map(string)` | `{}` | no | | [cloudwatch\_config](#input\_cloudwatch\_config) | (optional) Replaces the module default cloudwatch log config. See https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/CloudWatch-Agent-Configuration-File-Details.html for details. | `string` | `null` | no | | [create\_service\_linked\_role\_spot](#input\_create\_service\_linked\_role\_spot) | (optional) create the service linked role for spot instances that is required by the scale-up lambda. | `bool` | `false` | no | +| [disable\_runner\_autoupdate](#input\_disable\_runner\_autoupdate) | Disable the auto update of the github runner agent. Be-aware there is a grace period of 30 days, see also the [GitHub article](https://github.blog/changelog/2022-02-01-github-actions-self-hosted-runners-can-now-disable-automatic-updates/) | `bool` | `true` | no | | [egress\_rules](#input\_egress\_rules) | List of egress rules for the GitHub runner instances. | 
list(object({
    cidr_blocks      = list(string)
    ipv6_cidr_blocks = list(string)
    prefix_list_ids  = list(string)
    from_port        = number
    protocol         = string
    security_groups  = list(string)
    self             = bool
    to_port          = number
    description      = string
  }))
| 
[
  {
    "cidr_blocks": [
      "0.0.0.0/0"
    ],
    "description": null,
    "from_port": 0,
    "ipv6_cidr_blocks": [
      "::/0"
    ],
    "prefix_list_ids": null,
    "protocol": "-1",
    "security_groups": null,
    "self": null,
    "to_port": 0
  }
]
| no | | [enable\_cloudwatch\_agent](#input\_enable\_cloudwatch\_agent) | Enabling the cloudwatch agent on the ec2 runner instances, the runner contains default config. Configuration can be overridden via `cloudwatch_config`. | `bool` | `true` | no | | [enable\_ephemeral\_runners](#input\_enable\_ephemeral\_runners) | Enable ephemeral runners, runners will only be used once. | `bool` | `false` | no | -| [enable\_managed\_runner\_security\_group](#inputenable\_managed\_runner\_security\_group) | Enabling the default managed security group creation. Unmanaged security groups can be specified via `runner_additional_security_group_ids`. | `bool` | `true` | no | +| [enable\_managed\_runner\_security\_group](#input\_enable\_managed\_runner\_security\_group) | Enabling the default managed security group creation. Unmanaged security groups can be specified via `runner_additional_security_group_ids`. | `bool` | `true` | no | | [enable\_organization\_runners](#input\_enable\_organization\_runners) | n/a | `bool` | n/a | yes | | [enable\_ssm\_on\_runners](#input\_enable\_ssm\_on\_runners) | Enable to allow access to the runner instances for debugging purposes via SSM. Note that this adds additional permissions to the runner instances. | `bool` | n/a | yes | | [enabled\_userdata](#input\_enabled\_userdata) | Should the userdata script be enabled for the runner. Set this to false if you are using your own prebuilt AMI | `bool` | `true` | no | @@ -167,7 +168,7 @@ yarn run dist | [runner\_group\_name](#input\_runner\_group\_name) | Name of the runner group. | `string` | `"Default"` | no | | [runner\_iam\_role\_managed\_policy\_arns](#input\_runner\_iam\_role\_managed\_policy\_arns) | Attach AWS or customer-managed IAM policies (by ARN) to the runner IAM role | `list(string)` | `[]` | no | | [runner\_log\_files](#input\_runner\_log\_files) | (optional) List of logfiles to send to CloudWatch, will only be used if `enable_cloudwatch_agent` is set to true. Object description: `log_group_name`: Name of the log group, `prefix_log_group`: If true, the log group name will be prefixed with `/github-self-hosted-runners/`, `file_path`: path to the log file, `log_stream_name`: name of the log stream. | 
list(object({
    log_group_name   = string
    prefix_log_group = bool
    file_path        = string
    log_stream_name  = string
  }))
| `null` | no | -| [runner\_os](#input\_runner\_os) | The EC2 Operating System type to use for action runner instances (linux,win). | `string` | `"linux"` | no | +| [runner\_os](#input\_runner\_os) | The EC2 Operating System type to use for action runner instances (linux,windows). | `string` | `"linux"` | no | | [runner\_run\_as](#input\_runner\_run\_as) | Run the GitHub actions agent as user. | `string` | `"ec2-user"` | no | | [runners\_lambda\_s3\_key](#input\_runners\_lambda\_s3\_key) | S3 key for runners lambda function. Required if using S3 bucket to specify lambdas. | `any` | `null` | no | | [runners\_lambda\_s3\_object\_version](#input\_runners\_lambda\_s3\_object\_version) | S3 object version for runners lambda function. Useful if S3 versioning is enabled on source bucket. | `any` | `null` | no | diff --git a/modules/runners/lambdas/runners/src/pool/pool.ts b/modules/runners/lambdas/runners/src/pool/pool.ts index eea185ef5d..e98200073b 100644 --- a/modules/runners/lambdas/runners/src/pool/pool.ts +++ b/modules/runners/lambdas/runners/src/pool/pool.ts @@ -21,6 +21,7 @@ export async function adjust(event: PoolEvent): Promise { const instanceTypes = process.env.INSTANCE_TYPES.split(','); const instanceTargetTargetCapacityType = process.env.INSTANCE_TARGET_CAPACITY_TYPE; const ephemeral = yn(process.env.ENABLE_EPHEMERAL_RUNNERS, { default: false }); + const disableAutoUpdate = yn(process.env.DISABLE_RUNNER_AUTOUPDATE, { default: false }); const launchTemplateName = process.env.LAUNCH_TEMPLATE_NAME; const instanceMaxSpotPrice = process.env.INSTANCE_MAX_SPOT_PRICE; const instanceAllocationStrategy = process.env.INSTANCE_ALLOCATION_STRATEGY || 'lowest-price'; // same as AWS default @@ -60,7 +61,15 @@ export async function adjust(event: PoolEvent): Promise { if (topUp > 0) { logger.info(`The pool will be topped up with ${topUp} runners.`); await createRunners( - { ephemeral, ghesBaseUrl, runnerExtraLabels, runnerGroup, runnerOwner, runnerType: 'Org' }, + { + ephemeral, + ghesBaseUrl, + runnerExtraLabels, + runnerGroup, + runnerOwner, + runnerType: 'Org', + disableAutoUpdate: disableAutoUpdate, + }, { ec2instanceCriteria: { instanceTypes, diff --git a/modules/runners/lambdas/runners/src/scale-runners/scale-up.test.ts b/modules/runners/lambdas/runners/src/scale-runners/scale-up.test.ts index 00158b9bf4..09e143ee46 100644 --- a/modules/runners/lambdas/runners/src/scale-runners/scale-up.test.ts +++ b/modules/runners/lambdas/runners/src/scale-runners/scale-up.test.ts @@ -528,6 +528,13 @@ describe('scaleUp with public GH', () => { expect(createRunner).toBeCalledWith(expectedRunnerParams); }); + it('disable auto update on the runner.', async () => { + process.env.DISABLE_RUNNER_AUTOUPDATE = 'true'; + await scaleUpModule.scaleUp('aws:sqs', TEST_DATA); + expectedRunnerParams.runnerServiceConfig = expectedRunnerParams.runnerServiceConfig + ` --disableupdate`; + expect(createRunner).toBeCalledWith(expectedRunnerParams); + }); + it('Scaling error should cause reject so retry can be triggered.', async () => { process.env.RUNNERS_MAXIMUM_COUNT = '1'; process.env.ENABLE_EPHEMERAL_RUNNERS = 'true'; diff --git a/modules/runners/lambdas/runners/src/scale-runners/scale-up.ts b/modules/runners/lambdas/runners/src/scale-runners/scale-up.ts index 8e6106e47f..f643c95813 100644 --- a/modules/runners/lambdas/runners/src/scale-runners/scale-up.ts +++ b/modules/runners/lambdas/runners/src/scale-runners/scale-up.ts @@ -23,6 +23,7 @@ interface CreateGitHubRunnerConfig { runnerGroup: string | undefined; runnerOwner: string; runnerType: 'Org' | 'Repo'; + disableAutoUpdate: boolean; } interface CreateEC2RunnerConfig { @@ -40,7 +41,8 @@ function generateRunnerServiceConfig(githubRunnerConfig: CreateGitHubRunnerConfi githubRunnerConfig.runnerGroup !== undefined ? `--runnergroup ${githubRunnerConfig.runnerGroup} ` : ''; const configBaseUrl = githubRunnerConfig.ghesBaseUrl ? githubRunnerConfig.ghesBaseUrl : 'https://github.com'; const ephemeralArgument = githubRunnerConfig.ephemeral ? '--ephemeral ' : ''; - const runnerArgs = `--token ${token} ${labelsArgument}${ephemeralArgument}`; + const disableUpdateArgument = githubRunnerConfig.disableAutoUpdate ? '--disableupdate ' : ''; + const runnerArgs = `--token ${token} ${labelsArgument}${ephemeralArgument}${disableUpdateArgument}`; return githubRunnerConfig.runnerType === 'Org' ? `--url ${configBaseUrl}/${githubRunnerConfig.runnerOwner} ${runnerArgs}${runnerGroupArgument}`.trim() : `--url ${configBaseUrl}/${githubRunnerConfig.runnerOwner} ${runnerArgs}`.trim(); @@ -141,6 +143,7 @@ export async function scaleUp(eventSource: string, payload: ActionRequestMessage const instanceTypes = process.env.INSTANCE_TYPES.split(','); const instanceTargetTargetCapacityType = process.env.INSTANCE_TARGET_CAPACITY_TYPE; const ephemeralEnabled = yn(process.env.ENABLE_EPHEMERAL_RUNNERS, { default: false }); + const disableAutoUpdate = yn(process.env.DISABLE_RUNNER_AUTOUPDATE, { default: false }); const launchTemplateName = process.env.LAUNCH_TEMPLATE_NAME; const instanceMaxSpotPrice = process.env.INSTANCE_MAX_SPOT_PRICE; const instanceAllocationStrategy = process.env.INSTANCE_ALLOCATION_STRATEGY || 'lowest-price'; // same as AWS default @@ -195,6 +198,7 @@ export async function scaleUp(eventSource: string, payload: ActionRequestMessage runnerGroup, runnerOwner, runnerType, + disableAutoUpdate, }, { ec2instanceCriteria: { diff --git a/modules/runners/pool.tf b/modules/runners/pool.tf index 235ee877dd..6c749a5ca5 100644 --- a/modules/runners/pool.tf +++ b/modules/runners/pool.tf @@ -32,12 +32,13 @@ module "pool" { role_path = local.role_path role_permissions_boundary = var.role_permissions_boundary runner = { - ephemeral = var.enable_ephemeral_runners - extra_labels = var.runner_extra_labels - launch_template = aws_launch_template.runner - group_name = var.runner_group_name - pool_owner = var.pool_runner_owner - role = aws_iam_role.runner + disable_runner_autoupdate = var.disable_runner_autoupdate + ephemeral = var.enable_ephemeral_runners + extra_labels = var.runner_extra_labels + launch_template = aws_launch_template.runner + group_name = var.runner_group_name + pool_owner = var.pool_runner_owner + role = aws_iam_role.runner } subnet_ids = var.subnet_ids tags = local.tags diff --git a/modules/runners/pool/main.tf b/modules/runners/pool/main.tf index 7de74454f7..8c8c24bc04 100644 --- a/modules/runners/pool/main.tf +++ b/modules/runners/pool/main.tf @@ -16,9 +16,14 @@ resource "aws_lambda_function" "pool" { environment { variables = { - RUNNER_OWNER = var.config.runner.pool_owner + DISABLE_RUNNER_AUTOUPDATE = var.config.runner.disable_runner_autoupdate + ENABLE_EPHEMERAL_RUNNERS = var.config.runner.ephemeral ENVIRONMENT = var.config.environment GHES_URL = var.config.ghes.url + INSTANCE_ALLOCATION_STRATEGY = var.config.instance_allocation_strategy + INSTANCE_MAX_SPOT_PRICE = var.config.instance_max_spot_price + INSTANCE_TARGET_CAPACITY_TYPE = var.config.instance_target_capacity_type + INSTANCE_TYPES = join(",", var.config.instance_types) LAUNCH_TEMPLATE_NAME = var.config.runner.launch_template.name LOG_LEVEL = var.config.lambda.log_level LOG_TYPE = var.config.lambda.log_type @@ -27,12 +32,8 @@ resource "aws_lambda_function" "pool" { PARAMETER_GITHUB_APP_KEY_BASE64_NAME = var.config.github_app_parameters.key_base64.name RUNNER_EXTRA_LABELS = var.config.runner.extra_labels RUNNER_GROUP_NAME = var.config.runner.group_name + RUNNER_OWNER = var.config.runner.pool_owner SUBNET_IDS = join(",", var.config.subnet_ids) - ENABLE_EPHEMERAL_RUNNERS = var.config.runner.ephemeral - INSTANCE_TYPES = join(",", var.config.instance_types) - INSTANCE_TARGET_CAPACITY_TYPE = var.config.instance_target_capacity_type - INSTANCE_MAX_SPOT_PRICE = var.config.instance_max_spot_price - INSTANCE_ALLOCATION_STRATEGY = var.config.instance_allocation_strategy } } diff --git a/modules/runners/pool/variables.tf b/modules/runners/pool/variables.tf index 66b8eeadc2..0fcbe345f9 100644 --- a/modules/runners/pool/variables.tf +++ b/modules/runners/pool/variables.tf @@ -24,8 +24,9 @@ variable "config" { }) subnet_ids = list(string) runner = object({ - ephemeral = bool - extra_labels = string + disable_runner_autoupdate = bool + ephemeral = bool + extra_labels = string launch_template = object({ name = string }) diff --git a/modules/runners/scale-up.tf b/modules/runners/scale-up.tf index 632439520e..234e4f7667 100644 --- a/modules/runners/scale-up.tf +++ b/modules/runners/scale-up.tf @@ -15,9 +15,15 @@ resource "aws_lambda_function" "scale_up" { environment { variables = { + DISABLE_RUNNER_AUTOUPDATE = var.disable_runner_autoupdate + ENABLE_EPHEMERAL_RUNNERS = var.enable_ephemeral_runners ENABLE_ORGANIZATION_RUNNERS = var.enable_organization_runners ENVIRONMENT = var.environment GHES_URL = var.ghes_url + INSTANCE_ALLOCATION_STRATEGY = var.instance_allocation_strategy + INSTANCE_MAX_SPOT_PRICE = var.instance_max_spot_price + INSTANCE_TARGET_CAPACITY_TYPE = var.instance_target_capacity_type + INSTANCE_TYPES = join(",", var.instance_types) LAUNCH_TEMPLATE_NAME = aws_launch_template.runner.name LOG_LEVEL = var.log_level LOG_TYPE = var.log_type @@ -28,11 +34,6 @@ resource "aws_lambda_function" "scale_up" { RUNNER_GROUP_NAME = var.runner_group_name RUNNERS_MAXIMUM_COUNT = var.runners_maximum_count SUBNET_IDS = join(",", var.subnet_ids) - ENABLE_EPHEMERAL_RUNNERS = var.enable_ephemeral_runners - INSTANCE_TYPES = join(",", var.instance_types) - INSTANCE_TARGET_CAPACITY_TYPE = var.instance_target_capacity_type - INSTANCE_MAX_SPOT_PRICE = var.instance_max_spot_price - INSTANCE_ALLOCATION_STRATEGY = var.instance_allocation_strategy } } diff --git a/modules/runners/variables.tf b/modules/runners/variables.tf index 1a4802a44d..59d93f6749 100644 --- a/modules/runners/variables.tf +++ b/modules/runners/variables.tf @@ -495,3 +495,9 @@ variable "pool_config" { })) default = [] } + +variable "disable_runner_autoupdate" { + description = "Disable the auto update of the github runner agent. Be-aware there is a grace period of 30 days, see also the [GitHub article](https://github.blog/changelog/2022-02-01-github-actions-self-hosted-runners-can-now-disable-automatic-updates/)" + type = bool + default = true +} diff --git a/variables.tf b/variables.tf index a6e635fe08..5865e32225 100644 --- a/variables.tf +++ b/variables.tf @@ -585,3 +585,9 @@ variable "pool_config" { })) default = [] } + +variable "disable_runner_autoupdate" { + description = "Disable the auto update of the github runner agent. Be-aware there is a grace period of 30 days, see also the [GitHub article](https://github.blog/changelog/2022-02-01-github-actions-self-hosted-runners-can-now-disable-automatic-updates/)" + type = bool + default = false +}