diff --git a/README.md b/README.md index 719d570..e817900 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,71 @@ # Nomad Cluster Setup -This repository contains Terraform modules for setting up a [Nomad](https://www.nomadproject.io/) cluster on AWS with server and client nodes. +Terraform modules to deploy a [HashiCorp Nomad]((https://www.nomadproject.io/)) cluster on AWS using an Auto Scaling Group (ASG). The modules are designed to provision Nomad servers and clients in ASG, making it easy to manage the infrastructure for Nomad cluster. Additionally, the repository includes Packer scripts to build a custom Amazon Machine Image (AMI) with Nomad pre-installed. + +![Nomad architecture](./docs/architecture.png) + +- [Nomad Cluster Setup](#nomad-cluster-setup) + - [AMI](#ami) + - [AWS Resources](#aws-resources) + - [Auto Scaling Group (ASG)](#auto-scaling-group-asg) + - [Security Group](#security-group) + - [IAM Role](#iam-role) + - [ALB](#alb) + - [Nomad Server](#nomad-server) + - [Example Usage](#example-usage) + - [Inputs](#inputs) + - [Outputs](#outputs) + - [Nomad Client](#nomad-client) + - [Example Usage](#example-usage-1) + - [Inputs](#inputs-1) + - [Outputs](#outputs-1) + - [Contributors](#contributors) + - [Contributing](#contributing) + - [LICENSE](#license) + +## AMI + +The repository includes a [Packer file](./packer/ami.pkr.hcl), to build a custom Amazon Machine Image (AMI) with Nomad and `docker` pre-installed. This AMI is used by the Terraform modules when creating the ASG instances. + +To build the AMI, run: + +```bash +cd packer +make build +``` + +NOTE: `dry_run` mode is toggled as true by default. To build the AMI, set the `dry_run` variable in [`Makefile`](./packer/Makefile) to `false`. + +## AWS Resources + +The key resources provisioned by this module are: + +1. Auto Scaling Group (ASG) +2. Security Group +3. IAM Role +4. Application Load Balancer (ALB) (optional) + +### Auto Scaling Group (ASG) + +The module deploys Nomad on top of an Auto Scaling Group (ASG). For optimal performance and fault tolerance, it is recommended to run the Nomad server ASG with 3 or 5 EC2 instances distributed across multiple Availability Zones. Each EC2 instance should utilize an AMI built using the provided Packer script. + +### Security Group + +Each EC2 instance within the ASG is assigned a Security Group that permits: + +- All outbound requests +- All inbound ports specified in the [Nomad documentation](https://developer.hashicorp.com/nomad/docs/install/production/requirements#ports-used) + +The common Security Group is attached to both client and server nodes, enabling the Nomad agent to communicate and discover other agents within the cluster. The Security Group ID is exposed as an output variable for adding additional rules as needed. Furthermore, you can provide your own list of security groups as a variable to the module. + +### IAM Role + +An IAM Role is attached to each EC2 instance within the ASG. This role is granted a minimal set of IAM permissions, allowing each instance to automatically discover other instances in the same ASG and form a cluster with them. + +### ALB + +An internal Application Load Balancer (ALB) is _optionally_ created for the Nomad servers. The ALB is configured to listen on port 80/443 and forward requests to the Nomad servers on port 4646. The ALB is exposed as an output variable for adding additional rules as needed. ## Nomad Server @@ -100,6 +164,7 @@ module "nomad_client_kite_alerts" { } ``` + ### Inputs @@ -148,6 +213,11 @@ module "nomad_client_kite_alerts" { - [Chinmay](https://github.com/thunderbottom) +## Contributing + +Contributions to this repository are welcome. Please submit a pull request or open an issue to suggest improvements or report bugs. + + ## LICENSE [LICENSE](./LICENSE) diff --git a/docs/architecture.png b/docs/architecture.png index 5f6e97b..09d20bc 100644 Binary files a/docs/architecture.png and b/docs/architecture.png differ diff --git a/packer/Makefile b/packer/Makefile new file mode 100644 index 0000000..6de063e --- /dev/null +++ b/packer/Makefile @@ -0,0 +1,11 @@ +DRY_RUN := true + +.PHONY: build +build: + PKR_VAR_dry_run=${DRY_RUN} PKR_VAR_install_docker=true \ + packer build ./ami.pkr.hcl + +.PHONY: build-no-docker +build-no-docker: + PKR_VAR_dry_run=${DRY_RUN} PKR_VAR_install_docker=false \ + packer build ./ami.pkr.hcl diff --git a/packer/ami.pkr.hcl b/packer/ami.pkr.hcl new file mode 100644 index 0000000..025f911 --- /dev/null +++ b/packer/ami.pkr.hcl @@ -0,0 +1,82 @@ +packer { + required_version = ">= 1.8.0" +} + +locals { + timestamp = regex_replace(timestamp(), "[- TZ:]", "") + ami_prefix = "nomad-${var.nomad_version}" +} + +variable "aws_region" { + type = string + default = "ap-south-1" +} + +variable "dry_run" { + type = bool + default = true +} + +variable "nomad_version" { + type = string + default = "1.5.3" +} + +variable "install_docker" { + type = bool + default = false +} + +// Get the most recent Ubuntu AMI ID. +data "amazon-ami" "autogenerated_1" { + filters = { + architecture = "x86_64" + "block-device-mapping.volume-type" = "gp2" + name = "ubuntu/images/hvm-ssd/ubuntu-jammy-22.04-amd64-server-*" + root-device-type = "ebs" + virtualization-type = "hvm" + } + + most_recent = true + owners = ["099720109477"] + region = var.aws_region +} + +source "amazon-ebs" "nomad" { + skip_create_ami = var.dry_run + + # https://github.com/hashicorp/packer/issues/11656#issuecomment-1106564406 + temporary_key_pair_type = "ed25519" + + ami_description = "AMI for Nomad agents based on Ubuntu" + ami_name = var.install_docker ? "${local.ami_prefix}-docker-${local.timestamp}" : "${local.ami_prefix}-${local.timestamp}" + + instance_type = "t2.micro" + shutdown_behavior = "terminate" + force_deregister = true + force_delete_snapshot = true + tags = { + nomad_version = var.nomad_version + source_ami_id = "{{ .SourceAMI }}" + source_ami_name = "{{ .SourceAMIName }}" + Name = var.install_docker ? "${local.ami_prefix}-docker" : "${local.ami_prefix}" + } + + region = var.aws_region + source_ami = data.amazon-ami.autogenerated_1.id + ssh_username = "ubuntu" +} + +build { + sources = ["source.amazon-ebs.nomad"] + + // wait for sometime for instance to properly initialise. + provisioner "shell" { + inline = ["sleep 15"] + } + + provisioner "shell" { + environment_vars = ["NOMAD_VERSION=${var.nomad_version}", "INSTALL_DOCKER=${var.install_docker}"] + script = "./setup.sh" + } +} \ No newline at end of file diff --git a/packer/setup.sh b/packer/setup.sh new file mode 100644 index 0000000..002814c --- /dev/null +++ b/packer/setup.sh @@ -0,0 +1,74 @@ +#!/bin/bash + +set -e + +# Disable interactive apt prompts +export DEBIAN_FRONTEND=noninteractive + +NOMAD_VERSION="${NOMAD_VERSION:-1.5.3}" +CNI_VERSION="${CNI_VERSION:-v1.2.0}" +INSTALL_DOCKER="${INSTALL_DOCKER:-false}" + +# Update packages +sudo apt-get -y update + +# Install software-properties-common +sudo apt-get install -y software-properties-common + +# Add HashiCorp GPG key +curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo apt-key add - + +# Add HashiCorp repository +sudo apt-add-repository "deb [arch=amd64] https://apt.releases.hashicorp.com $(lsb_release -cs) main" + +# Update packages again +sudo apt-get -y update + +# Install Nomad +sudo apt-get install -y nomad=${NOMAD_VERSION}-1 + +# Download and install CNI plugins +curl -L -o cni-plugins.tgz "https://github.com/containernetworking/plugins/releases/download/${CNI_VERSION}/cni-plugins-linux-$( [ $(uname -m) = aarch64 ] && echo arm64 || echo amd64)"-"${CNI_VERSION}".tgz && \ + sudo mkdir -p /opt/cni/bin && \ + sudo tar -C /opt/cni/bin -xzf cni-plugins.tgz + +# Stop and disable Nomad service +sudo systemctl stop nomad +sudo systemctl disable nomad + +# Install Docker if INSTALL_DOCKER is true +if [ "$INSTALL_DOCKER" = true ]; then + sudo apt-get -y update + sudo apt-get -y install \ + ca-certificates \ + curl \ + gnupg + sudo install -m 0755 -d /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg + sudo chmod a+r /etc/apt/keyrings/docker.gpg + echo \ + "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \ + "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \ + sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt-get -y update + sudo apt-get -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + + # Create daemon.json if it doesn't exist + if [ ! -f /etc/docker/daemon.json ]; then + sudo touch /etc/docker/daemon.json + fi + + # Add configuration to daemon.json + echo '{ + "log-driver": "json-file", + "log-opts": { + "max-size": "10m", + "max-file": "10" + } + }' | sudo tee /etc/docker/daemon.json >/dev/null + + + # Restart Docker + sudo systemctl restart docker + sudo usermod -aG docker ubuntu +fi