From aaa8e0261a01a62fa4a343788b3df20b9e93ee38 Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Sat, 30 Mar 2019 15:12:55 -0700 Subject: [PATCH] Add Google Cloud worker instances to a target pool * Background: A managed instance group of workers is used in backend services for global load balancing (HTTP/HTTPS Ingress) and output for custom global load balancing use cases * Add worker instances to a target pool load balancing TCP/UDP applications (NodePort or proxied). Output as `worker_target_pool` * Health check for workers with a healthy Ingress controller. Forward rules (regional) to target pools don't support different external and internal ports so choosing nodes with Ingress allows proxying as a workaround * A target pool is a logical grouping only. It doesn't add costs to clusters or worker pools --- CHANGES.md | 3 +++ .../container-linux/kubernetes/network.tf | 22 ++++++++++++++----- .../container-linux/kubernetes/outputs.tf | 7 +++++- .../kubernetes/workers/outputs.tf | 11 +++++++++- .../kubernetes/workers/target_pool.tf | 21 ++++++++++++++++++ .../kubernetes/workers/workers.tf | 5 +++-- 6 files changed, 60 insertions(+), 9 deletions(-) create mode 100644 google-cloud/container-linux/kubernetes/workers/target_pool.tf diff --git a/CHANGES.md b/CHANGES.md index 95b392c95..6a774efe3 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -8,6 +8,9 @@ Notable changes between versions. #### Google Cloud +* Add ability to load balance TCP/UDP applications ([#442](https://github.com/poseidon/typhoon/pull/442)) + * Add worker instances to a target pool, output as `worker_target_pool` + * Health check for workers with Ingress controllers. Forward rules don't support differing internal/external ports, but some Ingress controllers support TCP/UDP proxy as a workaround * Remove Haswell minimum CPU platform requirement ([#439](https://github.com/poseidon/typhoon/pull/439)) * Google Cloud API implements `min_cpu_platform` to mean "use exactly this CPU". Revert [#405](https://github.com/poseidon/typhoon/pull/405) added in v1.13.4. * Fix error creating clusters in new regions without Haswell (e.g. europe-west2) ([#438](https://github.com/poseidon/typhoon/issues/438)) diff --git a/google-cloud/container-linux/kubernetes/network.tf b/google-cloud/container-linux/kubernetes/network.tf index a507ce8ea..04496186e 100644 --- a/google-cloud/container-linux/kubernetes/network.tf +++ b/google-cloud/container-linux/kubernetes/network.tf @@ -138,8 +138,8 @@ resource "google_compute_firewall" "allow-ingress" { target_tags = ["${var.cluster_name}-worker"] } -resource "google_compute_firewall" "google-health-checks" { - name = "${var.cluster_name}-google-health-checks" +resource "google_compute_firewall" "google-ingress-health-checks" { + name = "${var.cluster_name}-ingress-health" network = "${google_compute_network.network.name}" allow { @@ -147,7 +147,19 @@ resource "google_compute_firewall" "google-health-checks" { ports = [10254] } - # https://cloud.google.com/compute/docs/load-balancing/tcp-ssl/tcp-proxy#health-checking - source_ranges = ["130.211.0.0/22", "35.191.0.0/16"] - target_tags = ["${var.cluster_name}-worker"] + # https://cloud.google.com/load-balancing/docs/health-check-concepts#method + source_ranges = [ + # Global LB health checks + "35.191.0.0/16", + + "130.211.0.0/22", + + # Region LB health checks + "35.191.0.0/16", + + "209.85.152.0/22", + "209.85.204.0/22", + ] + + target_tags = ["${var.cluster_name}-worker"] } diff --git a/google-cloud/container-linux/kubernetes/outputs.tf b/google-cloud/container-linux/kubernetes/outputs.tf index f1a09ece4..aefdd9b81 100644 --- a/google-cloud/container-linux/kubernetes/outputs.tf +++ b/google-cloud/container-linux/kubernetes/outputs.tf @@ -33,6 +33,11 @@ output "network_self_link" { # Outputs for custom load balancing output "worker_instance_group" { - description = "Full URL of the worker managed instance group" + description = "Worker managed instance group full URL" value = "${module.workers.instance_group}" } + +output "worker_target_pool" { + description = "Worker target pool self link" + value = "${module.workers.target_pool}" +} diff --git a/google-cloud/container-linux/kubernetes/workers/outputs.tf b/google-cloud/container-linux/kubernetes/workers/outputs.tf index 4521d7923..0817d675d 100644 --- a/google-cloud/container-linux/kubernetes/workers/outputs.tf +++ b/google-cloud/container-linux/kubernetes/workers/outputs.tf @@ -1,4 +1,13 @@ +# Outputs for global load balancing + output "instance_group" { - description = "Full URL of the worker managed instance group" + description = "Worker managed instance group full URL" value = "${google_compute_region_instance_group_manager.workers.instance_group}" } + +# Outputs for regional load balancing + +output "target_pool" { + description = "Worker target pool self link" + value = "${google_compute_target_pool.workers.self_link}" +} diff --git a/google-cloud/container-linux/kubernetes/workers/target_pool.tf b/google-cloud/container-linux/kubernetes/workers/target_pool.tf new file mode 100644 index 000000000..949591d57 --- /dev/null +++ b/google-cloud/container-linux/kubernetes/workers/target_pool.tf @@ -0,0 +1,21 @@ +# Target pool for TCP/UDP load balancing +resource "google_compute_target_pool" "workers" { + name = "${var.name}-worker-pool" + session_affinity = "NONE" + + health_checks = [ + "${google_compute_http_health_check.workers.name}", + ] +} + +# HTTP Health Check (for TCP/UDP load balancing) +# Forward rules (regional) to target pools don't support different external +# and internal ports. Health check for nodes with Ingress controllers that +# may support proxying or otherwise satisfy the check. +resource "google_compute_http_health_check" "workers" { + name = "${var.name}-target-pool-health" + description = "Health check for the worker target pool" + + port = 10254 + request_path = "/healthz" +} diff --git a/google-cloud/container-linux/kubernetes/workers/workers.tf b/google-cloud/container-linux/kubernetes/workers/workers.tf index f0114ef3d..1dcf0cbc8 100644 --- a/google-cloud/container-linux/kubernetes/workers/workers.tf +++ b/google-cloud/container-linux/kubernetes/workers/workers.tf @@ -1,4 +1,4 @@ -# Regional managed instance group of workers +# Managed instance group of workers resource "google_compute_region_instance_group_manager" "workers" { name = "${var.name}-worker-group" description = "Compute instance group of ${var.name} workers" @@ -8,7 +8,8 @@ resource "google_compute_region_instance_group_manager" "workers" { instance_template = "${google_compute_instance_template.worker.self_link}" region = "${var.region}" - target_size = "${var.count}" + target_size = "${var.count}" + target_pools = ["${google_compute_target_pool.workers.self_link}"] named_port { name = "http"