From 90d5278c0f647c354d89779043cc32cface6e30e Mon Sep 17 00:00:00 2001 From: Dalton Hubble Date: Mon, 18 Jun 2018 21:57:58 -0700 Subject: [PATCH] Change kube-apiserver port from 443 to 6443 * Adjust firewall rules, security groups, cloud load balancers, and generated kubeconfig's * Facilitates some future simplifications and cost reductions * Bare-Metal users who exposed kube-apiserver on a WAN via their router or load balancer will need to adjust its configuration. This is uncommon, most apiserver are on LAN and/or behind VPN so no routing infrastructure is configured with the port number --- CHANGES.md | 13 +++++++++++++ aws/container-linux/kubernetes/apiserver.tf | 8 ++++---- aws/container-linux/kubernetes/bootkube.tf | 2 +- aws/container-linux/kubernetes/security.tf | 4 ++-- aws/fedora-atomic/kubernetes/bootkube.tf | 2 +- aws/fedora-atomic/kubernetes/security.tf | 4 ++-- bare-metal/container-linux/kubernetes/bootkube.tf | 2 +- bare-metal/fedora-atomic/kubernetes/bootkube.tf | 2 +- .../container-linux/kubernetes/bootkube.tf | 2 +- digital-ocean/container-linux/kubernetes/network.tf | 7 ++++++- digital-ocean/fedora-atomic/kubernetes/bootkube.tf | 2 +- digital-ocean/fedora-atomic/kubernetes/network.tf | 7 ++++++- docs/addons/ingress.md | 2 +- google-cloud/container-linux/kubernetes/bootkube.tf | 4 +++- google-cloud/fedora-atomic/kubernetes/bootkube.tf | 5 ++++- 15 files changed, 47 insertions(+), 19 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 417c967a7..c00effe2f 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -6,10 +6,23 @@ Notable changes between versions. * Update etcd from v3.3.6 to v3.3.8 ([#243](https://github.com/poseidon/typhoon/pull/243), [#247](https://github.com/poseidon/typhoon/pull/247)) +#### AWS + +* Switch `kube-apiserver` port from 443 to 6443 ([#248](https://github.com/poseidon/typhoon/pull/248)) + * Adjust NLB, security groups, and generated kubeconfig's + * Facilitates some future simplifications and cost reductions + #### Bare-Metal +* Switch `kube-apiserver` port from 443 to 6443 ([#248](https://github.com/poseidon/typhoon/pull/248)) + * Adjust router or load balancer configuration for 6443 if you exposed apiserver on a WAN (uncommon, most apiservers are internal or behind VPN) * Fix possible deadlock when provisioning clusters larger than 10 nodes ([#244](https://github.com/poseidon/typhoon/pull/244)) +#### DigitalOcean + +* Switch `kube-apiserver` port from 443 to 6443 ([#248](https://github.com/poseidon/typhoon/pull/248)) + * Adjust firewall rules and generated kubeconfig's + #### Addons * Update CLUO from v0.6.0 to v0.7.0 ([#242](https://github.com/poseidon/typhoon/pull/242)) diff --git a/aws/container-linux/kubernetes/apiserver.tf b/aws/container-linux/kubernetes/apiserver.tf index 8cc5eed6e..93cbcee45 100644 --- a/aws/container-linux/kubernetes/apiserver.tf +++ b/aws/container-linux/kubernetes/apiserver.tf @@ -28,7 +28,7 @@ resource "aws_lb" "apiserver" { resource "aws_lb_listener" "apiserver-https" { load_balancer_arn = "${aws_lb.apiserver.arn}" protocol = "TCP" - port = "443" + port = "6443" default_action { type = "forward" @@ -43,12 +43,12 @@ resource "aws_lb_target_group" "controllers" { target_type = "instance" protocol = "TCP" - port = 443 + port = 6443 # TCP health check for apiserver health_check { protocol = "TCP" - port = 443 + port = 6443 # NLBs required to use same healthy and unhealthy thresholds healthy_threshold = 3 @@ -65,5 +65,5 @@ resource "aws_lb_target_group_attachment" "controllers" { target_group_arn = "${aws_lb_target_group.controllers.arn}" target_id = "${element(aws_instance.controllers.*.id, count.index)}" - port = 443 + port = 6443 } diff --git a/aws/container-linux/kubernetes/bootkube.tf b/aws/container-linux/kubernetes/bootkube.tf index 4da810ccc..bb4694a6f 100644 --- a/aws/container-linux/kubernetes/bootkube.tf +++ b/aws/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=2bcf61b2b5f6268fcf99e6b803165b08fd0b73c0" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/aws/container-linux/kubernetes/security.tf b/aws/container-linux/kubernetes/security.tf index 8534200be..8104080e6 100644 --- a/aws/container-linux/kubernetes/security.tf +++ b/aws/container-linux/kubernetes/security.tf @@ -36,8 +36,8 @@ resource "aws_security_group_rule" "controller-apiserver" { type = "ingress" protocol = "tcp" - from_port = 443 - to_port = 443 + from_port = 6443 + to_port = 6443 cidr_blocks = ["0.0.0.0/0"] } diff --git a/aws/fedora-atomic/kubernetes/bootkube.tf b/aws/fedora-atomic/kubernetes/bootkube.tf index 7f990934e..a2c3aabcd 100644 --- a/aws/fedora-atomic/kubernetes/bootkube.tf +++ b/aws/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=2bcf61b2b5f6268fcf99e6b803165b08fd0b73c0" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/aws/fedora-atomic/kubernetes/security.tf b/aws/fedora-atomic/kubernetes/security.tf index 8534200be..8104080e6 100644 --- a/aws/fedora-atomic/kubernetes/security.tf +++ b/aws/fedora-atomic/kubernetes/security.tf @@ -36,8 +36,8 @@ resource "aws_security_group_rule" "controller-apiserver" { type = "ingress" protocol = "tcp" - from_port = 443 - to_port = 443 + from_port = 6443 + to_port = 6443 cidr_blocks = ["0.0.0.0/0"] } diff --git a/bare-metal/container-linux/kubernetes/bootkube.tf b/bare-metal/container-linux/kubernetes/bootkube.tf index 7662cb65c..b5ed364dd 100644 --- a/bare-metal/container-linux/kubernetes/bootkube.tf +++ b/bare-metal/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=2bcf61b2b5f6268fcf99e6b803165b08fd0b73c0" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/bare-metal/fedora-atomic/kubernetes/bootkube.tf b/bare-metal/fedora-atomic/kubernetes/bootkube.tf index df40c89f2..9b71c6570 100644 --- a/bare-metal/fedora-atomic/kubernetes/bootkube.tf +++ b/bare-metal/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=2bcf61b2b5f6268fcf99e6b803165b08fd0b73c0" cluster_name = "${var.cluster_name}" api_servers = ["${var.k8s_domain_name}"] diff --git a/digital-ocean/container-linux/kubernetes/bootkube.tf b/digital-ocean/container-linux/kubernetes/bootkube.tf index 1195cae05..392098992 100644 --- a/digital-ocean/container-linux/kubernetes/bootkube.tf +++ b/digital-ocean/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=2bcf61b2b5f6268fcf99e6b803165b08fd0b73c0" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/digital-ocean/container-linux/kubernetes/network.tf b/digital-ocean/container-linux/kubernetes/network.tf index fc257a030..312d7966e 100644 --- a/digital-ocean/container-linux/kubernetes/network.tf +++ b/digital-ocean/container-linux/kubernetes/network.tf @@ -3,7 +3,7 @@ resource "digitalocean_firewall" "rules" { tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] - # allow ssh, http/https ingress, and peer-to-peer traffic + # allow ssh, apiserver, http/https ingress, and peer-to-peer traffic inbound_rule = [ { protocol = "tcp" @@ -20,6 +20,11 @@ resource "digitalocean_firewall" "rules" { port_range = "443" source_addresses = ["0.0.0.0/0", "::/0"] }, + { + protocol = "tcp" + port_range = "6443" + source_addresses = ["0.0.0.0/0", "::/0"] + }, { protocol = "udp" port_range = "1-65535" diff --git a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf index b9b57058b..adbbb3a71 100644 --- a/digital-ocean/fedora-atomic/kubernetes/bootkube.tf +++ b/digital-ocean/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=2bcf61b2b5f6268fcf99e6b803165b08fd0b73c0" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] diff --git a/digital-ocean/fedora-atomic/kubernetes/network.tf b/digital-ocean/fedora-atomic/kubernetes/network.tf index fc257a030..312d7966e 100644 --- a/digital-ocean/fedora-atomic/kubernetes/network.tf +++ b/digital-ocean/fedora-atomic/kubernetes/network.tf @@ -3,7 +3,7 @@ resource "digitalocean_firewall" "rules" { tags = ["${var.cluster_name}-controller", "${var.cluster_name}-worker"] - # allow ssh, http/https ingress, and peer-to-peer traffic + # allow ssh, apiserver, http/https ingress, and peer-to-peer traffic inbound_rule = [ { protocol = "tcp" @@ -20,6 +20,11 @@ resource "digitalocean_firewall" "rules" { port_range = "443" source_addresses = ["0.0.0.0/0", "::/0"] }, + { + protocol = "tcp" + port_range = "6443" + source_addresses = ["0.0.0.0/0", "::/0"] + }, { protocol = "udp" port_range = "1-65535" diff --git a/docs/addons/ingress.md b/docs/addons/ingress.md index 754bec808..e6c9e81e0 100644 --- a/docs/addons/ingress.md +++ b/docs/addons/ingress.md @@ -4,7 +4,7 @@ Nginx Ingress controller pods accept and demultiplex HTTP, HTTPS, TCP, or UDP tr ## AWS -On AWS, an elastic load balancer distributes traffic across worker nodes (i.e. an auto-scaling group) running an Ingress controller deployment on host ports 80 and 443. Firewall rules allow traffic to ports 80 and 443. Health check rules ensure only workers with a health Ingress controller receive traffic. +On AWS, a network load balancer (NLB) distributes traffic across a target group of worker nodes running an Ingress controller deployment on host ports 80 and 443. Firewall rules allow traffic to ports 80 and 443. Health check rules ensure only workers with a health Ingress controller receive traffic. Create the Ingress controller deployment, service, RBAC roles, RBAC bindings, default backend, and namespace. diff --git a/google-cloud/container-linux/kubernetes/bootkube.tf b/google-cloud/container-linux/kubernetes/bootkube.tf index 51ab437a8..1e1e9da54 100644 --- a/google-cloud/container-linux/kubernetes/bootkube.tf +++ b/google-cloud/container-linux/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=2bcf61b2b5f6268fcf99e6b803165b08fd0b73c0" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] @@ -11,4 +11,6 @@ module "bootkube" { pod_cidr = "${var.pod_cidr}" service_cidr = "${var.service_cidr}" cluster_domain_suffix = "${var.cluster_domain_suffix}" + // temporary + apiserver_port = 443 } diff --git a/google-cloud/fedora-atomic/kubernetes/bootkube.tf b/google-cloud/fedora-atomic/kubernetes/bootkube.tf index 4a93e35df..fc516b13b 100644 --- a/google-cloud/fedora-atomic/kubernetes/bootkube.tf +++ b/google-cloud/fedora-atomic/kubernetes/bootkube.tf @@ -1,6 +1,6 @@ # Self-hosted Kubernetes assets (kubeconfig, manifests) module "bootkube" { - source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=0e98e89e14a074768db13c4e050ed0c13319a0c1" + source = "git::https://github.com/poseidon/terraform-render-bootkube.git?ref=2bcf61b2b5f6268fcf99e6b803165b08fd0b73c0" cluster_name = "${var.cluster_name}" api_servers = ["${format("%s.%s", var.cluster_name, var.dns_zone)}"] @@ -14,4 +14,7 @@ module "bootkube" { # Fedora trusted_certs_dir = "/etc/pki/tls/certs" + + // temporary + apiserver_port = 443 }