From c243d930b026eaa8dcaa72d3ab58c139d851cb7b Mon Sep 17 00:00:00 2001 From: bprashanth Date: Tue, 31 Jan 2017 17:22:09 -0800 Subject: [PATCH] Add an example for static-ip and deployment --- examples/deployment/gce/README.md | 78 +++++++++++ .../gce/gce-ingress-controller.yaml | 82 +++++++++++ .../gce}/gce-tls-ingress.yaml | 0 examples/static-ip/gce.md | 3 - examples/static-ip/gce/README.md | 128 ++++++++++++++++++ examples/static-ip/gce/gce-http-ingress.yaml | 12 ++ .../static-ip/gce/gce-static-ip-ingress.yaml | 19 +++ examples/static-ip/nginx/README.md | 118 ++++++++++++++++ .../nginx/nginx-ingress-controller.yaml | 52 +++++++ examples/static-ip/nginx/nginx-ingress.yaml | 17 +++ examples/static-ip/nginx/static-ip-svc.yaml | 23 ++++ .../tls-termination/{gce.md => gce/README.md} | 0 .../tls-termination/gce/gce-tls-ingress.yaml | 15 ++ .../{nginx.md => nginx/README.md} | 0 .../{ => nginx}/nginx-tls-ingress.yaml | 0 15 files changed, 544 insertions(+), 3 deletions(-) create mode 100644 examples/deployment/gce/README.md create mode 100644 examples/deployment/gce/gce-ingress-controller.yaml rename examples/{tls-termination => deployment/gce}/gce-tls-ingress.yaml (100%) delete mode 100644 examples/static-ip/gce.md create mode 100644 examples/static-ip/gce/README.md create mode 100644 examples/static-ip/gce/gce-http-ingress.yaml create mode 100644 examples/static-ip/gce/gce-static-ip-ingress.yaml create mode 100644 examples/static-ip/nginx/README.md create mode 100644 examples/static-ip/nginx/nginx-ingress-controller.yaml create mode 100644 examples/static-ip/nginx/nginx-ingress.yaml create mode 100644 examples/static-ip/nginx/static-ip-svc.yaml rename examples/tls-termination/{gce.md => gce/README.md} (100%) create mode 100644 examples/tls-termination/gce/gce-tls-ingress.yaml rename examples/tls-termination/{nginx.md => nginx/README.md} (100%) rename examples/tls-termination/{ => nginx}/nginx-tls-ingress.yaml (100%) diff --git a/examples/deployment/gce/README.md b/examples/deployment/gce/README.md new file mode 100644 index 00000000000..be711de6e67 --- /dev/null +++ b/examples/deployment/gce/README.md @@ -0,0 +1,78 @@ +# Deploying the GCE Ingress controller + +This example demonstrates the deployment of a GCE Ingress controller. + +Note: __all GCE/GKE clusters already have an Ingress controller running +on the master. The only reason to deploy another GCE controller is if you want +to debug or otherwise observe its operation (eg via kubectl logs). Before +deploying another one in your cluster, make sure you disable the master +controller.__ + +## Disabling the master controller + +As of Kubernetes 1.3, GLBC runs as a static pod on the master. If you want to +totally disable it, you can ssh into the master node and delete the GLBC +manifest file found at `/etc/kubernetes/manifests/glbc.manifest`. You can also +disable it on GKE at cluster bring-up time through the `disable-addons` flag: + +```console +gcloud container clusters create mycluster --network "default" --num-nodes 1 \ +--machine-type n1-standard-2 --zone $ZONE \ +--disable-addons HttpLoadBalancing \ +--disk-size 50 --scopes storage-full +``` + +## Deploying a new controller + +The following command deploys a GCE Ingress controller in your cluster + +```console +$ kubectl create -f gce-ingress-controller.yaml +service "default-http-backend" created +replicationcontroller "l7-lb-controller" created + +$ kubectl get po -l name=glbc +NAME READY STATUS RESTARTS AGE +l7-lb-controller-1s22c 2/2 Running 0 27s +``` + +now you can create an Ingress and observe the controller + +```console +$ kubectl create -f gce-tls-ingress.yaml +ingress "test" created + +$ kubectl logs l7-lb-controller-1s22c -c l7-lb-controller +I0201 01:03:17.387548 1 main.go:179] Starting GLBC image: glbc:0.9.0, cluster name +I0201 01:03:18.459740 1 main.go:291] Using saved cluster uid "32658fa96c080068" +I0201 01:03:18.459771 1 utils.go:122] Changing cluster name from to 32658fa96c080068 +I0201 01:03:18.461652 1 gce.go:331] Using existing Token Source &oauth2.reuseTokenSource{new:google.computeSource{account:""}, mu:sync.Mutex{state:0, sema:0x0}, t:(*oauth2.Token)(nil)} +I0201 01:03:18.553142 1 cluster_manager.go:264] Created GCE client without a config file +I0201 01:03:18.553773 1 controller.go:234] Starting loadbalancer controller +I0201 01:04:58.314271 1 event.go:217] Event(api.ObjectReference{Kind:"Ingress", Namespace:"default", Name:"test", UID:"73549716-e81a-11e6-a8c5-42010af00002", APIVersion:"extensions", ResourceVersion:"673016", FieldPath:""}): type: 'Normal' reason: 'ADD' default/test +I0201 01:04:58.413616 1 instances.go:76] Creating instance group k8s-ig--32658fa96c080068 in zone us-central1-b +I0201 01:05:01.998169 1 gce.go:2084] Adding port 30301 to instance group k8s-ig--32658fa96c080068 with 0 ports +I0201 01:05:02.444014 1 backends.go:149] Creating backend for 1 instance groups, port 30301 named port &{port30301 30301 []} +I0201 01:05:02.444175 1 utils.go:495] No pod in service http-svc with node port 30301 has declared a matching readiness probe for health checks. +I0201 01:05:02.555599 1 healthchecks.go:62] Creating health check k8s-be-30301--32658fa96c080068 +I0201 01:05:11.300165 1 gce.go:2084] Adding port 31938 to instance group k8s-ig--32658fa96c080068 with 1 ports +I0201 01:05:11.743914 1 backends.go:149] Creating backend for 1 instance groups, port 31938 named port &{port31938 31938 []} +I0201 01:05:11.744008 1 utils.go:495] No pod in service default-http-backend with node port 31938 has declared a matching readiness probe for health checks. +I0201 01:05:11.811972 1 healthchecks.go:62] Creating health check k8s-be-31938--32658fa96c080068 +I0201 01:05:19.871791 1 loadbalancers.go:121] Creating l7 default-test--32658fa96c080068 +... + +$ kubectl get ing test +NAME HOSTS ADDRESS PORTS AGE +test * 35.186.208.106 80, 443 4m + +$ curl 35.186.208.106 -kL +CLIENT VALUES: +client_address=10.180.3.1 +command=GET +real path=/ +query=nil +request_version=1.1 +request_uri=http://35.186.208.106:8080/ +... +``` diff --git a/examples/deployment/gce/gce-ingress-controller.yaml b/examples/deployment/gce/gce-ingress-controller.yaml new file mode 100644 index 00000000000..a0a53cd3c67 --- /dev/null +++ b/examples/deployment/gce/gce-ingress-controller.yaml @@ -0,0 +1,82 @@ +apiVersion: v1 +kind: Service +metadata: + # This must match the --default-backend-service argument of the l7 lb + # controller and is required because GCE mandates a default backend. + name: default-http-backend + labels: + k8s-app: glbc +spec: + # The default backend must be of type NodePort. + type: NodePort + ports: + - port: 80 + targetPort: 8080 + protocol: TCP + name: http + selector: + k8s-app: glbc + +--- +apiVersion: v1 +kind: ReplicationController +metadata: + name: l7-lb-controller + labels: + k8s-app: glbc + version: v0.9.0 +spec: + # There should never be more than 1 controller alive simultaneously. + replicas: 1 + selector: + k8s-app: glbc + version: v0.9.0 + template: + metadata: + labels: + k8s-app: glbc + version: v0.9.0 + name: glbc + spec: + terminationGracePeriodSeconds: 600 + containers: + - name: default-http-backend + # Any image is permissable as long as: + # 1. It serves a 404 page at / + # 2. It serves 200 on a /healthz endpoint + image: gcr.io/google_containers/defaultbackend:1.0 + livenessProbe: + httpGet: + path: /healthz + port: 8080 + scheme: HTTP + initialDelaySeconds: 30 + timeoutSeconds: 5 + ports: + - containerPort: 8080 + resources: + limits: + cpu: 10m + memory: 20Mi + requests: + cpu: 10m + memory: 20Mi + - image: gcr.io/google_containers/glbc:0.9.0-beta.1 + livenessProbe: + httpGet: + path: /healthz + port: 8081 + scheme: HTTP + initialDelaySeconds: 30 + timeoutSeconds: 5 + name: l7-lb-controller + resources: + limits: + cpu: 100m + memory: 100Mi + requests: + cpu: 100m + memory: 50Mi + args: + - --default-backend-service=default/default-http-backend + - --sync-period=300s diff --git a/examples/tls-termination/gce-tls-ingress.yaml b/examples/deployment/gce/gce-tls-ingress.yaml similarity index 100% rename from examples/tls-termination/gce-tls-ingress.yaml rename to examples/deployment/gce/gce-tls-ingress.yaml diff --git a/examples/static-ip/gce.md b/examples/static-ip/gce.md deleted file mode 100644 index 92f00416051..00000000000 --- a/examples/static-ip/gce.md +++ /dev/null @@ -1,3 +0,0 @@ -# Static IPs through the GCE Ingress controller - -Placeholder diff --git a/examples/static-ip/gce/README.md b/examples/static-ip/gce/README.md new file mode 100644 index 00000000000..fe7ee92f7df --- /dev/null +++ b/examples/static-ip/gce/README.md @@ -0,0 +1,128 @@ +# Static IPs + +This example demonstrates how to assign a [static-ip](https://cloud.google.com/compute/docs/configure-instance-ip-addresses#reserve_new_static) to an Ingress on GCE. + +## Prerequisites + +You need a [TLS cert](/examples/PREREQUISITES.md#tls-certificates) and a [test HTTP service](/examples/PREREQUISITES.md#test-http-service) for this example. +You will also need to make sure you Ingress targets exactly one Ingress +controller by specifying the [ingress.class annotation](/examples/PREREQUISITES.md#ingress-class). + +## Acquiring a static IP + +In GCE, static IP belongs to a given project until the owner decides to release +it. If you create a static IP and assign it to an Ingress, deleting the Ingress +or tearing down the GKE cluster *will not* delete the static IP. You can check +the static IPs you have as follows + +```console +$ gcloud compute addresses list --global +NAME REGION ADDRESS STATUS +test-ip 35.186.221.137 RESERVED + +$ gcloud compute addresses list +NAME REGION ADDRESS STATUS +test-ip 35.186.221.137 RESERVED +test-ip us-central1 35.184.21.228 RESERVED +``` + +Note the difference between a regional and a global static ip. Only global +static-ips will work with Ingress. If you don't already have an IP, you can +create it + +```console +$ gcloud compute addresses create test-ip --global +Created [https://www.googleapis.com/compute/v1/projects/kubernetesdev/global/addresses/test-ip]. +--- +address: 35.186.221.137 +creationTimestamp: '2017-01-31T10:32:29.889-08:00' +description: '' +id: '9221457935391876818' +kind: compute#address +name: test-ip +selfLink: https://www.googleapis.com/compute/v1/projects/kubernetesdev/global/addresses/test-ip +status: RESERVED +``` + +## Assigning a static IP to an Ingress + +You can now add the static IP from the previous step to an Ingress, +by specifying the `kubernetes.io/global-static-ip-name` annotation, +the example yaml in this directory already has it set to `test-ip` + +```console +$ kubectl create -f gce-static-ip-ingress.yaml +ingress "static-ip" created + +$ gcloud compute addresses list test-ip +NAME REGION ADDRESS STATUS +test-ip 35.186.221.137 IN_USE +test-ip us-central1 35.184.21.228 RESERVED + +$ kubectl get ing +NAME HOSTS ADDRESS PORTS AGE +static-ip * 35.186.221.137 80, 443 1m + +$ curl 35.186.221.137 -Lk +CLIENT VALUES: +client_address=10.180.1.1 +command=GET +real path=/ +query=nil +request_version=1.1 +request_uri=http://35.186.221.137:8080/ +... +``` + +## Retaining the static IP + +You can test retention by deleting the Ingress + +```console +$ kubectl delete -f gce-static-ip-ingress.yaml +ingress "static-ip" deleted + +$ kubectl get ing +No resources found. + +$ gcloud compute addresses list test-ip --global +NAME REGION ADDRESS STATUS +test-ip 35.186.221.137 RESERVED +``` + +## Promote ephemeral to static IP + +If you simply create a HTTP Ingress resource, it gets an ephemeral IP + +```console +$ kubectl create -f gce-http-ingress.yaml +ingress "http-ingress" created + +$ kubectl get ing +NAME HOSTS ADDRESS PORTS AGE +http-ingress * 35.186.195.33 80 1h + +$ gcloud compute forwarding-rules list +NAME REGION IP_ADDRESS IP_PROTOCOL TARGET +k8s-fw-default-http-ingress--32658fa96c080068 35.186.195.33 TCP k8s-tp-default-http-ingress--32658fa96c080068 +``` + +Note that because this is an ephemeral IP, it won't show up in the output of +`gcloud compute addresses list`. + +If you either directly create an Ingress with a TLS section, or modify a HTTP +Ingress to have a TLS section, it gets a static IP. + +```console +$ kubectl patch ing http-ingress -p '{"spec":{"tls":[{"secretName":"tls-secret"}]}}' +"http-ingress" patched + +$ kubectl get ing +NAME HOSTS ADDRESS PORTS AGE +http-ingress * 35.186.195.33 80, 443 1h + +$ gcloud compute addresses list +NAME REGION ADDRESS STATUS +k8s-fw-default-http-ingress--32658fa96c080068 35.186.195.33 IN_USE +``` + diff --git a/examples/static-ip/gce/gce-http-ingress.yaml b/examples/static-ip/gce/gce-http-ingress.yaml new file mode 100644 index 00000000000..ca0e34ca578 --- /dev/null +++ b/examples/static-ip/gce/gce-http-ingress.yaml @@ -0,0 +1,12 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: http-ingress + annotations: + kubernetes.io/ingress.class: "gce" +spec: + backend: + # This assumes http-svc exists and routes to healthy endpoints. + serviceName: http-svc + servicePort: 80 + diff --git a/examples/static-ip/gce/gce-static-ip-ingress.yaml b/examples/static-ip/gce/gce-static-ip-ingress.yaml new file mode 100644 index 00000000000..7742b87055d --- /dev/null +++ b/examples/static-ip/gce/gce-static-ip-ingress.yaml @@ -0,0 +1,19 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: static-ip + # Assumes a global static ip with the same name exists. + # You can acquire a static IP by running + # gcloud compute addresses create test-ip --global + annotations: + kubernetes.io/ingress.global-static-ip-name: "test-ip" + kubernetes.io/ingress.class: "gce" +spec: + tls: + # This assumes tls-secret exists. + - secretName: tls-secret + backend: + # This assumes http-svc exists and routes to healthy endpoints. + serviceName: http-svc + servicePort: 80 + diff --git a/examples/static-ip/nginx/README.md b/examples/static-ip/nginx/README.md new file mode 100644 index 00000000000..e13dfb9157b --- /dev/null +++ b/examples/static-ip/nginx/README.md @@ -0,0 +1,118 @@ +# Static IPs + + +This example demonstrates how to assign a static-ip to an Ingress on through +the Nginx controller. + +## Prerequisites + +You need a [TLS cert](/examples/PREREQUISITES.md#tls-certificates) and a [test HTTP service](/examples/PREREQUISITES.md#test-http-service) for this example. +You will also need to make sure you Ingress targets exactly one Ingress +controller by specifying the [ingress.class annotation](/examples/PREREQUISITES.md#ingress-class). + +## Acquiring an IP + +Since instances of the nginx controller actually run on nodes in your cluster, +by default nginx Ingresses will only get static IPs if your cloudprovider +supports static IP assignments to nodes. On GKE/GCE for example, even though +nodes get static IPs, the IPs are not retained across upgrade. + +To acquire a static IP for the nginx ingress controller, simply put it +behind a Service of `Type=LoadBalancer`. + +First, create a loadbalancer Service and wait for it to acquire an IP + +```console +$ kubectl create -f static-ip-svc.yaml +service "nginx-ingress-lb" created + +$ kubectl get svc nginx-ingress-lb +NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE +nginx-ingress-lb 10.0.138.113 104.154.109.191 80:31457/TCP,443:32240/TCP 15m +``` + +then, update the ingress controller so it adopts the static IP of the Service +by passing the `--publish-service` flag (the example yaml used in the next step +already has it set to "nginx-ingress-lb"). + +```console +$ kubectl create -f nginx-ingress-controller.yaml +deployment "nginx-ingress-controller" created +``` + +## Assigning the IP to an Ingress + +From here on every Ingress created with the `ingress.class` annotation set to +`nginx` will get the IP allocated in the previous step + +```console +$ kubectl create -f nginx-ingress.yaml +ingress "nginx-ingress" created + +$ kubectl get ing nginx-ingress +NAME HOSTS ADDRESS PORTS AGE +nginx-ingress * 104.154.109.191 80, 443 13m + +$ curl 104.154.109.191 -kL +CLIENT VALUES: +client_address=10.180.1.25 +command=GET +real path=/ +query=nil +request_version=1.1 +request_uri=http://104.154.109.191:8080/ +... +``` + +## Retaining the IP + +You can test retention by deleting the Ingress + +```console +$ kubectl delete ing nginx-ingress +ingress "nginx-ingress" deleted + +$ kubectl create -f nginx-ingress.yaml +ingress "nginx-ingress" created + +$ kubectl get ing nginx-ingress +NAME HOSTS ADDRESS PORTS AGE +nginx-ingress * 104.154.109.191 80, 443 13m +``` + +Note that unlike the GCE Ingress, the same loadbalancer IP is shared amongst all +Ingresses, because all requests are proxied through the same set of nginx +controllers. + +## Promote ephemeral to static IP + +To promote the allocated IP to static, you can update the Service manifest + +```console +$ kubectl patch svc nginx-ingress-lb -p '{"spec": {"loadBalancerIP": "104.154.109.191"}}' +"nginx-ingress-lb" patched +``` + +and promote the IP to static (promotion works differently for cloudproviders, +provided example is for GKE/GCE) +` +```console +$ gcloud compute addresses create nginx-ingress-lb --addresses 104.154.109.191 --region us-central1 +Created [https://www.googleapis.com/compute/v1/projects/kubernetesdev/regions/us-central1/addresses/nginx-ingress-lb]. +--- +address: 104.154.109.191 +creationTimestamp: '2017-01-31T16:34:50.089-08:00' +description: '' +id: '5208037144487826373' +kind: compute#address +name: nginx-ingress-lb +region: us-central1 +selfLink: https://www.googleapis.com/compute/v1/projects/kubernetesdev/regions/us-central1/addresses/nginx-ingress-lb +status: IN_USE +users: +- us-central1/forwardingRules/a09f6913ae80e11e6a8c542010af0000 +``` + +Now even if the Service is deleted, the IP will persist, so you can recreate the +Service with `spec.loadBalancerIP` set to `104.154.109.191`. + diff --git a/examples/static-ip/nginx/nginx-ingress-controller.yaml b/examples/static-ip/nginx/nginx-ingress-controller.yaml new file mode 100644 index 00000000000..a7a78971e97 --- /dev/null +++ b/examples/static-ip/nginx/nginx-ingress-controller.yaml @@ -0,0 +1,52 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: nginx-ingress-controller + labels: + k8s-app: nginx-ingress-controller +spec: + replicas: 1 + template: + metadata: + labels: + k8s-app: nginx-ingress-controller + spec: + # hostNetwork makes it possible to use ipv6 and to preserve the source IP correctly regardless of docker configuration + # however, it is not a hard dependency of the nginx-ingress-controller itself and it may cause issues if port 10254 already is taken on the host + # that said, since hostPort is broken on CNI (https://github.com/kubernetes/kubernetes/issues/31307) we have to use hostNetwork where CNI is used + # like with kubeadm + # hostNetwork: true + terminationGracePeriodSeconds: 60 + containers: + - image: gcr.io/google_containers/nginx-ingress-controller:0.9.0-beta.1 + name: nginx-ingress-controller + readinessProbe: + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + livenessProbe: + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 10 + timeoutSeconds: 1 + ports: + - containerPort: 80 + hostPort: 80 + - containerPort: 443 + hostPort: 443 + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + args: + - /nginx-ingress-controller + - --default-backend-service=$(POD_NAMESPACE)/default-http-backend + - --publish-service=$(POD_NAMESPACE)/nginx-ingress-lb diff --git a/examples/static-ip/nginx/nginx-ingress.yaml b/examples/static-ip/nginx/nginx-ingress.yaml new file mode 100644 index 00000000000..6cdd81fc891 --- /dev/null +++ b/examples/static-ip/nginx/nginx-ingress.yaml @@ -0,0 +1,17 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: nginx-ingress + annotations: + kubernetes.io/ingress.class: "nginx" +spec: + tls: + # This assumes tls-secret exists. + - secretName: tls-secret + rules: + - http: + paths: + - backend: + # This assumes http-svc exists and routes to healthy endpoints. + serviceName: http-svc + servicePort: 80 diff --git a/examples/static-ip/nginx/static-ip-svc.yaml b/examples/static-ip/nginx/static-ip-svc.yaml new file mode 100644 index 00000000000..57d02d1aca2 --- /dev/null +++ b/examples/static-ip/nginx/static-ip-svc.yaml @@ -0,0 +1,23 @@ +# This is the backend service +apiVersion: v1 +kind: Service +metadata: + name: nginx-ingress-lb + annotations: + service.beta.kubernetes.io/external-traffic: OnlyLocal + labels: + app: nginx-ingress-lb +spec: + type: LoadBalancer + loadBalancerIP: 104.154.109.191 + ports: + - port: 80 + name: http + targetPort: 80 + - port: 443 + name: https + targetPort: 443 + selector: + # Selects nginx-ingress-controller pods + k8s-app: nginx-ingress-controller + diff --git a/examples/tls-termination/gce.md b/examples/tls-termination/gce/README.md similarity index 100% rename from examples/tls-termination/gce.md rename to examples/tls-termination/gce/README.md diff --git a/examples/tls-termination/gce/gce-tls-ingress.yaml b/examples/tls-termination/gce/gce-tls-ingress.yaml new file mode 100644 index 00000000000..705a17d36ec --- /dev/null +++ b/examples/tls-termination/gce/gce-tls-ingress.yaml @@ -0,0 +1,15 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: test + annotations: + kubernetes.io/ingress.class: "gce" +spec: + tls: + # This assumes tls-secret exists. + - secretName: tls-secret + backend: + # This assumes http-svc exists and routes to healthy endpoints. + serviceName: http-svc + servicePort: 80 + diff --git a/examples/tls-termination/nginx.md b/examples/tls-termination/nginx/README.md similarity index 100% rename from examples/tls-termination/nginx.md rename to examples/tls-termination/nginx/README.md diff --git a/examples/tls-termination/nginx-tls-ingress.yaml b/examples/tls-termination/nginx/nginx-tls-ingress.yaml similarity index 100% rename from examples/tls-termination/nginx-tls-ingress.yaml rename to examples/tls-termination/nginx/nginx-tls-ingress.yaml