kubernetes · k8s-ci-robot · May 2, 2019 · Feb 21, 2019
diff --git a/cmd/nginx/flags.go b/cmd/nginx/flags.go
@@ -159,6 +159,14 @@ Feature backed by OpenResty Lua libraries. Requires that OCSP stapling is not en
 
 		disableCatchAll = flags.Bool("disable-catch-all", false,
 			`Disable support for catch-all Ingresses`)
+
+		validationWebhook = flags.String("validating-webhook", "",
+			`The address to start an admission controller on to validate incoming ingresses.
+Takes the form "<host>:port". If not provided, no admission controller is started.`)
+		validationWebhookCert = flags.String("validating-webhook-certificate", "",
+			`The path of the validating webhook certificate PEM.`)
+		validationWebhookKey = flags.String("validating-webhook-key", "",
+			`The path of the validating webhook key PEM.`)
 	)
 
 	flags.MarkDeprecated("status-port", `The status port is a unix socket now.`)
@@ -255,7 +263,10 @@ Feature backed by OpenResty Lua libraries. Requires that OCSP stapling is not en
 			HTTPS:    *httpsPort,
 			SSLProxy: *sslProxyPort,
 		},
-		DisableCatchAll: *disableCatchAll,
+		DisableCatchAll:           *disableCatchAll,
+		ValidationWebhook:         *validationWebhook,
+		ValidationWebhookCertPath: *validationWebhookCert,
+		ValidationWebhookKeyPath:  *validationWebhookKey,
 	}
 
 	return false, config, nil

diff --git a/deploy/validating-webhook.yaml.tpl b/deploy/validating-webhook.yaml.tpl
@@ -0,0 +1,25 @@
+---
+apiVersion: admissionregistration.k8s.io/v1beta1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: check-ingress
+webhooks:
+- name: validate.nginx.ingress.kubernetes.io
+  rules:
+  - apiGroups:
+    - extensions
+    apiVersions:
+    - v1beta1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - ingresses
+  failurePolicy: Fail
+  clientConfig:
+    service:
+      namespace: ingress-nginx
+      name: nginx-ingress-webhook
+      path: /extensions/v1beta1/ingresses
+    caBundle: <certificate.pem | base64>
+---
diff --git a/deploy/with-validating-webhook.yaml.tpl b/deploy/with-validating-webhook.yaml.tpl
@@ -0,0 +1,115 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: ingress-validation-webhook
+  namespace: ingress-nginx
+spec:
+  ports:
+  - name: admission
+    port: 443
+    protocol: TCP
+    targetPort: 8080
+  selector:
+    app.kubernetes.io/name: ingress-nginx
+---
+apiVersion: v1
+data:
+  key.pem: <key.pem | base64>
+  certificate.pem: <certificate.pem | base64>
+kind: Secret
+metadata:
+  name: nginx-ingress-webhook-certificate
+  namespace: ingress-nginx
+type: Opaque
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: nginx-ingress-controller
+  namespace: ingress-nginx
+  labels:
+    app.kubernetes.io/name: ingress-nginx
+    app.kubernetes.io/part-of: ingress-nginx
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: ingress-nginx
+      app.kubernetes.io/part-of: ingress-nginx
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/name: ingress-nginx
+        app.kubernetes.io/part-of: ingress-nginx
+      annotations:
+        prometheus.io/port: "10254"
+        prometheus.io/scrape: "true"
+    spec:
+      serviceAccountName: nginx-ingress-serviceaccount
+      containers:
+        - name: nginx-ingress-controller
+          image: containers.schibsted.io/thibault-jamet/ingress-nginx:0.23.0-schibsted
+          args:
+            - /nginx-ingress-controller
+            - --configmap=$(POD_NAMESPACE)/nginx-configuration
+            - --tcp-services-configmap=$(POD_NAMESPACE)/tcp-services
+            - --udp-services-configmap=$(POD_NAMESPACE)/udp-services
+            - --publish-service=$(POD_NAMESPACE)/ingress-nginx
+            - --annotations-prefix=nginx.ingress.kubernetes.io
+            - --validating-webhook=:8080
+            - --validating-webhook-certificate=/usr/local/certificates/certificate.pem
+            - --validating-webhook-key=/usr/local/certificates/key.pem
+          volumeMounts:
+          - name: webhook-cert
+            mountPath: "/usr/local/certificates/"
+            readOnly: true
+          securityContext:
+            allowPrivilegeEscalation: true
+            capabilities:
+              drop:
+                - ALL
+              add:
+                - NET_BIND_SERVICE
+            # www-data -> 33
+            runAsUser: 33
+          env:
+            - name: POD_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.name
+            - name: POD_NAMESPACE
+              valueFrom:
+                fieldRef:
+                  fieldPath: metadata.namespace
+          ports:
+            - name: http
+              containerPort: 80
+            - name: https
+              containerPort: 443
+            - name: webhook
+              containerPort: 8080
+          livenessProbe:
+            failureThreshold: 3
+            httpGet:
+              path: /healthz
+              port: 10254
+              scheme: HTTP
+            initialDelaySeconds: 10
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 10
+          readinessProbe:
+            failureThreshold: 3
+            httpGet:
+              path: /healthz
+              port: 10254
+              scheme: HTTP
+            periodSeconds: 10
+            successThreshold: 1
+            timeoutSeconds: 10
+      volumes:
+      - name: webhook-cert
+        secret:
+          secretName: nginx-ingress-webhook-certificate
+---
diff --git a/docs/deploy/validating-webhook.md b/docs/deploy/validating-webhook.md
@@ -0,0 +1,168 @@
+# Validating webhook (admission controller)
+
+## Overview
+
+Nginx ingress controller offers the option to validate ingresses before they enter the cluster, ensuring controller will generate a valid configuration.
+
+This controller is called, when [ValidatingAdmissionWebhook][1] is enabled, by the Kubernetes API server each time a new ingress is to enter the cluster, and rejects objects for which the generated nginx configuration fails to be validated.
+
+This feature requires some further configuration of the cluster, hence it is an optional feature, this section explains how to enable it for your cluster.
+
+## Configure the webhook
+
+### Generate the webhook certificate
+
+
+#### Self signed certificate
+
+Validating webhook must be served using TLS, you need to generate a certificate. Note that kube API server is checking the hostname of the certificate, the common name of your certificate will need to match the service name.
+
+!!! example
+    To run the validating webhook with a service named `ingress-validation-webhook` in the namespace `ingress-nginx`, run
+
+    ```bash
+    openssl req -x509 -newkey rsa:2048 -keyout certificate.pem -out key.pem -days 365 -nodes -subj "/CN=ingress-validation-webhook.ingress-nginx.svc"
+    ```
+
+##### Using Kubernetes CA
+
+Kubernetes also provides primitives to sign a certificate request. Here is an example on how to use it
+
+!!! example
+    ```
+    #!/bin/bash
+
+    SERVICE_NAME=ingress-nginx
+    NAMESPACE=ingress-nginx
+
+    TEMP_DIRECTORY=$(mktemp -d)
+    echo "creating certs in directory ${TEMP_DIRECTORY}"
+
+    cat <<EOF >> ${TEMP_DIRECTORY}/csr.conf
+    [req]
+    req_extensions = v3_req
+    distinguished_name = req_distinguished_name
+    [req_distinguished_name]
+    [ v3_req ]
+    basicConstraints = CA:FALSE
+    keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+    extendedKeyUsage = serverAuth
+    subjectAltName = @alt_names
+    [alt_names]
+    DNS.1 = ${SERVICE_NAME}
+    DNS.2 = ${SERVICE_NAME}.${NAMESPACE}
+    DNS.3 = ${SERVICE_NAME}.${NAMESPACE}.svc
+    EOF
+
+    openssl genrsa -out ${TEMP_DIRECTORY}/server-key.pem 2048
+    openssl req -new -key ${TEMP_DIRECTORY}/server-key.pem \
+        -subj "/CN=${SERVICE_NAME}.${NAMESPACE}.svc" \
+        -out ${TEMP_DIRECTORY}/server.csr \
+        -config ${TEMP_DIRECTORY}/csr.conf
+
+    cat <<EOF | kubectl create -f -
+    apiVersion: certificates.k8s.io/v1beta1
+    kind: CertificateSigningRequest
+    metadata:
+      name: ${SERVICE_NAME}.${NAMESPACE}.svc
+    spec:
+      request: $(cat ${TEMP_DIRECTORY}/server.csr | base64 | tr -d '\n')
+      usages:
+      - digital signature
+      - key encipherment
+      - server auth
+    EOF
+
+    kubectl certificate approve ${SERVICE_NAME}.${NAMESPACE}.svc
+
+    for x in $(seq 10); do
+        SERVER_CERT=$(kubectl get csr ${SERVICE_NAME}.${NAMESPACE}.svc -o jsonpath='{.status.certificate}')
+        if [[ ${SERVER_CERT} != '' ]]; then
+            break
+        fi
+        sleep 1
+    done
+    if [[ ${SERVER_CERT} == '' ]]; then
+        echo "ERROR: After approving csr ${SERVICE_NAME}.${NAMESPACE}.svc, the signed certificate did not appear on the resource. Giving up after 10 attempts." >&2
+        exit 1
+    fi
+    echo ${SERVER_CERT} | openssl base64 -d -A -out ${TEMP_DIRECTORY}/server-cert.pem
+
+    kubectl create secret generic ingress-nginx.svc \
+        --from-file=key.pem=${TEMP_DIRECTORY}/server-key.pem \
+        --from-file=cert.pem=${TEMP_DIRECTORY}/server-cert.pem \
+        -n ${NAMESPACE}
+    ```
+
+#### Using helm
+
+To generate the certificate using helm, you can use the following snippet
+
+!!! example
+    ```
+    {{- $cn := printf "%s.%s.svc" ( include "nginx-ingress.validatingWebhook.fullname" . ) .Release.Namespace }}
+    {{- $ca := genCA (printf "%s-ca" ( include "nginx-ingress.validatingWebhook.fullname" . )) .Values.validatingWebhook.certificateValidity -}}
+    {{- $cert := genSignedCert $cn nil nil .Values.validatingWebhook.certificateValidity $ca -}}
+    ```
+
+### Ingress controller flags
+
+To enable the feature in the ingress controller, you _need_ to provide 3 flags to the command line.
+
+|flag|description|example usage|
+|-|-|-|
+|`--validating-webhook`|The address to start an admission controller on|`:8080`|
+|`--validating-webhook-certificate`|The certificate the webhook is using for its TLS handling|`/usr/local/certificates/validating-webhook.pem`|
+|`--validating-webhook-key`|The key the webhook is using for its TLS handling|`/usr/local/certificates/validating-webhook-key.pem`|
+
+### kube API server flags
+
+Validating webhook feature requires specific setup on the kube API server side. Depending on your kubernetes version, the flag can, or not, be enabled by default.
+To check that your kube API server runs with the required flags, please refer to the [kubernetes][1] documentation.
+
+### Additional kubernetes objects
+
+Once both the ingress controller and the kube API server are configured to serve the webhook, add the you can configure the webhook with the following objects:
+
+```yaml
+apiVersion: v1
+kind: Service
+metadata:
+  name: ingress-validation-webhook
+  namespace: ingress-nginx
+spec:
+  ports:
+  - name: admission
+    port: 443
+    protocol: TCP
+    targetPort: 8080
+  selector:
+    app: nginx-ingress
+    component: controller
+---
+apiVersion: admissionregistration.k8s.io/v1beta1
+kind: ValidatingWebhookConfiguration
+metadata:
+  name: check-ingress
+webhooks:
+- name: validate.nginx.ingress.kubernetes.io
+  rules:
+  - apiGroups:
+    - extensions
+    apiVersions:
+    - v1beta1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - ingresses
+  failurePolicy: Fail
+  clientConfig:
+    service:
+      namespace: ingress-nginx
+      name: ingress-validation-webhook
+      path: /extensions/v1beta1/ingress
+    caBundle: <pem encoded ca cert that signs the server cert used by the webhook>
+```
+
+[1]: https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#validatingadmissionwebhook
diff --git a/docs/how-it-works.md b/docs/how-it-works.md
@@ -56,6 +56,13 @@ On every endpoint change the controller fetches endpoints from all the services
 
 In a relatively big clusters with frequently deploying apps this feature saves significant number of Nginx reloads which can otherwise affect response latency, load balancing quality (after every reload Nginx resets the state of load balancing) and so on.
 
+### Avoiding outage from wrong configuration
+
+Because the ingress controller works using the [synchronization loop pattern](https://coreos.com/kubernetes/docs/latest/replication-controller.html#the-reconciliation-loop-in-detail), it is applying the configuration for all matching objects. In case some Ingress objects have a broken configuration, for example a syntax error in the `nginx.ingress.kubernetes.io/configuration-snippet` annotation, the generated configuration becomes invalid, does not reload and hence no more ingresses will be taken into account.
+
+To prevent this situation to happen, the nginx ingress controller exposes optionnally a [validating admission webhook server][8] to ensure the validity of incoming ingress objects.
+This webhook appends the incoming ingress objects to the list of ingresses, generates the configuration and calls nginx to ensure the configuration has no syntax errors.
+
 [0]: https://github.com/openresty/lua-nginx-module/pull/1259
 [1]: https://coreos.com/kubernetes/docs/latest/replication-controller.html#the-reconciliation-loop-in-detail
 [2]: https://godoc.org/k8s.io/client-go/informers#NewFilteredSharedInformerFactory
@@ -64,3 +71,4 @@ In a relatively big clusters with frequently deploying apps this feature saves s
 [5]: https://golang.org/pkg/sync/#Mutex
 [6]: https://github.com/kubernetes/ingress-nginx/blob/master/rootfs/etc/nginx/template/nginx.tmpl
 [7]: http://nginx.org/en/docs/beginners_guide.html#control
+[8]: https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#validatingadmissionwebhook
diff --git a/docs/user-guide/cli-arguments.md b/docs/user-guide/cli-arguments.md
@@ -44,3 +44,7 @@ They are set in the container spec of the `nginx-ingress-controller` Deployment
 | `--version`                       | Show release information about the NGINX Ingress controller and exit. |
 | `--vmodule moduleSpec`            | comma-separated list of pattern=N settings for file-filtered logging |
 | `--watch-namespace string`        | Namespace the controller watches for updates to Kubernetes objects. This includes Ingresses, Services and all configuration resources. All namespaces are watched if this parameter is left empty. |
+| `--disable-catch-all`             | Disable support for catch-all Ingresses. |
+|`--validating-webhook`|The address to start an admission controller on|
+|`--validating-webhook-certificate`|The certificate the webhook is using for its TLS handling|
+|`--validating-webhook-key`|The key the webhook is using for its TLS handling|
diff --git a/go.mod b/go.mod
@@ -19,8 +19,10 @@ require (
 	github.com/evanphx/json-patch v4.1.0+incompatible // indirect
 	github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa // indirect
 	github.com/go-openapi/spec v0.19.0 // indirect
+	github.com/gofortune/gofortune v0.0.1-snapshot // indirect
 	github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect
 	github.com/google/gofuzz v1.0.0 // indirect
+	github.com/google/uuid v1.0.0
 	github.com/googleapis/gnostic v0.2.0 // indirect
 	github.com/gophercloud/gophercloud v0.0.0-20190410012400-2c55d17f707c // indirect
 	github.com/imdario/mergo v0.3.7
@@ -54,7 +56,9 @@ require (
 	github.com/spf13/cobra v0.0.3
 	github.com/spf13/pflag v1.0.3
 	github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926
+	github.com/vromero/gofortune v0.0.1-snapshot
 	github.com/zakjan/cert-chain-resolver v0.0.0-20180703112424-6076e1ded272
+	google.golang.org/grpc v1.19.1
 	gopkg.in/fsnotify/fsnotify.v1 v1.4.7
 	gopkg.in/go-playground/assert.v1 v1.2.1 // indirect
 	gopkg.in/go-playground/pool.v3 v3.1.1