Merge pull request #246 from aledbf/set-headers

Add support for custom proxy headers using a ConfigMap

controllers/nginx/pkg/cmd/controller/nginx.go

@@ -102,6 +102,8 @@ type NGINXController struct {
 
 	configmap *api.ConfigMap
 
+	storeLister ingress.StoreLister
+
 	binary string
 }
 
@@ -282,11 +284,16 @@ Error: %v
 	return nil
 }
 
-// SetConfig ...
+// SetConfig sets the configured configmap
 func (n *NGINXController) SetConfig(cmap *api.ConfigMap) {
 	n.configmap = cmap
 }
 
+// SetListers sets the configured store listers in the generic ingress controller
+func (n *NGINXController) SetListers(lister ingress.StoreLister) {
+	n.storeLister = lister
+}
+
 // OnUpdate is called by syncQueue in https://github.com/aledbf/ingress-controller/blob/master/pkg/ingress/controller/controller.go#L82
 // periodically to keep the configuration in sync.
 //
@@ -330,7 +337,20 @@ func (n *NGINXController) OnUpdate(ingressCfg ingress.Configuration) ([]byte, er
 	// and we leave some room to avoid consuming all the FDs available
 	maxOpenFiles := (sysctlFSFileMax() / cfg.WorkerProcesses) - 1024
 
+	setHeaders := map[string]string{}
+	if cfg.ProxySetHeaders != "" {
+		cmap, exists, err := n.storeLister.ConfigMap.GetByKey(cfg.ProxySetHeaders)
+		if err != nil {
+			glog.Warningf("unexpected error reading configmap %v: %v", cfg.ProxySetHeaders, err)
+		}
+
+		if exists {
+			setHeaders = cmap.(*api.ConfigMap).Data
+		}
+	}
+
 	return n.t.Write(config.TemplateConfig{
+		ProxySetHeaders:     setHeaders,
 		MaxOpenFiles:        maxOpenFiles,
 		BacklogSize:         sysctlSomaxconn(),
 		Backends:            ingressCfg.Backends,

controllers/nginx/pkg/config/config.go

@@ -156,6 +156,9 @@ type Configuration struct {
 	// of your external load balancer
 	ProxyRealIPCIDR string `json:"proxy-real-ip-cidr,omitempty"`
 
+	// Sets the name of the configmap that contains the headers to pass to the backend
+	ProxySetHeaders string `json:"proxy-set-headers,omitempty"`
+
 	// Maximum size of the server names hash tables used in server names, map directive’s values,
 	// MIME types, names of request header strings, etcd.
 	// http://nginx.org/en/docs/hash.html
@@ -288,6 +291,7 @@ func NewDefault() Configuration {
 
 // TemplateConfig contains the nginx configuration to render the file nginx.conf
 type TemplateConfig struct {
+	ProxySetHeaders     map[string]string
 	MaxOpenFiles        int
 	BacklogSize         int
 	Backends            []*ingress.Backend

controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl

@@ -1,4 +1,4 @@
-{{ $cfg := .Cfg }}{{ $healthzURI := .HealthzURI }}{{ $backends := .Backends }}
+{{ $cfg := .Cfg }}{{ $healthzURI := .HealthzURI }}{{ $backends := .Backends }}{{ $proxyHeaders := .ProxySetHeaders }}
 daemon off;
 
 worker_processes {{ $cfg.WorkerProcesses }};
@@ -311,6 +311,11 @@ http {
             # https://www.nginx.com/blog/mitigating-the-httpoxy-vulnerability-with-nginx/
             proxy_set_header Proxy                  "";
 
+            # Custom headers
+            {{ range $k, $v := $proxyHeaders }}
+            proxy_set_header {{ $k }}                    "{{ $v }}";
+            {{ end }}
+
             proxy_connect_timeout                   {{ $location.Proxy.ConnectTimeout }}s;
             proxy_send_timeout                      {{ $location.Proxy.SendTimeout }}s;
             proxy_read_timeout                      {{ $location.Proxy.ReadTimeout }}s;

core/pkg/ingress/controller/controller.go

@@ -304,6 +304,14 @@ func newIngressController(config *Configuration) *GenericController {
 
 	ic.annotations = newAnnotationExtractor(ic)
 
+	ic.cfg.Backend.SetListers(ingress.StoreLister{
+		Ingress:   ic.ingLister,
+		Service:   ic.svcLister,
+		Endpoint:  ic.endpLister,
+		Secret:    ic.secrLister,
+		ConfigMap: ic.mapLister,
+	})
+
 	return &ic
 }
 

core/pkg/ingress/types.go

@@ -20,8 +20,10 @@ import (
 	"github.com/spf13/pflag"
 
 	"k8s.io/kubernetes/pkg/api"
+	"k8s.io/kubernetes/pkg/client/cache"
 	"k8s.io/kubernetes/pkg/healthz"
 
+	cache_store "k8s.io/ingress/core/pkg/cache"
 	"k8s.io/ingress/core/pkg/ingress/annotations/auth"
 	"k8s.io/ingress/core/pkg/ingress/annotations/authreq"
 	"k8s.io/ingress/core/pkg/ingress/annotations/ipwhitelist"
@@ -83,6 +85,9 @@ type Controller interface {
 	OnUpdate(Configuration) ([]byte, error)
 	// ConfigMap content of --configmap
 	SetConfig(*api.ConfigMap)
+	// SetListers allows the access of store listers present in the generic controller
+	// This avoid the use of the kubernetes client.
+	SetListers(StoreLister)
 	// BackendDefaults returns the minimum settings required to configure the
 	// communication to endpoints
 	BackendDefaults() defaults.Backend
@@ -92,6 +97,16 @@ type Controller interface {
 	OverrideFlags(*pflag.FlagSet)
 }
 
+// StoreLister returns the configured stores for ingresses, services,
+// endpoints, secrets and configmaps.
+type StoreLister struct {
+	Ingress   cache_store.StoreToIngressLister
+	Service   cache.StoreToServiceLister
+	Endpoint  cache.StoreToEndpointsLister
+	Secret    cache_store.StoreToSecretsLister
+	ConfigMap cache_store.StoreToConfigmapLister
+}
+
 // BackendInfo returns information about the backend.
 // This fields contains information that helps to track issues or to
 // map the running ingress controller to source code

examples/README.md

@@ -75,4 +75,8 @@ Name | Description | Platform   | Complexity Level
 -----| ----------- | ---------- | ----------------
 Dummy  | A simple dummy controller that logs updates | * | Advanced
 
+## Custommization
 
+Name | Description | Platform   | Complexity Level
+-----| ----------- | ---------- | ----------------
+custom-headers  | set custom headers before send traffic to backends  | nginx | Advanced

examples/customization/custom-headers/nginx/README.md

@@ -0,0 +1,76 @@
+# Deploying the Nginx Ingress controller
+
+This example aims to demonstrate the deployment of an nginx ingress controller and
+use a ConfigMap to configure a custom list of headers to be passed to the upstream
+server
+
+## Default Backend
+
+The default backend is a Service capable of handling all url paths and hosts the
+nginx controller doesn't understand. This most basic implementation just returns
+a 404 page:
+
+```console
+$ kubectl apply -f default-backend.yaml
+deployment "default-http-backend" created
+service "default-http-backend" created
+
+$ kubectl -n kube-system get po
+NAME                                    READY     STATUS    RESTARTS   AGE
+default-http-backend-2657704409-qgwdd   1/1       Running   0          28s
+```
+
+## Custom configuration
+
+```console
+$ cat nginx-load-balancer-conf.yaml
+apiVersion: v1
+data:
+  proxy-set-headers: "default/custom-headers"
+kind: ConfigMap
+metadata:
+  name: nginx-load-balancer-conf
+```
+
+```console
+$ kubectl create -f nginx-load-balancer-conf.yaml
+```
+
+## Custom headers
+
+```console
+$ cat custom-headers.yaml
+apiVersion: v1
+data:
+  X-Different-Name: "true"
+  X-Request-Start: t=${msec}
+  X-Using-Nginx-Controller: "true"
+kind: ConfigMap
+metadata:
+  name: proxy-headers
+  namespace: default
+
+```
+
+```console
+$ kubectl create -f custom-headers.yaml
+```
+
+## Controller
+
+You can deploy the controller as follows:
+
+```console
+$ kubectl apply -f nginx-ingress-controller.yaml
+deployment "nginx-ingress-controller" created
+
+$ kubectl -n kube-system get po
+NAME                                       READY     STATUS    RESTARTS   AGE
+default-http-backend-2657704409-qgwdd      1/1       Running   0          2m
+nginx-ingress-controller-873061567-4n3k2   1/1       Running   0          42s
+```
+
+## Test
+
+Check the contents of the configmap is present in the nginx.conf file using:
+`kubectl exec nginx-ingress-controller-873061567-4n3k2 -n kube-system cat /etc/nginx/nginx.conf`

examples/customization/custom-headers/nginx/custom-headers.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+data:
+  X-Different-Name: "true"
+  X-Request-Start: t=${msec}
+  X-Using-Nginx-Controller: "true"
+kind: ConfigMap
+metadata:
+  name: proxy-headers
+  namespace: kube-system

examples/customization/custom-headers/nginx/default-backend.yaml

@@ -0,0 +1,51 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: default-http-backend
+  labels:
+    k8s-app: default-http-backend
+  namespace: kube-system
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        k8s-app: default-http-backend
+    spec:
+      terminationGracePeriodSeconds: 60
+      containers:
+      - name: default-http-backend
+        # Any image is permissable as long as:
+        # 1. It serves a 404 page at /
+        # 2. It serves 200 on a /healthz endpoint
+        image: gcr.io/google_containers/defaultbackend:1.0
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8080
+            scheme: HTTP
+          initialDelaySeconds: 30
+          timeoutSeconds: 5
+        ports:
+        - containerPort: 8080
+        resources:
+          limits:
+            cpu: 10m
+            memory: 20Mi
+          requests:
+            cpu: 10m
+            memory: 20Mi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: default-http-backend
+  namespace: kube-system
+  labels:
+    k8s-app: default-http-backend
+spec:
+  ports:
+  - port: 80
+    targetPort: 8080
+  selector:
+    k8s-app: default-http-backend

examples/customization/custom-headers/nginx/nginx-ingress-controller.yaml

@@ -0,0 +1,53 @@
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: nginx-ingress-controller
+  labels:
+    k8s-app: nginx-ingress-controller
+  namespace: kube-system
+spec:
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        k8s-app: nginx-ingress-controller
+    spec:
+      # hostNetwork makes it possible to use ipv6 and to preserve the source IP correctly regardless of docker configuration
+      # however, it is not a hard dependency of the nginx-ingress-controller itself and it may cause issues if port 10254 already is taken on the host
+      # that said, since hostPort is broken on CNI (https://github.com/kubernetes/kubernetes/issues/31307) we have to use hostNetwork where CNI is used
+      # like with kubeadm
+      # hostNetwork: true
+      terminationGracePeriodSeconds: 60
+      containers:
+      - image: gcr.io/google_containers/nginx-ingress-controller:0.9.0-beta.1
+        name: nginx-ingress-controller
+        readinessProbe:
+          httpGet:
+            path: /healthz
+            port: 10254
+            scheme: HTTP
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 10254
+            scheme: HTTP
+          initialDelaySeconds: 10
+          timeoutSeconds: 1
+        ports:
+        - containerPort: 80
+          hostPort: 80
+        - containerPort: 443
+          hostPort: 443
+        env:
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+        args:
+        - /nginx-ingress-controller
+        - --default-backend-service=$(POD_NAMESPACE)/default-http-backend
+        - --configmap=$(POD_NAMESPACE)/nginx-load-balancer-conf

examples/customization/custom-headers/nginx/nginx-load-balancer-conf.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+data:
+  proxy-set-headers: "kube-system/custom-headers"
+kind: ConfigMap
+metadata:
+  name: nginx-load-balancer-conf
+  namespace: kube-system