From 84324af14075b6bed206755149deacd9e9136cd9 Mon Sep 17 00:00:00 2001 From: Manuel de Brito Fontes Date: Fri, 24 Feb 2017 18:46:39 -0300 Subject: [PATCH] Refactoring of TCP and UDP services --- controllers/nginx/pkg/config/config.go | 4 +- controllers/nginx/pkg/template/template.go | 29 ----------- .../rootfs/etc/nginx/template/nginx.tmpl | 42 +++++++++------ controllers/nginx/test/data/config.json | 52 +------------------ core/pkg/ingress/annotations/proxy/main.go | 11 ++-- core/pkg/ingress/controller/controller.go | 26 ++++++---- core/pkg/ingress/types.go | 23 +++++++- 7 files changed, 71 insertions(+), 116 deletions(-) diff --git a/controllers/nginx/pkg/config/config.go b/controllers/nginx/pkg/config/config.go index c4ac3dfd63..bfe4cc79e2 100644 --- a/controllers/nginx/pkg/config/config.go +++ b/controllers/nginx/pkg/config/config.go @@ -299,8 +299,8 @@ type TemplateConfig struct { Backends []*ingress.Backend PassthroughBackends []*ingress.SSLPassthroughBackend Servers []*ingress.Server - TCPBackends []*ingress.Location - UDPBackends []*ingress.Location + TCPBackends []ingress.L4Service + UDPBackends []ingress.L4Service HealthzURI string CustomErrors bool Cfg Configuration diff --git a/controllers/nginx/pkg/template/template.go b/controllers/nginx/pkg/template/template.go index a4ec4b4e4c..1945bc6cf7 100644 --- a/controllers/nginx/pkg/template/template.go +++ b/controllers/nginx/pkg/template/template.go @@ -134,7 +134,6 @@ var ( "buildSSLPassthroughUpstreams": buildSSLPassthroughUpstreams, "buildResolvers": buildResolvers, "isLocationAllowed": isLocationAllowed, - "buildStreamUpstreams": buildStreamUpstreams, "contains": strings.Contains, "hasPrefix": strings.HasPrefix, @@ -193,34 +192,6 @@ func buildSSLPassthroughUpstreams(b interface{}, sslb interface{}) string { return buf.String() } -func buildStreamUpstreams(proto string, b interface{}, s interface{}) string { - backends := b.([]*ingress.Backend) - streams := s.([]*ingress.Location) - buf := bytes.NewBuffer(make([]byte, 0, 10)) - // multiple services can use the same upstream. - // avoid duplications using a map[name]=true - u := make(map[string]bool) - for _, stream := range streams { - if u[stream.Backend] { - continue - } - u[stream.Backend] = true - fmt.Fprintf(buf, "upstream %v-%v {\n", proto, stream.Backend) - // TODO: find a better way to avoid empty stream upstreams - fmt.Fprintf(buf, "\t\tserver 127.0.0.1:8181 down;\n") - for _, backend := range backends { - if backend.Name == stream.Backend { - for _, server := range backend.Endpoints { - fmt.Fprintf(buf, "\t\tserver %v:%v;\n", server.Address, server.Port) - } - break - } - } - fmt.Fprint(buf, "\t}\n\n") - } - return buf.String() -} - // buildLocation produces the location string, if the ingress has redirects // (specified through the ingress.kubernetes.io/rewrite-to annotation) func buildLocation(input interface{}) string { diff --git a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl index 7de3474060..11440b1957 100644 --- a/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl +++ b/controllers/nginx/rootfs/etc/nginx/template/nginx.tmpl @@ -326,8 +326,8 @@ http { proxy_http_version 1.1; - proxy_cookie_domain {{ $location.Proxy.CookiePath }}; - proxy_cookie_path {{ $location.Proxy.CookieDomain }}; + proxy_cookie_domain {{ $location.Proxy.CookieDomain }}; + proxy_cookie_path {{ $location.Proxy.CookiePath }}; {{/* rewrite only works if the content is not compressed */}} {{ if $location.Redirect.AddBaseURL }} @@ -460,25 +460,33 @@ stream { ssl_preread on; } - {{ buildStreamUpstreams "tcp" $backends .TCPBackends }} - - {{ buildStreamUpstreams "udp" $backends .UDPBackends }} - # TCP services {{ range $i, $tcpServer := .TCPBackends }} - server { - listen {{ $tcpServer.Path }}; - proxy_pass tcp-{{ $tcpServer.Backend }}; - } + upstream {{ $tcpServer.Backend.Namespace }}-{{ $tcpServer.Backend.Name }}-{{ $tcpServer.Backend.Port }} { + {{ range $j, $endpoint := $tcpServer.Endpoints }} + server {{ $endpoint.Address }}:{{ $endpoint.Port }}; + {{ end }} + } + + server { + listen {{ $tcpServer.Port }}; + proxy_pass {{ $tcpServer.Backend.Namespace }}-{{ $tcpServer.Backend.Name }}-{{ $tcpServer.Backend.Port }}; + } {{ end }} - # UDP services - {{ range $i, $udpServer := .UDPBackends }} - server { - listen {{ $udpServer.Path }} udp; - proxy_responses 1; - proxy_pass udp-{{ $udpServer.Backend }}; - } + # UDP services + {{ range $i, $udpServer := .UDPBackends }} + upstream {{ $udpServer.Backend.Namespace }}-{{ $udpServer.Backend.Name }}-{{ $udpServer.Backend.Port }} { + {{ range $j, $endpoint := $udpServer.Endpoints }} + server {{ $endpoint.Address }}:{{ $endpoint.Port }}; + {{ end }} + } + + server { + listen {{ $udpServer.Port }}; + proxy_responses 1; + proxy_pass {{ $udpServer.Backend.Namespace }}-{{ $udpServer.Backend.Name }}-{{ $udpServer.Backend.Port }}; + } {{ end }} } diff --git a/controllers/nginx/test/data/config.json b/controllers/nginx/test/data/config.json index 4aaa2dd11f..087a8fe838 100644 --- a/controllers/nginx/test/data/config.json +++ b/controllers/nginx/test/data/config.json @@ -57134,57 +57134,7 @@ }] }], "sslDHParam": "", - "tcpBackends": [{ - "path": "2222", - "isDefBackend": false, - "backend": "default-echoheaders-2222", - "basicDigestAuth": { - "type": "", - "realm": "", - "file": "", - "secured": false - }, - "externalAuth": { - "url": "", - "method": "", - "sendBody": false - }, - "rateLimit": { - "connections": { - "name": "", - "limit": 0, - "burst": 0, - "sharedSize": 0 - }, - "rps": { - "name": "", - "limit": 0, - "burst": 0, - "sharedSize": 0 - } - }, - "redirect": { - "target": "", - "addBaseUrl": false, - "sslRedirect": false - }, - "whitelist": { - "cidr": null - }, - "proxy": { - "conectTimeout": 0, - "sendTimeout": 0, - "readTimeout": 0, - "bufferSize": "" - }, - "certificateAuth": { - "secret": "", - "certFilename": "", - "keyFilename": "", - "caFilename": "", - "pemSha": "" - } - }], + "tcpBackends": [], "udpBackends": [], "backends": [{ "name": "default-echoheaders-80", diff --git a/core/pkg/ingress/annotations/proxy/main.go b/core/pkg/ingress/annotations/proxy/main.go index 93f83eb63f..ab7d2e07aa 100644 --- a/core/pkg/ingress/annotations/proxy/main.go +++ b/core/pkg/ingress/annotations/proxy/main.go @@ -40,8 +40,8 @@ type Configuration struct { SendTimeout int `json:"sendTimeout"` ReadTimeout int `json:"readTimeout"` BufferSize string `json:"bufferSize"` - CookieDomain string `json:"proxyCookieDomain"` - CookiePath string `json:"proxyCookiePath"` + CookieDomain string `json:"cookieDomain"` + CookiePath string `json:"cookiePath"` } type proxy struct { @@ -83,8 +83,8 @@ func (a proxy) Parse(ing *extensions.Ingress) (interface{}, error) { } cd, err := parser.GetStringAnnotation(cookieDomain, ing) - if err != nil || cp == "" { - cp = defBackend.ProxyCookieDomain + if err != nil || cd == "" { + cd = defBackend.ProxyCookieDomain } bs, err := parser.GetStringAnnotation(bodySize, ing) @@ -92,6 +92,5 @@ func (a proxy) Parse(ing *extensions.Ingress) (interface{}, error) { bs = defBackend.ProxyBodySize } - return &Configuration{bs, ct, st, rt, bufs, - cd, cp}, nil + return &Configuration{bs, ct, st, rt, bufs, cd, cp}, nil } diff --git a/core/pkg/ingress/controller/controller.go b/core/pkg/ingress/controller/controller.go index 5a315dbfdf..58e647f9e2 100644 --- a/core/pkg/ingress/controller/controller.go +++ b/core/pkg/ingress/controller/controller.go @@ -426,30 +426,30 @@ func (ic *GenericController) sync(key interface{}) error { return nil } -func (ic *GenericController) getStreamServices(configmapName string, proto api.Protocol) []*ingress.Location { +func (ic *GenericController) getStreamServices(configmapName string, proto api.Protocol) []ingress.L4Service { glog.V(3).Infof("obtaining information about stream services of type %v located in configmap %v", proto, configmapName) if configmapName == "" { // no configmap configured - return []*ingress.Location{} + return []ingress.L4Service{} } ns, name, err := k8s.ParseNameNS(configmapName) if err != nil { glog.Errorf("unexpected error reading configmap %v: %v", name, err) - return []*ingress.Location{} + return []ingress.L4Service{} } configmap, err := ic.getConfigMap(ns, name) if err != nil { glog.Errorf("unexpected error reading configmap %v: %v", name, err) - return []*ingress.Location{} + return []ingress.L4Service{} } - var svcs []*ingress.Location + var svcs []ingress.L4Service // k -> port to expose // v -> /: for k, v := range configmap.Data { - _, err := strconv.Atoi(k) + externalPort, err := strconv.Atoi(k) if err != nil { glog.Warningf("%v is not valid as a TCP/UDP port", k) continue @@ -517,9 +517,15 @@ func (ic *GenericController) getStreamServices(configmapName string, proto api.P continue } - svcs = append(svcs, &ingress.Location{ - Path: k, - Backend: fmt.Sprintf("%v-%v-%v", svcNs, svcName, svcPort), + svcs = append(svcs, ingress.L4Service{ + Port: externalPort, + Backend: ingress.L4Backend{ + Name: svcName, + Namespace: svcNs, + Port: intstr.FromString(svcPort), + Protocol: proto, + }, + Endpoints: endps, }) } @@ -817,6 +823,8 @@ func (ic *GenericController) createServers(data []interface{}, SendTimeout: bdef.ProxySendTimeout, ReadTimeout: bdef.ProxyReadTimeout, BufferSize: bdef.ProxyBufferSize, + CookieDomain: bdef.ProxyCookieDomain, + CookiePath: bdef.ProxyCookiePath, } // This adds the Default Certificate to Default Backend and also for vhosts missing the secret diff --git a/core/pkg/ingress/types.go b/core/pkg/ingress/types.go index ef7fc4d836..0d8a8b7358 100644 --- a/core/pkg/ingress/types.go +++ b/core/pkg/ingress/types.go @@ -22,6 +22,7 @@ import ( "k8s.io/kubernetes/pkg/api" "k8s.io/kubernetes/pkg/client/cache" "k8s.io/kubernetes/pkg/healthz" + "k8s.io/kubernetes/pkg/util/intstr" cache_store "k8s.io/ingress/core/pkg/cache" "k8s.io/ingress/core/pkg/ingress/annotations/auth" @@ -132,10 +133,10 @@ type Configuration struct { Servers []*Server `json:"servers"` // TCPEndpoints contain endpoints for tcp streams handled by this backend // +optional - TCPEndpoints []*Location `json:"tcpEndpoints,omitempty"` + TCPEndpoints []L4Service `json:"tcpEndpoints,omitempty"` // UDPEndpoints contain endpoints for udp streams handled by this backend // +optional - UDPEndpoints []*Location `json:"udpEndpoints,omitempty"` + UDPEndpoints []L4Service `json:"udpEndpoints,omitempty"` // PassthroughBackend contains the backends used for SSL passthrough. // It contains information about the associated Server Name Indication (SNI). // +optional @@ -292,3 +293,21 @@ type SSLPassthroughBackend struct { // Hostname returns the FQDN of the server Hostname string `json:"hostname"` } + +// L4Service describes a L4 Ingress service. +type L4Service struct { + // Port external port to expose + Port int `json:"port"` + // Backend of the service + Backend L4Backend `json:"backend"` + // Endpoints active endpoints of the service + Endpoints []Endpoint `json:"endpoins"` +} + +// L4Backend describes the kubernetes service behind L4 Ingress service +type L4Backend struct { + Port intstr.IntOrString `json:"port"` + Name string `json:"name"` + Namespace string `json:"namespace"` + Protocol api.Protocol `json:"protocol"` +}