diff --git a/docs/user-guide/nginx-configuration/annotations.md b/docs/user-guide/nginx-configuration/annotations.md index 3027e19616..76ff807b94 100644 --- a/docs/user-guide/nginx-configuration/annotations.md +++ b/docs/user-guide/nginx-configuration/annotations.md @@ -30,6 +30,10 @@ You can add these Kubernetes annotations to specific Ingress objects to customiz |[nginx.ingress.kubernetes.io/auth-snippet](#external-authentication)|string| |[nginx.ingress.kubernetes.io/backend-protocol](#backend-protocol)|string|HTTP,HTTPS,GRPC,GRPCS,AJP| |[nginx.ingress.kubernetes.io/base-url-scheme](#rewrite)|string| +|[nginx.ingress.kubernetes.io/canary](#canary)|"true" or "false"| +|[nginx.ingress.kubernetes.io/canary-by-header](#canary)|string| +|[nginx.ingress.kubernetes.io/canary-by-cookie](#canary)|string| +|[nginx.ingress.kubernetes.io/canary-weight](#canary)|number| |[nginx.ingress.kubernetes.io/client-body-buffer-size](#client-body-buffer-size)|string| |[nginx.ingress.kubernetes.io/configuration-snippet](#configuration-snippet)|string| |[nginx.ingress.kubernetes.io/custom-http-errors](#custom-http-errors)|[]int| @@ -92,6 +96,23 @@ You can add these Kubernetes annotations to specific Ingress objects to customiz |[nginx.ingress.kubernetes.io/influxdb-server-name](#influxdb)|string| |[nginx.ingress.kubernetes.io/use-regex](#use-regex)|bool| +### Canary + +In some cases, you may want to "canary" a new set of changes by sending a small number of requests to a different service than the production service. The canary annotation enables the Ingress spec to act as an alternative service for requests to route to depending on the rules applied. The following annotations to configure canary can be enabled after `nginx.ingress.kubernetes.io/canary: "true"` is set: + +* `nginx.ingress.kubernetes.io/canary-by-header`: The header to use for notifying the Ingress to route the request to the service specified in the Canary Ingress. When the request header is set to `always`, it will be routed to the canary. When the header is set to `never`, it will never be routed to the canary. For any other value, the header will be ignored and the request compared against the other canary rules by precedence. + +* `nginx.ingress.kubernetes.io/canary-by-cookie`: The cookie to use for notifying the Ingress to route the request to the service specified in the Canary Ingress. When the cookie value is set to `always`, it will be routed to the canary. When the cookie is set to `never`, it will never be routed to the canary. For any other value, the cookie will be ingored and the request compared against the other canary rules by precedence. + +* `nginx.ingress.kubernetes.io/canary-weight`: The integer based (0 - 100) percent of random requests that should be routed to the service specified in the canary Ingress. A weight of 0 implies that no requests will be sent to the service in the Canary ingress by this canary rule. A weight of 100 means implies all requests will be sent to the alternative service specified in the Ingress. + +Canary rules are evaluated in order of precedence. Precedence is as follows: +`canary-by-header -> canary-by-cookie -> canary-weight` + +**Known Limitations** + +Currently a maximum of one canary ingress can be applied per Ingress rule. + ### Rewrite In some scenarios the exposed URL in the backend service differs from the specified path in the Ingress rule. Without a rewrite any request will return 404. diff --git a/internal/ingress/annotations/annotations.go b/internal/ingress/annotations/annotations.go index c59ab19d10..9cd1f12d32 100644 --- a/internal/ingress/annotations/annotations.go +++ b/internal/ingress/annotations/annotations.go @@ -19,6 +19,7 @@ package annotations import ( "github.com/golang/glog" "github.com/imdario/mergo" + "k8s.io/ingress-nginx/internal/ingress/annotations/canary" "k8s.io/ingress-nginx/internal/ingress/annotations/sslcipher" apiv1 "k8s.io/api/core/v1" @@ -68,6 +69,7 @@ type Ingress struct { BackendProtocol string Alias string BasicDigestAuth auth.Config + Canary canary.Config CertificateAuth authtls.Config ClientBodyBufferSize string ConfigurationSnippet string @@ -109,6 +111,7 @@ func NewAnnotationExtractor(cfg resolver.Resolver) Extractor { map[string]parser.IngressAnnotation{ "Alias": alias.NewParser(cfg), "BasicDigestAuth": auth.NewParser(auth.AuthDirectory, cfg), + "Canary": canary.NewParser(cfg), "CertificateAuth": authtls.NewParser(cfg), "ClientBodyBufferSize": clientbodybuffersize.NewParser(cfg), "ConfigurationSnippet": snippet.NewParser(cfg), diff --git a/internal/ingress/annotations/canary/main.go b/internal/ingress/annotations/canary/main.go new file mode 100644 index 0000000000..2e7305934b --- /dev/null +++ b/internal/ingress/annotations/canary/main.go @@ -0,0 +1,75 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package canary + +import ( + extensions "k8s.io/api/extensions/v1beta1" + + "k8s.io/ingress-nginx/internal/ingress/annotations/parser" + "k8s.io/ingress-nginx/internal/ingress/errors" + "k8s.io/ingress-nginx/internal/ingress/resolver" +) + +type canary struct { + r resolver.Resolver +} + +// Config returns the configuration rules for setting up the Canary +type Config struct { + Enabled bool + Weight int + Header string + Cookie string +} + +// NewParser parses the ingress for canary related annotations +func NewParser(r resolver.Resolver) parser.IngressAnnotation { + return canary{r} +} + +// Parse parses the annotations contained in the ingress +// rule used to indicate if the canary should be enabled and with what config +func (c canary) Parse(ing *extensions.Ingress) (interface{}, error) { + config := &Config{} + var err error + + config.Enabled, err = parser.GetBoolAnnotation("canary", ing) + if err != nil { + config.Enabled = false + } + + config.Weight, err = parser.GetIntAnnotation("canary-weight", ing) + if err != nil { + config.Weight = 0 + } + + config.Header, err = parser.GetStringAnnotation("canary-by-header", ing) + if err != nil { + config.Header = "" + } + + config.Cookie, err = parser.GetStringAnnotation("canary-by-cookie", ing) + if err != nil { + config.Cookie = "" + } + + if !config.Enabled && (config.Weight > 0 || len(config.Header) > 0 || len(config.Cookie) > 0) { + return nil, errors.NewInvalidAnnotationConfiguration("canary", "configured but not enabled") + } + + return config, nil +} diff --git a/internal/ingress/annotations/canary/main_test.go b/internal/ingress/annotations/canary/main_test.go new file mode 100644 index 0000000000..ddac491f2d --- /dev/null +++ b/internal/ingress/annotations/canary/main_test.go @@ -0,0 +1,126 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package canary + +import ( + api "k8s.io/api/core/v1" + extensions "k8s.io/api/extensions/v1beta1" + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/ingress-nginx/internal/ingress/annotations/parser" + "testing" + + "k8s.io/ingress-nginx/internal/ingress/resolver" + "strconv" +) + +func buildIngress() *extensions.Ingress { + defaultBackend := extensions.IngressBackend{ + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), + } + + return &extensions.Ingress{ + ObjectMeta: metaV1.ObjectMeta{ + Name: "foo", + Namespace: api.NamespaceDefault, + }, + Spec: extensions.IngressSpec{ + Backend: &extensions.IngressBackend{ + ServiceName: "default-backend", + ServicePort: intstr.FromInt(80), + }, + Rules: []extensions.IngressRule{ + { + Host: "foo.bar.com", + IngressRuleValue: extensions.IngressRuleValue{ + HTTP: &extensions.HTTPIngressRuleValue{ + Paths: []extensions.HTTPIngressPath{ + { + Path: "/foo", + Backend: defaultBackend, + }, + }, + }, + }, + }, + }, + }, + } +} + +func TestAnnotations(t *testing.T) { + ing := buildIngress() + + data := map[string]string{} + ing.SetAnnotations(data) + + tests := []struct { + title string + canaryEnabled bool + canaryWeight int + canaryHeader string + canaryCookie string + expErr bool + }{ + {"canary disabled and no weight", false, 0, "", "", false}, + {"canary disabled and weight", false, 20, "", "", true}, + {"canary disabled and header", false, 0, "X-Canary", "", true}, + {"canary disabled and cookie", false, 0, "", "canary_enabled", true}, + {"canary enabled and weight", true, 20, "", "", false}, + {"canary enabled and no weight", true, 0, "", "", false}, + {"canary enabled by header", true, 20, "X-Canary", "", false}, + {"canary enabled by cookie", true, 20, "", "canary_enabled", false}, + } + + for _, test := range tests { + data[parser.GetAnnotationWithPrefix("canary")] = strconv.FormatBool(test.canaryEnabled) + data[parser.GetAnnotationWithPrefix("canary-weight")] = strconv.Itoa(test.canaryWeight) + data[parser.GetAnnotationWithPrefix("canary-by-header")] = test.canaryHeader + data[parser.GetAnnotationWithPrefix("canary-by-cookie")] = test.canaryCookie + + i, err := NewParser(&resolver.Mock{}).Parse(ing) + if test.expErr { + if err == nil { + t.Errorf("%v: expected error but returned nil", test.title) + } + + continue + } else { + if err != nil { + t.Errorf("%v: expected nil but returned error %v", test.title, err) + } + } + + canaryConfig, ok := i.(*Config) + if !ok { + t.Errorf("%v: expected an External type", test.title) + } + if canaryConfig.Enabled != test.canaryEnabled { + t.Errorf("%v: expected \"%v\", but \"%v\" was returned", test.title, test.canaryEnabled, canaryConfig.Enabled) + } + if canaryConfig.Weight != test.canaryWeight { + t.Errorf("%v: expected \"%v\", but \"%v\" was returned", test.title, test.canaryWeight, canaryConfig.Weight) + } + if canaryConfig.Header != test.canaryHeader { + t.Errorf("%v: expected \"%v\", but \"%v\" was returned", test.title, test.canaryHeader, canaryConfig.Header) + } + if canaryConfig.Cookie != test.canaryCookie { + t.Errorf("%v: expected \"%v\", but \"%v\" was returned", test.title, test.canaryCookie, canaryConfig.Cookie) + } + } +} diff --git a/internal/ingress/controller/controller.go b/internal/ingress/controller/controller.go index f4eac58a56..45145e5ad6 100644 --- a/internal/ingress/controller/controller.go +++ b/internal/ingress/controller/controller.go @@ -268,6 +268,7 @@ func (n *NGINXController) getBackendServers(ingresses []*extensions.Ingress) ([] if host == "" { host = defServerName } + server := servers[host] if server == nil { server = servers[defServerName] @@ -300,13 +301,15 @@ func (n *NGINXController) getBackendServers(ingresses []*extensions.Ingress) ([] } for _, path := range rule.HTTP.Paths { - upsName := fmt.Sprintf("%v-%v-%v", - ing.Namespace, - path.Backend.ServiceName, - path.Backend.ServicePort.String()) + upsName := upstreamName(ing.Namespace, path.Backend.ServiceName, path.Backend.ServicePort) ups := upstreams[upsName] + // Backend is not referenced to by a server + if ups.NoServer { + continue + } + nginxPath := rootLocation if path.Path != "" { nginxPath = path.Path @@ -420,6 +423,11 @@ func (n *NGINXController) getBackendServers(ingresses []*extensions.Ingress) ([] } } } + + if anns.Canary.Enabled { + glog.Infof("Canary ingress %v detected. Finding eligible backends to merge into.", ing.Name) + mergeAlternativeBackends(ing, upstreams, servers) + } } aUpstreams := make([]*ingress.Backend, 0, len(upstreams)) @@ -508,10 +516,7 @@ func (n *NGINXController) createUpstreams(data []*extensions.Ingress, du *ingres var defBackend string if ing.Spec.Backend != nil { - defBackend = fmt.Sprintf("%v-%v-%v", - ing.Namespace, - ing.Spec.Backend.ServiceName, - ing.Spec.Backend.ServicePort.String()) + defBackend = upstreamName(ing.Namespace, ing.Spec.Backend.ServiceName, ing.Spec.Backend.ServicePort) glog.V(3).Infof("Creating upstream %q", defBackend) upstreams[defBackend] = newUpstream(defBackend) @@ -537,6 +542,16 @@ func (n *NGINXController) createUpstreams(data []*extensions.Ingress, du *ingres } } + // configure traffic shaping for canary + if anns.Canary.Enabled { + upstreams[defBackend].NoServer = true + upstreams[defBackend].TrafficShapingPolicy = ingress.TrafficShapingPolicy{ + Weight: anns.Canary.Weight, + Header: anns.Canary.Header, + Cookie: anns.Canary.Cookie, + } + } + if len(upstreams[defBackend].Endpoints) == 0 { endps, err := n.serviceEndpoints(svcKey, ing.Spec.Backend.ServicePort.String()) upstreams[defBackend].Endpoints = append(upstreams[defBackend].Endpoints, endps...) @@ -558,10 +573,7 @@ func (n *NGINXController) createUpstreams(data []*extensions.Ingress, du *ingres } for _, path := range rule.HTTP.Paths { - name := fmt.Sprintf("%v-%v-%v", - ing.Namespace, - path.Backend.ServiceName, - path.Backend.ServicePort.String()) + name := upstreamName(ing.Namespace, path.Backend.ServiceName, path.Backend.ServicePort) if _, ok := upstreams[name]; ok { continue @@ -595,6 +607,16 @@ func (n *NGINXController) createUpstreams(data []*extensions.Ingress, du *ingres } } + // configure traffic shaping for canary + if anns.Canary.Enabled { + upstreams[name].NoServer = true + upstreams[name].TrafficShapingPolicy = ingress.TrafficShapingPolicy{ + Weight: anns.Canary.Weight, + Header: anns.Canary.Header, + Cookie: anns.Canary.Cookie, + } + } + if len(upstreams[name].Endpoints) == 0 { endp, err := n.serviceEndpoints(svcKey, path.Backend.ServicePort.String()) if err != nil { @@ -970,6 +992,63 @@ func (n *NGINXController) createServers(data []*extensions.Ingress, return servers } +// Compares an Ingress of a potential alternative backend's rules with each existing server and finds matching host + path pairs. +// If a match is found, we know that this server should back the alternative backend and add the alternative backend +// to a backend's alternative list. +// If no match is found, then the serverless backend is deleted. +func mergeAlternativeBackends(ing *extensions.Ingress, upstreams map[string]*ingress.Backend, + servers map[string]*ingress.Server) { + + // merge catch-all alternative backends + if ing.Spec.Backend != nil { + upsName := upstreamName(ing.Namespace, ing.Spec.Backend.ServiceName, ing.Spec.Backend.ServicePort) + + ups := upstreams[upsName] + + defLoc := servers[defServerName].Locations[0] + + glog.Infof("matching backend %v found for alternative backend %v", + upstreams[defLoc.Backend].Name, ups.Name) + + upstreams[defLoc.Backend].AlternativeBackends = + append(upstreams[defLoc.Backend].AlternativeBackends, ups.Name) + } + + for _, rule := range ing.Spec.Rules { + for _, path := range rule.HTTP.Paths { + upsName := upstreamName(ing.Namespace, path.Backend.ServiceName, path.Backend.ServicePort) + + ups := upstreams[upsName] + + merged := false + + server := servers[rule.Host] + + // find matching paths + for _, location := range server.Locations { + if location.Backend == defUpstreamName { + continue + } + + if location.Path == path.Path && !upstreams[location.Backend].NoServer { + glog.Infof("matching backend %v found for alternative backend %v", + upstreams[location.Backend].Name, ups.Name) + + upstreams[location.Backend].AlternativeBackends = + append(upstreams[location.Backend].AlternativeBackends, ups.Name) + + merged = true + } + } + + if !merged { + glog.Warningf("unable to find real backend for alternative backend %v. Deleting.", ups.Name) + delete(upstreams, ups.Name) + } + } + } +} + // extractTLSSecretName returns the name of the Secret containing a SSL // certificate for the given host name, or an empty string. func extractTLSSecretName(host string, ing *extensions.Ingress, diff --git a/internal/ingress/controller/controller_test.go b/internal/ingress/controller/controller_test.go index 12c89619aa..5edd51d0c0 100644 --- a/internal/ingress/controller/controller_test.go +++ b/internal/ingress/controller/controller_test.go @@ -20,6 +20,7 @@ import ( "crypto/x509" "crypto/x509/pkix" "encoding/asn1" + "k8s.io/apimachinery/pkg/util/intstr" "testing" extensions "k8s.io/api/extensions/v1beta1" @@ -27,6 +28,184 @@ import ( "k8s.io/ingress-nginx/internal/ingress" ) +func TestMergeAlternativeBackends(t *testing.T) { + testCases := map[string]struct { + ingress *extensions.Ingress + upstreams map[string]*ingress.Backend + servers map[string]*ingress.Server + expNumAlternativeBackends int + expNumLocations int + }{ + "alternative backend has no server and embeds into matching real backend": { + &extensions.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "example", + }, + Spec: extensions.IngressSpec{ + Rules: []extensions.IngressRule{ + { + Host: "example.com", + IngressRuleValue: extensions.IngressRuleValue{ + HTTP: &extensions.HTTPIngressRuleValue{ + Paths: []extensions.HTTPIngressPath{ + { + Path: "/", + Backend: extensions.IngressBackend{ + ServiceName: "http-svc-canary", + ServicePort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 80, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + map[string]*ingress.Backend{ + "example-http-svc-80": { + Name: "example-http-svc-80", + NoServer: false, + }, + "example-http-svc-canary-80": { + Name: "example-http-svc-canary-80", + NoServer: true, + TrafficShapingPolicy: ingress.TrafficShapingPolicy{ + Weight: 20, + }, + }, + }, + map[string]*ingress.Server{ + "example.com": { + Hostname: "example.com", + Locations: []*ingress.Location{ + { + Path: "/", + Backend: "example-http-svc-80", + }, + }, + }, + }, + 1, + 1, + }, + "merging a alternative backend matches with the correct host": { + &extensions.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "example", + }, + Spec: extensions.IngressSpec{ + Rules: []extensions.IngressRule{ + { + Host: "foo.bar", + IngressRuleValue: extensions.IngressRuleValue{ + HTTP: &extensions.HTTPIngressRuleValue{ + Paths: []extensions.HTTPIngressPath{ + { + Path: "/", + Backend: extensions.IngressBackend{ + ServiceName: "foo-http-svc-canary", + ServicePort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 80, + }, + }, + }, + }, + }, + }, + }, + { + Host: "example.com", + IngressRuleValue: extensions.IngressRuleValue{ + HTTP: &extensions.HTTPIngressRuleValue{ + Paths: []extensions.HTTPIngressPath{ + { + Path: "/", + Backend: extensions.IngressBackend{ + ServiceName: "http-svc-canary", + ServicePort: intstr.IntOrString{ + Type: intstr.Int, + IntVal: 80, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + map[string]*ingress.Backend{ + "example-foo-http-svc-80": { + Name: "example-foo-http-svc-80", + NoServer: false, + }, + "example-foo-http-svc-canary-80": { + Name: "example-foo-http-svc-canary-80", + NoServer: true, + TrafficShapingPolicy: ingress.TrafficShapingPolicy{ + Weight: 20, + }, + }, + "example-http-svc-80": { + Name: "example-http-svc-80", + NoServer: false, + }, + "example-http-svc-canary-80": { + Name: "example-http-svc-canary-80", + NoServer: true, + TrafficShapingPolicy: ingress.TrafficShapingPolicy{ + Weight: 20, + }, + }, + }, + map[string]*ingress.Server{ + "foo.bar": { + Hostname: "foo.bar", + Locations: []*ingress.Location{ + { + Path: "/", + Backend: "example-foo-http-svc-80", + }, + }, + }, + "example.com": { + Hostname: "example.com", + Locations: []*ingress.Location{ + { + Path: "/", + Backend: "example-http-svc-80", + }, + }, + }, + }, + 1, + 1, + }, + } + + for title, tc := range testCases { + t.Run(title, func(t *testing.T) { + mergeAlternativeBackends(tc.ingress, tc.upstreams, tc.servers) + + numAlternativeBackends := len(tc.upstreams["example-http-svc-80"].AlternativeBackends) + if numAlternativeBackends != tc.expNumAlternativeBackends { + t.Errorf("expected %d alternative backends (got %d)", tc.expNumAlternativeBackends, numAlternativeBackends) + } + + numLocations := len(tc.servers["example.com"].Locations) + if numLocations != tc.expNumLocations { + t.Errorf("expected %d locations (got %d)", tc.expNumLocations, numLocations) + } + }) + } +} + func TestExtractTLSSecretName(t *testing.T) { testCases := map[string]struct { host string diff --git a/internal/ingress/controller/nginx.go b/internal/ingress/controller/nginx.go index f99b8d58e7..8392eab627 100644 --- a/internal/ingress/controller/nginx.go +++ b/internal/ingress/controller/nginx.go @@ -759,13 +759,16 @@ func configureDynamically(pcfg *ingress.Configuration, port int, isDynamicCertif service = &apiv1.Service{Spec: backend.Service.Spec} } luaBackend := &ingress.Backend{ - Name: backend.Name, - Port: backend.Port, - SSLPassthrough: backend.SSLPassthrough, - SessionAffinity: backend.SessionAffinity, - UpstreamHashBy: backend.UpstreamHashBy, - LoadBalancing: backend.LoadBalancing, - Service: service, + Name: backend.Name, + Port: backend.Port, + SSLPassthrough: backend.SSLPassthrough, + SessionAffinity: backend.SessionAffinity, + UpstreamHashBy: backend.UpstreamHashBy, + LoadBalancing: backend.LoadBalancing, + Service: service, + NoServer: backend.NoServer, + TrafficShapingPolicy: backend.TrafficShapingPolicy, + AlternativeBackends: backend.AlternativeBackends, } var endpoints []ingress.Endpoint diff --git a/internal/ingress/controller/util.go b/internal/ingress/controller/util.go index a6e5e20c0b..4e03b5e8f0 100644 --- a/internal/ingress/controller/util.go +++ b/internal/ingress/controller/util.go @@ -17,10 +17,13 @@ limitations under the License. package controller import ( + "k8s.io/apimachinery/pkg/util/intstr" "os" "os/exec" "syscall" + "fmt" + "github.com/golang/glog" api "k8s.io/api/core/v1" @@ -43,6 +46,11 @@ func newUpstream(name string) *ingress.Backend { } } +// upstreamName returns a formatted upstream name based on namespace, service, and port +func upstreamName(namespace string, service string, port intstr.IntOrString) string { + return fmt.Sprintf("%v-%v-%v", namespace, service, port.String()) +} + // sysctlSomaxconn returns the maximum number of connections that can be queued // for acceptance (value of net.core.somaxconn) // http://nginx.org/en/docs/http/ngx_http_core_module.html#listen diff --git a/internal/ingress/errors/errors.go b/internal/ingress/errors/errors.go index b58c83fbbb..11ca0f3cf4 100644 --- a/internal/ingress/errors/errors.go +++ b/internal/ingress/errors/errors.go @@ -32,6 +32,14 @@ var ( ErrInvalidAnnotationName = errors.New("invalid annotation name") ) +// NewInvalidAnnotationConfiguration returns a new InvalidConfiguration error for use when +// annotations are not correctly configured +func NewInvalidAnnotationConfiguration(name string, reason string) error { + return InvalidConfiguration{ + Name: fmt.Sprintf("the annotation %v does not contain a valid configuration: %v", name, reason), + } +} + // NewInvalidAnnotationContent returns a new InvalidContent error func NewInvalidAnnotationContent(name string, val interface{}) error { return InvalidContent{ @@ -46,6 +54,15 @@ func NewLocationDenied(reason string) error { } } +// InvalidConfiguration Error +type InvalidConfiguration struct { + Name string +} + +func (e InvalidConfiguration) Error() string { + return e.Name +} + // InvalidContent error type InvalidContent struct { Name string diff --git a/internal/ingress/types.go b/internal/ingress/types.go index 90af5a9be6..64c5d0eae7 100644 --- a/internal/ingress/types.go +++ b/internal/ingress/types.go @@ -85,6 +85,30 @@ type Backend struct { UpstreamHashBy string `json:"upstream-hash-by,omitempty"` // LB algorithm configuration per ingress LoadBalancing string `json:"load-balance,omitempty"` + // Denotes if a backend has no server. The backend instead shares a server with another backend and acts as an + // alternative backend. + // This can be used to share multiple upstreams in the sam nginx server block. + NoServer bool `json:"noServer"` + // Policies to describe the characteristics of an alternative backend. + // +optional + TrafficShapingPolicy TrafficShapingPolicy `json:"trafficShapingPolicy,omitempty"` + // Contains a list of backends without servers that are associated with this backend. + // +optional + AlternativeBackends []string `json:"alternativeBackends,omitempty"` +} + +// TrafficShapingPolicy describes the policies to put in place when a backend has no server and is used as an +// alternative backend +// +k8s:deepcopy-gen=true +type TrafficShapingPolicy struct { + // Weight (0-100) of traffic to redirect to the backend. + // e.g. Weight 20 means 20% of traffic will be redirected to the backend and 80% will remain + // with the other backend. 0 weight will not send any traffic to this backend + Weight int `json:"weight"` + // Header on which to redirect requests to this backend + Header string `json:"header"` + // Cookie on which to redirect requests to this backend + Cookie string `json:"cookie"` } // HashInclude defines if a field should be used or not to calculate the hash diff --git a/internal/ingress/types_equals.go b/internal/ingress/types_equals.go index 31e88d1cda..659d85d0f6 100644 --- a/internal/ingress/types_equals.go +++ b/internal/ingress/types_equals.go @@ -84,6 +84,9 @@ func (b1 *Backend) Equal(b2 *Backend) bool { if b1.Name != b2.Name { return false } + if b1.NoServer != b2.NoServer { + return false + } if b1.Service != b2.Service { if b1.Service == nil || b2.Service == nil { @@ -133,6 +136,23 @@ func (b1 *Backend) Equal(b2 *Backend) bool { } } + if !b1.TrafficShapingPolicy.Equal(b2.TrafficShapingPolicy) { + return false + } + + for _, vb1 := range b1.AlternativeBackends { + found := false + for _, vb2 := range b2.AlternativeBackends { + if vb1 == vb2 { + found = true + break + } + } + if !found { + return false + } + } + return true } @@ -202,6 +222,21 @@ func (e1 *Endpoint) Equal(e2 *Endpoint) bool { return true } +// Equal checks for equality between two TrafficShapingPolicies +func (tsp1 TrafficShapingPolicy) Equal(tsp2 TrafficShapingPolicy) bool { + if tsp1.Weight != tsp2.Weight { + return false + } + if tsp1.Header != tsp2.Header { + return false + } + if tsp1.Cookie != tsp2.Cookie { + return false + } + + return true +} + // Equal tests for equality between two Server types func (s1 *Server) Equal(s2 *Server) bool { if s1 == s2 { diff --git a/rootfs/etc/nginx/lua/balancer.lua b/rootfs/etc/nginx/lua/balancer.lua index cefa86373e..2d88f8f798 100644 --- a/rootfs/etc/nginx/lua/balancer.lua +++ b/rootfs/etc/nginx/lua/balancer.lua @@ -127,9 +127,57 @@ local function sync_backends() end end +local function route_to_alternative_balancer(balancer) + if not balancer.alternative_backends then + return false + end + + -- TODO: support traffic shaping for n > 1 alternative backends + local alternative_balancer = balancers[balancer.alternative_backends[1]] + + local clean_target_header = util.replace_special_char(alternative_balancer.traffic_shaping_policy.header, "-", "_") + + local header = ngx.var["http_" .. clean_target_header] + if header then + if header == "always" then + return true + elseif header == "never" then + return false + end + end + + local clean_target_cookie = util.replace_special_char(alternative_balancer.traffic_shaping_policy.cookie, "-", "_") + + local cookie = ngx.var["cookie_" .. clean_target_cookie] + if cookie then + if cookie == "always" then + return true + elseif cookie == "never" then + return false + end + end + + if math.random(100) <= alternative_balancer.traffic_shaping_policy.weight then + return true + end + + return false +end + local function get_balancer() local backend_name = ngx.var.proxy_upstream_name - return balancers[backend_name] + + local balancer = balancers[backend_name] + if not balancer then + return + end + + if route_to_alternative_balancer(balancer) then + local alternative_balancer = balancers[balancer.alternative_backends[1]] + return alternative_balancer + end + + return balancer end function _M.init_worker() diff --git a/rootfs/etc/nginx/lua/balancer/chash.lua b/rootfs/etc/nginx/lua/balancer/chash.lua index c7ebacbd99..d9108a8b3a 100644 --- a/rootfs/etc/nginx/lua/balancer/chash.lua +++ b/rootfs/etc/nginx/lua/balancer/chash.lua @@ -6,7 +6,10 @@ local _M = balancer_resty:new({ factory = resty_chash, name = "chash" }) function _M.new(self, backend) local nodes = util.get_nodes(backend.endpoints) - local o = { instance = self.factory:new(nodes), hash_by = backend["upstream-hash-by"] } + local o = { + instance = self.factory:new(nodes), + hash_by = backend["upstream-hash-by"], + } setmetatable(o, self) self.__index = self return o diff --git a/rootfs/etc/nginx/lua/balancer/ewma.lua b/rootfs/etc/nginx/lua/balancer/ewma.lua index 352b654f7b..8e4512000d 100644 --- a/rootfs/etc/nginx/lua/balancer/ewma.lua +++ b/rootfs/etc/nginx/lua/balancer/ewma.lua @@ -102,6 +102,9 @@ function _M.after_balance(_) end function _M.sync(self, backend) + self.traffic_shaping_policy = backend.trafficShapingPolicy + self.alternative_backends = backend.alternativeBackends + local changed = not util.deep_compare(self.peers, backend.endpoints) if not changed then return @@ -115,7 +118,9 @@ function _M.sync(self, backend) end function _M.new(self, backend) - local o = { peers = backend.endpoints } + local o = { + peers = backend.endpoints, + } setmetatable(o, self) self.__index = self return o diff --git a/rootfs/etc/nginx/lua/balancer/resty.lua b/rootfs/etc/nginx/lua/balancer/resty.lua index 457300ed5f..fa236246f2 100644 --- a/rootfs/etc/nginx/lua/balancer/resty.lua +++ b/rootfs/etc/nginx/lua/balancer/resty.lua @@ -10,6 +10,9 @@ function _M.new(self, o) end function _M.sync(self, backend) + self.traffic_shaping_policy = backend.trafficShapingPolicy + self.alternative_backends = backend.alternativeBackends + local nodes = util.get_nodes(backend.endpoints) local changed = not util.deep_compare(self.instance.nodes, nodes) if not changed then diff --git a/rootfs/etc/nginx/lua/balancer/round_robin.lua b/rootfs/etc/nginx/lua/balancer/round_robin.lua index 181e0daec3..db6c8e7ddb 100644 --- a/rootfs/etc/nginx/lua/balancer/round_robin.lua +++ b/rootfs/etc/nginx/lua/balancer/round_robin.lua @@ -6,7 +6,9 @@ local _M = balancer_resty:new({ factory = resty_roundrobin, name = "round_robin" function _M.new(self, backend) local nodes = util.get_nodes(backend.endpoints) - local o = { instance = self.factory:new(nodes) } + local o = { + instance = self.factory:new(nodes), + } setmetatable(o, self) self.__index = self return o diff --git a/rootfs/etc/nginx/lua/util.lua b/rootfs/etc/nginx/lua/util.lua index 2ef42ee833..331a82a239 100644 --- a/rootfs/etc/nginx/lua/util.lua +++ b/rootfs/etc/nginx/lua/util.lua @@ -108,4 +108,10 @@ local function tablelength(T) end _M.tablelength = tablelength +-- replaces special character value a with value b for all occurences in a string +local function replace_special_char(str, a, b) + return string.gsub(str, "%" .. a, b) +end +_M.replace_special_char = replace_special_char + return _M diff --git a/test/e2e/annotations/canary.go b/test/e2e/annotations/canary.go new file mode 100644 index 0000000000..b2c5186678 --- /dev/null +++ b/test/e2e/annotations/canary.go @@ -0,0 +1,199 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package annotations + +import ( + "fmt" + "net/http" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/parnurzeal/gorequest" + + "k8s.io/ingress-nginx/test/e2e/framework" +) + +const ( + waitForLuaSync = 5 * time.Second +) + +var _ = framework.IngressNginxDescribe("Annotations - canary", func() { + f := framework.NewDefaultFramework("canary") + + BeforeEach(func() { + // Deployment for main backend + f.NewEchoDeployment() + + // Deployment for canary backend + f.NewDeployment("http-svc-canary", "gcr.io/kubernetes-e2e-test-images/echoserver:2.1", 8080, 1) + }) + + Context("when canaried by header", func() { + It("should route requests to the canary pod if header is set to 'always'", func() { + host := "foo" + annotations := map[string]string{} + + ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) + f.EnsureIngress(ing) + + f.WaitForNginxServer(host, + func(server string) bool { + return Expect(server).Should(ContainSubstring("server_name foo")) && + Expect(server).ShouldNot(ContainSubstring("return 503")) + }) + + canaryAnnotations := map[string]string{ + "nginx.ingress.kubernetes.io/canary": "true", + "nginx.ingress.kubernetes.io/canary-by-header": "CanaryByHeader", + } + + canaryIngName := fmt.Sprintf("%v-canary", host) + + canaryIng := framework.NewSingleIngress(canaryIngName, "/", host, f.IngressController.Namespace, "http-svc-canary", + 80, &canaryAnnotations) + f.EnsureIngress(canaryIng) + + time.Sleep(waitForLuaSync) + + resp, body, errs := gorequest.New(). + Get(f.IngressController.HTTPURL). + Set("Host", host). + Set("CanaryByHeader", "always"). + End() + + Expect(len(errs)).Should(BeNumerically("==", 0)) + Expect(resp.StatusCode).ShouldNot(Equal(http.StatusNotFound)) + Expect(body).Should(ContainSubstring("http-svc-canary")) + }) + + It("should not route requests to the canary pod if header is set to 'never'", func() { + host := "foo" + annotations := map[string]string{} + + ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) + f.EnsureIngress(ing) + + f.WaitForNginxServer(host, + func(server string) bool { + return Expect(server).Should(ContainSubstring("server_name foo")) && + Expect(server).ShouldNot(ContainSubstring("return 503")) + }) + + canaryAnnotations := map[string]string{ + "nginx.ingress.kubernetes.io/canary": "true", + "nginx.ingress.kubernetes.io/canary-by-header": "CanaryByHeader", + } + + canaryIngName := fmt.Sprintf("%v-canary", host) + + canaryIng := framework.NewSingleIngress(canaryIngName, "/", host, f.IngressController.Namespace, "http-svc-canary", + 80, &canaryAnnotations) + f.EnsureIngress(canaryIng) + + time.Sleep(waitForLuaSync) + + resp, body, errs := gorequest.New(). + Get(f.IngressController.HTTPURL). + Set("Host", host). + Set("CanaryByHeader", "never"). + End() + + Expect(len(errs)).Should(BeNumerically("==", 0)) + Expect(resp.StatusCode).ShouldNot(Equal(http.StatusNotFound)) + Expect(body).ShouldNot(ContainSubstring("http-svc-canary")) + }) + }) + + Context("when canaried by cookie", func() { + It("should route requests to the canary pod if cookie is set to 'always'", func() { + host := "foo" + annotations := map[string]string{} + + ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) + f.EnsureIngress(ing) + + f.WaitForNginxServer(host, + func(server string) bool { + return Expect(server).Should(ContainSubstring("server_name foo")) && + Expect(server).ShouldNot(ContainSubstring("return 503")) + }) + + canaryAnnotations := map[string]string{ + "nginx.ingress.kubernetes.io/canary": "true", + "nginx.ingress.kubernetes.io/canary-by-cookie": "CanaryByCookie", + } + + canaryIngName := fmt.Sprintf("%v-canary", host) + + canaryIng := framework.NewSingleIngress(canaryIngName, "/", host, f.IngressController.Namespace, "http-svc-canary", + 80, &canaryAnnotations) + f.EnsureIngress(canaryIng) + + time.Sleep(waitForLuaSync) + + resp, body, errs := gorequest.New(). + Get(f.IngressController.HTTPURL). + Set("Host", host). + AddCookie(&http.Cookie{Name: "CanaryByCookie", Value: "always"}). + End() + + Expect(len(errs)).Should(BeNumerically("==", 0)) + Expect(resp.StatusCode).ShouldNot(Equal(http.StatusNotFound)) + Expect(body).Should(ContainSubstring("http-svc-canary")) + }) + + It("should not route requests to the canary pod if cookie is set to 'never'", func() { + host := "foo" + annotations := map[string]string{} + + ing := framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, &annotations) + f.EnsureIngress(ing) + + Expect(ing).NotTo(BeNil()) + + f.WaitForNginxServer(host, + func(server string) bool { + return Expect(server).Should(ContainSubstring("server_name foo")) && + Expect(server).ShouldNot(ContainSubstring("return 503")) + }) + + canaryAnnotations := map[string]string{ + "nginx.ingress.kubernetes.io/canary": "true", + "nginx.ingress.kubernetes.io/canary-by-header": "CanaryByHeader", + } + + canaryIngName := fmt.Sprintf("%v-canary", host) + + canaryIng := framework.NewSingleIngress(canaryIngName, "/", host, f.IngressController.Namespace, "http-svc-canary", + 80, &canaryAnnotations) + f.EnsureIngress(canaryIng) + + time.Sleep(waitForLuaSync) + + resp, body, errs := gorequest.New(). + Get(f.IngressController.HTTPURL). + Set("Host", host). + AddCookie(&http.Cookie{Name: "CanaryByCookie", Value: "never"}). + End() + + Expect(len(errs)).Should(BeNumerically("==", 0)) + Expect(resp.StatusCode).ShouldNot(Equal(http.StatusNotFound)) + Expect(body).ShouldNot(ContainSubstring("http-svc-canary")) + }) + }) +})