Skip to content

Commit

Permalink
Merge pull request #3341 from Shopify/canary_upstream
Browse files Browse the repository at this point in the history
Add canary annotation and alternative backends for traffic shaping
  • Loading branch information
k8s-ci-robot authored Nov 6, 2018
2 parents 265f96b + 412cd70 commit 17cad51
Show file tree
Hide file tree
Showing 18 changed files with 859 additions and 23 deletions.
21 changes: 21 additions & 0 deletions docs/user-guide/nginx-configuration/annotations.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,10 @@ You can add these Kubernetes annotations to specific Ingress objects to customiz
|[nginx.ingress.kubernetes.io/auth-snippet](#external-authentication)|string|
|[nginx.ingress.kubernetes.io/backend-protocol](#backend-protocol)|string|HTTP,HTTPS,GRPC,GRPCS,AJP|
|[nginx.ingress.kubernetes.io/base-url-scheme](#rewrite)|string|
|[nginx.ingress.kubernetes.io/canary](#canary)|"true" or "false"|
|[nginx.ingress.kubernetes.io/canary-by-header](#canary)|string|
|[nginx.ingress.kubernetes.io/canary-by-cookie](#canary)|string|
|[nginx.ingress.kubernetes.io/canary-weight](#canary)|number|
|[nginx.ingress.kubernetes.io/client-body-buffer-size](#client-body-buffer-size)|string|
|[nginx.ingress.kubernetes.io/configuration-snippet](#configuration-snippet)|string|
|[nginx.ingress.kubernetes.io/custom-http-errors](#custom-http-errors)|[]int|
Expand Down Expand Up @@ -92,6 +96,23 @@ You can add these Kubernetes annotations to specific Ingress objects to customiz
|[nginx.ingress.kubernetes.io/influxdb-server-name](#influxdb)|string|
|[nginx.ingress.kubernetes.io/use-regex](#use-regex)|bool|

### Canary

In some cases, you may want to "canary" a new set of changes by sending a small number of requests to a different service than the production service. The canary annotation enables the Ingress spec to act as an alternative service for requests to route to depending on the rules applied. The following annotations to configure canary can be enabled after `nginx.ingress.kubernetes.io/canary: "true"` is set:

* `nginx.ingress.kubernetes.io/canary-by-header`: The header to use for notifying the Ingress to route the request to the service specified in the Canary Ingress. When the request header is set to `always`, it will be routed to the canary. When the header is set to `never`, it will never be routed to the canary. For any other value, the header will be ignored and the request compared against the other canary rules by precedence.

* `nginx.ingress.kubernetes.io/canary-by-cookie`: The cookie to use for notifying the Ingress to route the request to the service specified in the Canary Ingress. When the cookie value is set to `always`, it will be routed to the canary. When the cookie is set to `never`, it will never be routed to the canary. For any other value, the cookie will be ingored and the request compared against the other canary rules by precedence.

* `nginx.ingress.kubernetes.io/canary-weight`: The integer based (0 - 100) percent of random requests that should be routed to the service specified in the canary Ingress. A weight of 0 implies that no requests will be sent to the service in the Canary ingress by this canary rule. A weight of 100 means implies all requests will be sent to the alternative service specified in the Ingress.

Canary rules are evaluated in order of precedence. Precedence is as follows:
`canary-by-header -> canary-by-cookie -> canary-weight`

**Known Limitations**

Currently a maximum of one canary ingress can be applied per Ingress rule.

### Rewrite

In some scenarios the exposed URL in the backend service differs from the specified path in the Ingress rule. Without a rewrite any request will return 404.
Expand Down
3 changes: 3 additions & 0 deletions internal/ingress/annotations/annotations.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ package annotations
import (
"github.com/golang/glog"
"github.com/imdario/mergo"
"k8s.io/ingress-nginx/internal/ingress/annotations/canary"
"k8s.io/ingress-nginx/internal/ingress/annotations/sslcipher"

apiv1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -68,6 +69,7 @@ type Ingress struct {
BackendProtocol string
Alias string
BasicDigestAuth auth.Config
Canary canary.Config
CertificateAuth authtls.Config
ClientBodyBufferSize string
ConfigurationSnippet string
Expand Down Expand Up @@ -109,6 +111,7 @@ func NewAnnotationExtractor(cfg resolver.Resolver) Extractor {
map[string]parser.IngressAnnotation{
"Alias": alias.NewParser(cfg),
"BasicDigestAuth": auth.NewParser(auth.AuthDirectory, cfg),
"Canary": canary.NewParser(cfg),
"CertificateAuth": authtls.NewParser(cfg),
"ClientBodyBufferSize": clientbodybuffersize.NewParser(cfg),
"ConfigurationSnippet": snippet.NewParser(cfg),
Expand Down
75 changes: 75 additions & 0 deletions internal/ingress/annotations/canary/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package canary

import (
extensions "k8s.io/api/extensions/v1beta1"

"k8s.io/ingress-nginx/internal/ingress/annotations/parser"
"k8s.io/ingress-nginx/internal/ingress/errors"
"k8s.io/ingress-nginx/internal/ingress/resolver"
)

type canary struct {
r resolver.Resolver
}

// Config returns the configuration rules for setting up the Canary
type Config struct {
Enabled bool
Weight int
Header string
Cookie string
}

// NewParser parses the ingress for canary related annotations
func NewParser(r resolver.Resolver) parser.IngressAnnotation {
return canary{r}
}

// Parse parses the annotations contained in the ingress
// rule used to indicate if the canary should be enabled and with what config
func (c canary) Parse(ing *extensions.Ingress) (interface{}, error) {
config := &Config{}
var err error

config.Enabled, err = parser.GetBoolAnnotation("canary", ing)
if err != nil {
config.Enabled = false
}

config.Weight, err = parser.GetIntAnnotation("canary-weight", ing)
if err != nil {
config.Weight = 0
}

config.Header, err = parser.GetStringAnnotation("canary-by-header", ing)
if err != nil {
config.Header = ""
}

config.Cookie, err = parser.GetStringAnnotation("canary-by-cookie", ing)
if err != nil {
config.Cookie = ""
}

if !config.Enabled && (config.Weight > 0 || len(config.Header) > 0 || len(config.Cookie) > 0) {
return nil, errors.NewInvalidAnnotationConfiguration("canary", "configured but not enabled")
}

return config, nil
}
126 changes: 126 additions & 0 deletions internal/ingress/annotations/canary/main_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package canary

import (
api "k8s.io/api/core/v1"
extensions "k8s.io/api/extensions/v1beta1"
metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/ingress-nginx/internal/ingress/annotations/parser"
"testing"

"k8s.io/ingress-nginx/internal/ingress/resolver"
"strconv"
)

func buildIngress() *extensions.Ingress {
defaultBackend := extensions.IngressBackend{
ServiceName: "default-backend",
ServicePort: intstr.FromInt(80),
}

return &extensions.Ingress{
ObjectMeta: metaV1.ObjectMeta{
Name: "foo",
Namespace: api.NamespaceDefault,
},
Spec: extensions.IngressSpec{
Backend: &extensions.IngressBackend{
ServiceName: "default-backend",
ServicePort: intstr.FromInt(80),
},
Rules: []extensions.IngressRule{
{
Host: "foo.bar.com",
IngressRuleValue: extensions.IngressRuleValue{
HTTP: &extensions.HTTPIngressRuleValue{
Paths: []extensions.HTTPIngressPath{
{
Path: "/foo",
Backend: defaultBackend,
},
},
},
},
},
},
},
}
}

func TestAnnotations(t *testing.T) {
ing := buildIngress()

data := map[string]string{}
ing.SetAnnotations(data)

tests := []struct {
title string
canaryEnabled bool
canaryWeight int
canaryHeader string
canaryCookie string
expErr bool
}{
{"canary disabled and no weight", false, 0, "", "", false},
{"canary disabled and weight", false, 20, "", "", true},
{"canary disabled and header", false, 0, "X-Canary", "", true},
{"canary disabled and cookie", false, 0, "", "canary_enabled", true},
{"canary enabled and weight", true, 20, "", "", false},
{"canary enabled and no weight", true, 0, "", "", false},
{"canary enabled by header", true, 20, "X-Canary", "", false},
{"canary enabled by cookie", true, 20, "", "canary_enabled", false},
}

for _, test := range tests {
data[parser.GetAnnotationWithPrefix("canary")] = strconv.FormatBool(test.canaryEnabled)
data[parser.GetAnnotationWithPrefix("canary-weight")] = strconv.Itoa(test.canaryWeight)
data[parser.GetAnnotationWithPrefix("canary-by-header")] = test.canaryHeader
data[parser.GetAnnotationWithPrefix("canary-by-cookie")] = test.canaryCookie

i, err := NewParser(&resolver.Mock{}).Parse(ing)
if test.expErr {
if err == nil {
t.Errorf("%v: expected error but returned nil", test.title)
}

continue
} else {
if err != nil {
t.Errorf("%v: expected nil but returned error %v", test.title, err)
}
}

canaryConfig, ok := i.(*Config)
if !ok {
t.Errorf("%v: expected an External type", test.title)
}
if canaryConfig.Enabled != test.canaryEnabled {
t.Errorf("%v: expected \"%v\", but \"%v\" was returned", test.title, test.canaryEnabled, canaryConfig.Enabled)
}
if canaryConfig.Weight != test.canaryWeight {
t.Errorf("%v: expected \"%v\", but \"%v\" was returned", test.title, test.canaryWeight, canaryConfig.Weight)
}
if canaryConfig.Header != test.canaryHeader {
t.Errorf("%v: expected \"%v\", but \"%v\" was returned", test.title, test.canaryHeader, canaryConfig.Header)
}
if canaryConfig.Cookie != test.canaryCookie {
t.Errorf("%v: expected \"%v\", but \"%v\" was returned", test.title, test.canaryCookie, canaryConfig.Cookie)
}
}
}
Loading

0 comments on commit 17cad51

Please sign in to comment.