diff --git a/Makefile b/Makefile index 45b1577d26..2035be0f3d 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,8 @@ E2E_NODES ?= 10 SLOW_E2E_THRESHOLD ?= 50 K8S_VERSION ?= v1.14.1 +E2E_CHECK_LEAKS ?= + ifeq ($(GOHOSTOS),darwin) SED_I=sed -i '' endif @@ -70,6 +72,9 @@ export GOBUILD_FLAGS export REPO_INFO export BUSTED_ARGS export IMAGE +export E2E_NODES +export E2E_CHECK_LEAKS +export SLOW_E2E_THRESHOLD # Set default base image dynamically for each arch BASEIMAGE?=quay.io/kubernetes-ingress-controller/nginx-$(ARCH):0.90 @@ -174,28 +179,7 @@ lua-test: .PHONY: e2e-test e2e-test: - echo "Granting permissions to ingress-nginx e2e service account..." - kubectl create serviceaccount ingress-nginx-e2e || true - kubectl create clusterrolebinding permissive-binding \ - --clusterrole=cluster-admin \ - --user=admin \ - --user=kubelet \ - --serviceaccount=default:ingress-nginx-e2e || true - - until kubectl get secret | grep -q ^ingress-nginx-e2e-token; do \ - echo "waiting for api token"; \ - sleep 3; \ - done - - kubectl run --rm \ - --attach \ - --restart=Never \ - --generator=run-pod/v1 \ - --env="E2E_NODES=$(E2E_NODES)" \ - --env="FOCUS=$(FOCUS)" \ - --env="SLOW_E2E_THRESHOLD=$(SLOW_E2E_THRESHOLD)" \ - --overrides='{ "apiVersion": "v1", "spec":{"serviceAccountName": "ingress-nginx-e2e"}}' \ - e2e --image=nginx-ingress-controller:e2e + @build/run-e2e-suite.sh .PHONY: e2e-test-image e2e-test-image: e2e-test-binary diff --git a/build/run-e2e-suite.sh b/build/run-e2e-suite.sh new file mode 100755 index 0000000000..d8662673b1 --- /dev/null +++ b/build/run-e2e-suite.sh @@ -0,0 +1,80 @@ +#!/bin/bash + +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if ! [ -z "$DEBUG" ]; then + set -x +fi + +set -o errexit +set -o nounset +set -o pipefail + +RED='\e[35m' +NC='\e[0m' +BGREEN='\e[32m' + +declare -a mandatory +mandatory=( + E2E_NODES + SLOW_E2E_THRESHOLD +) + +missing=false +for var in "${mandatory[@]}"; do + if [[ -z "${!var:-}" ]]; then + echo -e "${RED}Environment variable $var must be set${NC}" + missing=true + fi +done + +if [ "$missing" = true ]; then + exit 1 +fi + +function cleanup { + kubectl delete pod e2e 2>/dev/null || true +} +trap cleanup EXIT + +E2E_CHECK_LEAKS=${E2E_CHECK_LEAKS:-} +FOCUS=${FOCUS:-.*} + +export E2E_CHECK_LEAKS +export FOCUS + +echo -e "${BGREEN}Granting permissions to ingress-nginx e2e service account...${NC}" +kubectl create serviceaccount ingress-nginx-e2e || true +kubectl create clusterrolebinding permissive-binding \ + --clusterrole=cluster-admin \ + --user=admin \ + --user=kubelet \ + --serviceaccount=default:ingress-nginx-e2e || true + +until kubectl get secret | grep -q ^ingress-nginx-e2e-token; do \ + echo -e "waiting for api token"; \ + sleep 3; \ +done + +kubectl run --rm \ + --attach \ + --restart=Never \ + --generator=run-pod/v1 \ + --env="E2E_NODES=${E2E_NODES}" \ + --env="FOCUS=${FOCUS}" \ + --env="E2E_CHECK_LEAKS=${E2E_CHECK_LEAKS}" \ + --env="SLOW_E2E_THRESHOLD=${SLOW_E2E_THRESHOLD}" \ + --overrides='{ "apiVersion": "v1", "spec":{"serviceAccountName": "ingress-nginx-e2e"}}' \ + e2e --image=nginx-ingress-controller:e2e diff --git a/rootfs/etc/nginx/template/nginx.tmpl b/rootfs/etc/nginx/template/nginx.tmpl index 6d66bb3294..a750a645c0 100755 --- a/rootfs/etc/nginx/template/nginx.tmpl +++ b/rootfs/etc/nginx/template/nginx.tmpl @@ -54,7 +54,6 @@ http { {{ buildLuaSharedDictionaries $servers $all.Cfg.DisableLuaRestyWAF }} init_by_lua_block { - require("resty.core") collectgarbage("collect") {{ if not $all.Cfg.DisableLuaRestyWAF }} @@ -632,7 +631,6 @@ stream { lua_shared_dict tcp_udp_configuration_data 5M; init_by_lua_block { - require("resty.core") collectgarbage("collect") -- init modules diff --git a/test/e2e-image/e2e.sh b/test/e2e-image/e2e.sh index 3178c8a3b6..9a95b25f61 100755 --- a/test/e2e-image/e2e.sh +++ b/test/e2e-image/e2e.sh @@ -16,36 +16,50 @@ set -e +NC='\e[0m' +BGREEN='\e[32m' + SLOW_E2E_THRESHOLD=${SLOW_E2E_THRESHOLD:-50} FOCUS=${FOCUS:-.*} E2E_NODES=${E2E_NODES:-5} +E2E_CHECK_LEAKS=${E2E_CHECK_LEAKS:-""} -if [ ! -f ${HOME}/.kube/config ]; then - kubectl config set-cluster dev --certificate-authority=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt --embed-certs=true --server="https://kubernetes.default/" - kubectl config set-credentials user --token="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" - kubectl config set-context default --cluster=dev --user=user - kubectl config use-context default +if [ ! -f "${HOME}/.kube/config" ]; then + kubectl config set-cluster dev --certificate-authority=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt --embed-certs=true --server="https://kubernetes.default/" + kubectl config set-credentials user --token="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" + kubectl config set-context default --cluster=dev --user=user + kubectl config use-context default fi ginkgo_args=( - "-randomizeSuites" - "-randomizeAllSpecs" - "-flakeAttempts=2" - "-p" - "-trace" - "--noColor=true" - "-slowSpecThreshold=${SLOW_E2E_THRESHOLD}" + "-randomizeSuites" + "-randomizeAllSpecs" + "-flakeAttempts=2" + "-p" + "-trace" + "-slowSpecThreshold=${SLOW_E2E_THRESHOLD}" + "-r" ) -echo "Running e2e test suite..." -ginkgo "${ginkgo_args[@]}" \ - -focus=${FOCUS} \ - -skip="\[Serial\]" \ - -nodes=${E2E_NODES} \ - /e2e.test +echo -e "${BGREEN}Running e2e test suite (FOCUS=${FOCUS})...${NC}" +ginkgo "${ginkgo_args[@]}" \ + -focus="${FOCUS}" \ + -skip="\[Serial\]|\[MemoryLeak\]" \ + -nodes="${E2E_NODES}" \ + /e2e.test -echo "Running e2e test suite with tests that require serial execution..." -ginkgo "${ginkgo_args[@]}" \ - -focus="\[Serial\]" \ - -nodes=1 \ +echo -e "${BGREEN}Running e2e test suite with tests that require serial execution...${NC}" +ginkgo "${ginkgo_args[@]}" \ + -focus="\[Serial\]" \ + -skip="\[MemoryLeak\]" \ + -nodes=1 \ + /e2e.test + +if [[ ${E2E_CHECK_LEAKS} != "" ]]; then + echo -e "${BGREEN}Running e2e test suite with tests that check for memory leaks...${NC}" + ginkgo "${ginkgo_args[@]}" \ + -focus="\[MemoryLeak\]" \ + -skip="\[Serial\]" \ + -nodes=1 \ /e2e.test +fi diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go index 54caf7568e..e06876c50e 100644 --- a/test/e2e/e2e.go +++ b/test/e2e/e2e.go @@ -35,6 +35,7 @@ import ( _ "k8s.io/ingress-nginx/test/e2e/dbg" _ "k8s.io/ingress-nginx/test/e2e/defaultbackend" _ "k8s.io/ingress-nginx/test/e2e/gracefulshutdown" + _ "k8s.io/ingress-nginx/test/e2e/leaks" _ "k8s.io/ingress-nginx/test/e2e/loadbalance" _ "k8s.io/ingress-nginx/test/e2e/lua" _ "k8s.io/ingress-nginx/test/e2e/servicebackend" diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index ca8aaf9ed4..0291d5d7b0 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -147,7 +147,12 @@ func (f *Framework) AfterEach() { // IngressNginxDescribe wrapper function for ginkgo describe. Adds namespacing. func IngressNginxDescribe(text string, body func()) bool { - return Describe("[nginx-ingress] "+text, body) + return Describe("[ingress-nginx] "+text, body) +} + +// MemoryLeakIt is wrapper function for ginkgo It. Adds "[MemoryLeak]" tag and makes static analysis easier. +func MemoryLeakIt(text string, body interface{}, timeout ...float64) bool { + return It(text+" [MemoryLeak]", body, timeout...) } // GetNginxIP returns the number of TCP port where NGINX is running diff --git a/test/e2e/leaks/lua_ssl.go b/test/e2e/leaks/lua_ssl.go new file mode 100644 index 0000000000..ee95d4ef3f --- /dev/null +++ b/test/e2e/leaks/lua_ssl.go @@ -0,0 +1,130 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package leaks + +import ( + "crypto/tls" + "fmt" + "net/http" + "strings" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/parnurzeal/gorequest" + pool "gopkg.in/go-playground/pool.v3" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "k8s.io/ingress-nginx/test/e2e/framework" +) + +var _ = framework.IngressNginxDescribe("DynamicCertificates", func() { + f := framework.NewDefaultFramework("lua-dynamic-certificates") + + BeforeEach(func() { + f.NewEchoDeployment() + }) + + AfterEach(func() { + }) + + framework.MemoryLeakIt("should not leak memory from ingress SSL certificates or configuration updates", func() { + hostCount := 1000 + iterations := 10 + + By("Waiting a minute before starting the test") + time.Sleep(1 * time.Minute) + + for iteration := 1; iteration <= iterations; iteration++ { + By(fmt.Sprintf("Running iteration %v", iteration)) + + p := pool.NewLimited(200) + + batch := p.Batch() + + for index := 1; index <= hostCount; index++ { + host := fmt.Sprintf("hostname-%v", index) + batch.Queue(run(host, f)) + } + + batch.QueueComplete() + batch.WaitAll() + + p.Close() + + By("waiting one minute before next iteration") + time.Sleep(1 * time.Minute) + } + }) +}) + +func privisionIngress(hostname string, f *framework.Framework) { + ing := f.EnsureIngress(framework.NewSingleIngressWithTLS(hostname, "/", hostname, []string{hostname}, f.Namespace, "http-svc", 80, nil)) + _, err := framework.CreateIngressTLSSecret(f.KubeClientSet, + ing.Spec.TLS[0].Hosts, + ing.Spec.TLS[0].SecretName, + ing.Namespace) + Expect(err).NotTo(HaveOccurred()) + + f.WaitForNginxServer(hostname, + func(server string) bool { + return strings.Contains(server, fmt.Sprintf("server_name %v", hostname)) && + strings.Contains(server, "listen 443") + }) +} + +func checkIngress(hostname string, f *framework.Framework) { + req := gorequest.New() + resp, _, errs := req. + Get(f.GetURL(framework.HTTPS)). + TLSClientConfig(&tls.Config{ServerName: hostname, InsecureSkipVerify: true}). + Set("Host", hostname). + End() + Expect(errs).Should(BeEmpty()) + Expect(resp.StatusCode).Should(Equal(http.StatusOK)) + + // check the returned secret is not the fake one + cert := resp.TLS.PeerCertificates[0] + Expect(cert.DNSNames[0]).Should(Equal(hostname)) +} + +func deleteIngress(hostname string, f *framework.Framework) { + err := f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.Namespace).Delete(hostname, &metav1.DeleteOptions{}) + Expect(err).NotTo(HaveOccurred(), "unexpected error deleting ingress") +} + +func run(host string, f *framework.Framework) pool.WorkFunc { + return func(wu pool.WorkUnit) (interface{}, error) { + if wu.IsCancelled() { + return nil, nil + } + + By(fmt.Sprintf("\tcreating ingress for host %v", host)) + privisionIngress(host, f) + + time.Sleep(100 * time.Millisecond) + + By(fmt.Sprintf("\tchecking ingress for host %v", host)) + checkIngress(host, f) + + By(fmt.Sprintf("\tdestroying ingress for host %v", host)) + deleteIngress(host, f) + + return true, nil + } +}