From 991f95f6bfeac41514909c2d01e6bc908c626094 Mon Sep 17 00:00:00 2001 From: Manuel Alejandro de Brito Fontes Date: Sun, 23 Jun 2019 22:29:11 -0400 Subject: [PATCH 1/4] Migrate to openresty --- Makefile | 2 +- internal/ingress/controller/util.go | 2 +- rootfs/Dockerfile | 21 ++++++++++++++------- rootfs/etc/nginx/template/nginx.tmpl | 4 ++-- 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 184c311a89..3508557c7f 100644 --- a/Makefile +++ b/Makefile @@ -70,7 +70,7 @@ IMAGE = $(REGISTRY)/$(IMGNAME) MULTI_ARCH_IMG = $(IMAGE)-$(ARCH) # Set default base image dynamically for each arch -BASEIMAGE?=quay.io/kubernetes-ingress-controller/nginx-$(ARCH):0.87 +BASEIMAGE?=quay.io/kubernetes-ingress-controller/nginx-$(ARCH):0.88 ifeq ($(ARCH),arm64) QEMUARCH=aarch64 diff --git a/internal/ingress/controller/util.go b/internal/ingress/controller/util.go index bfbb338fe1..b19145c4b3 100644 --- a/internal/ingress/controller/util.go +++ b/internal/ingress/controller/util.go @@ -73,7 +73,7 @@ func rlimitMaxNumFiles() int { } const ( - defBinary = "/usr/sbin/nginx" + defBinary = "/usr/local/openresty/nginx/sbin/nginx" cfgPath = "/etc/nginx/nginx.conf" ) diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile index d6d522fb24..717558927c 100644 --- a/rootfs/Dockerfile +++ b/rootfs/Dockerfile @@ -24,6 +24,13 @@ RUN clean-install \ COPY --chown=www-data:www-data . / +RUN cp /usr/local/openresty/nginx/conf/mime.types /etc/nginx/mime.types +RUN ln -s /usr/local/openresty/nginx/modules /etc/nginx/modules + +# Add LuaRocks paths +ENV LUA_PATH="/usr/local/openresty/site/lualib/?.ljbc;/usr/local/openresty/site/lualib/?/init.ljbc;/usr/local/openresty/lualib/?.ljbc;/usr/local/openresty/lualib/?/init.ljbc;/usr/local/openresty/site/lualib/?.lua;/usr/local/openresty/site/lualib/?/init.lua;/usr/local/openresty/lualib/?.lua;/usr/local/openresty/lualib/?/init.lua;./?.lua;/usr/local/openresty/luajit/share/luajit-2.1.0-beta3/?.lua;/usr/local/share/lua/5.1/?.lua;/usr/local/share/lua/5.1/?/init.lua;/usr/local/openresty/luajit/share/lua/5.1/?.lua;/usr/local/openresty/luajit/share/lua/5.1/?/init.lua;/usr/local/lib/lua/?.lua;/etc/nginx/lua/?.lua;/etc/nginx/lua/vendor/?.lua;;" +ENV LUA_CPATH="/etc/nginx/lua/?.lua;/etc/nginx/lua/vendor/?.lua;/usr/local/openresty/site/lualib/?.so;/usr/local/openresty/lualib/?.so;./?.so;/usr/local/lib/lua/5.1/?.so;/usr/local/openresty/luajit/lib/lua/5.1/?.so;/usr/local/lib/lua/5.1/loadall.so;/usr/local/openresty/luajit/lib/lua/5.1/?.so;;" + # Fix permission during the build to avoid issues at runtime # with volumes (custom templates) RUN bash -eu -c ' \ @@ -42,16 +49,16 @@ RUN bash -eu -c ' \ RUN setcap cap_net_bind_service=+ep /nginx-ingress-controller \ && setcap -v cap_net_bind_service=+ep /nginx-ingress-controller -RUN setcap cap_net_bind_service=+ep /usr/sbin/nginx \ - && setcap -v cap_net_bind_service=+ep /usr/sbin/nginx - -# Create symlinks to redirect nginx logs to stdout and stderr docker log collector -# This only works if nginx is started with CMD or ENTRYPOINT -RUN ln -sf /dev/stdout /var/log/nginx/access.log -RUN ln -sf /dev/stderr /var/log/nginx/error.log +RUN setcap cap_net_bind_service=+ep /usr/local/openresty/nginx/sbin/nginx \ + && setcap -v cap_net_bind_service=+ep /usr/local/openresty/nginx/sbin/nginx USER www-data +# Create symlinks to redirect nginx logs to stdout and stderr docker log collector +RUN ln -sf /dev/stdout /usr/local/openresty/nginx/logs/access.log \ + && ln -sf /dev/stderr /usr/local/openresty/nginx/logs/error.log \ + && ln -s /usr/local/openresty/nginx/logs/* /var/log/nginx + ENTRYPOINT ["/usr/bin/dumb-init", "--"] CMD ["/nginx-ingress-controller"] diff --git a/rootfs/etc/nginx/template/nginx.tmpl b/rootfs/etc/nginx/template/nginx.tmpl index f012957af1..b3cf12f9b6 100755 --- a/rootfs/etc/nginx/template/nginx.tmpl +++ b/rootfs/etc/nginx/template/nginx.tmpl @@ -48,8 +48,8 @@ events { } http { - lua_package_cpath "/usr/local/lib/lua/?.so;/usr/lib/lua-platform-path/lua/5.1/?.so;;"; - lua_package_path "/etc/nginx/lua/?.lua;/etc/nginx/lua/vendor/?.lua;/usr/local/lib/lua/?.lua;;"; + lua_package_path "/usr/local/openresty/site/lualib/?.ljbc;/usr/local/openresty/site/lualib/?/init.ljbc;/usr/local/openresty/lualib/?.ljbc;/usr/local/openresty/lualib/?/init.ljbc;/usr/local/openresty/site/lualib/?.lua;/usr/local/openresty/site/lualib/?/init.lua;/usr/local/openresty/lualib/?.lua;/usr/local/openresty/lualib/?/init.lua;./?.lua;/usr/local/openresty/luajit/share/luajit-2.1.0-beta3/?.lua;/usr/local/share/lua/5.1/?.lua;/usr/local/share/lua/5.1/?/init.lua;/usr/local/openresty/luajit/share/lua/5.1/?.lua;/usr/local/openresty/luajit/share/lua/5.1/?/init.lua;/usr/local/lib/lua/?.lua;;"; + lua_package_cpath "/usr/local/openresty/site/lualib/?.so;/usr/local/openresty/lualib/?.so;./?.so;/usr/local/lib/lua/5.1/?.so;/usr/local/openresty/luajit/lib/lua/5.1/?.so;/usr/local/lib/lua/5.1/loadall.so;/usr/local/openresty/luajit/lib/lua/5.1/?.so;;"; {{ buildLuaSharedDictionaries $servers $all.Cfg.DisableLuaRestyWAF }} From 73be06960e250b951c1ffd2e5d565e71cd1b75a4 Mon Sep 17 00:00:00 2001 From: Manuel Alejandro de Brito Fontes Date: Sun, 23 Jun 2019 22:33:26 -0400 Subject: [PATCH 2/4] Update test image and binaries --- build/run-in-docker.sh | 2 +- images/e2e/Dockerfile | 7 +++---- images/e2e/Makefile | 2 +- test/e2e-image/Dockerfile | 4 ++-- test/e2e-prow/Dockerfile | 2 +- test/e2e-prow/Makefile | 2 +- 6 files changed, 9 insertions(+), 10 deletions(-) diff --git a/build/run-in-docker.sh b/build/run-in-docker.sh index e8732b6117..c4a2cb1907 100755 --- a/build/run-in-docker.sh +++ b/build/run-in-docker.sh @@ -22,7 +22,7 @@ set -o errexit set -o nounset set -o pipefail -E2E_IMAGE=quay.io/kubernetes-ingress-controller/e2e:v06042019-0c7a34696 +E2E_IMAGE=quay.io/kubernetes-ingress-controller/e2e:v06232019-5bb168152 DOCKER_OPTS=${DOCKER_OPTS:-""} diff --git a/images/e2e/Dockerfile b/images/e2e/Dockerfile index 5501e48f2d..1d277ab581 100644 --- a/images/e2e/Dockerfile +++ b/images/e2e/Dockerfile @@ -12,7 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -FROM quay.io/kubernetes-ingress-controller/nginx-amd64:0.87 +FROM quay.io/kubernetes-ingress-controller/nginx-amd64:0.88 RUN clean-install \ g++ \ @@ -21,13 +21,12 @@ RUN clean-install \ libc6-dev \ make \ wget \ - luarocks \ python \ pkg-config -ENV GOLANG_VERSION 1.12.5 +ENV GOLANG_VERSION 1.12.6 ENV GO_ARCH linux-amd64 -ENV GOLANG_SHA aea86e3c73495f205929cfebba0d63f1382c8ac59be081b6351681415f4063cf +ENV GOLANG_SHA dbcf71a3c1ea53b8d54ef1b48c85a39a6c9a935d01fc8291ff2b92028e59913c RUN set -eux; \ url="https://golang.org/dl/go${GOLANG_VERSION}.${GO_ARCH}.tar.gz"; \ diff --git a/images/e2e/Makefile b/images/e2e/Makefile index 6c303c1738..57bcf20741 100644 --- a/images/e2e/Makefile +++ b/images/e2e/Makefile @@ -23,7 +23,7 @@ all: docker-build docker-push docker-build: $(DOCKER) build \ --pull \ - --build-arg K8S_RELEASE=v1.14.1 \ + --build-arg K8S_RELEASE=v1.15.0 \ --build-arg ETCD_VERSION=v3.3.12 \ -t $(IMAGE):$(TAG) . diff --git a/test/e2e-image/Dockerfile b/test/e2e-image/Dockerfile index cc7745ee7e..3c72f6d230 100644 --- a/test/e2e-image/Dockerfile +++ b/test/e2e-image/Dockerfile @@ -1,4 +1,4 @@ -FROM quay.io/kubernetes-ingress-controller/e2e:v06042019-0c7a34696 AS BASE +FROM quay.io/kubernetes-ingress-controller/e2e:v06232019-5bb168152 AS BASE FROM quay.io/kubernetes-ingress-controller/debian-base-amd64:0.1 @@ -9,7 +9,7 @@ RUN clean-install \ tzdata RUN curl -Lo /usr/local/bin/kubectl \ - https://storage.googleapis.com/kubernetes-release/release/v1.14.1/bin/linux/amd64/kubectl \ + https://storage.googleapis.com/kubernetes-release/release/v1.15.0/bin/linux/amd64/kubectl \ && chmod +x /usr/local/bin/kubectl COPY --from=BASE /go/bin/ginkgo /usr/local/bin/ diff --git a/test/e2e-prow/Dockerfile b/test/e2e-prow/Dockerfile index ffa6963676..b58da35fd9 100644 --- a/test/e2e-prow/Dockerfile +++ b/test/e2e-prow/Dockerfile @@ -56,7 +56,7 @@ RUN curl -L https://storage.googleapis.com/etcd/${ETCD_VERSION}/etcd-${ETCD_VERS && rm -rf /tmp/etcd-download # install go -ENV GO_VERSION 1.12.5 +ENV GO_VERSION 1.12.6 ENV GO_TARBALL "go${GO_VERSION}.linux-amd64.tar.gz" RUN wget -q "https://storage.googleapis.com/golang/${GO_TARBALL}" && \ tar xzf "${GO_TARBALL}" -C /usr/local && \ diff --git a/test/e2e-prow/Makefile b/test/e2e-prow/Makefile index a072e70140..7ce8df09b6 100644 --- a/test/e2e-prow/Makefile +++ b/test/e2e-prow/Makefile @@ -9,7 +9,7 @@ all: docker-build docker-push docker-build: $(DOCKER) build \ --pull \ - --build-arg K8S_RELEASE=v1.14.1 \ + --build-arg K8S_RELEASE=v1.15.0 \ --build-arg ETCD_VERSION=v3.3.12 \ -t $(IMAGE):$(TAG) . From ca6b61f639361cba725535747cdf854f2322d2b8 Mon Sep 17 00:00:00 2001 From: Manuel Alejandro de Brito Fontes Date: Sun, 23 Jun 2019 22:33:40 -0400 Subject: [PATCH 3/4] Update e2e tests for openresty --- test/e2e/annotations/redirect.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/e2e/annotations/redirect.go b/test/e2e/annotations/redirect.go index 7e6171ba8e..a165a7e5a6 100644 --- a/test/e2e/annotations/redirect.go +++ b/test/e2e/annotations/redirect.go @@ -72,7 +72,7 @@ var _ = framework.IngressNginxDescribe("Annotations - Redirect", func() { Expect(errs).To(BeNil()) Expect(resp.StatusCode).Should(BeNumerically("==", http.StatusMovedPermanently)) Expect(resp.Header.Get("Location")).Should(Equal(redirectURL)) - Expect(body).Should(ContainSubstring("nginx/")) + Expect(body).Should(ContainSubstring("openresty/")) }) It("should respond with a custom redirect code", func() { @@ -108,6 +108,6 @@ var _ = framework.IngressNginxDescribe("Annotations - Redirect", func() { Expect(errs).To(BeNil()) Expect(resp.StatusCode).Should(BeNumerically("==", redirectCode)) Expect(resp.Header.Get("Location")).Should(Equal(redirectURL)) - Expect(body).Should(ContainSubstring("nginx/")) + Expect(body).Should(ContainSubstring("openresty/")) }) }) From dca5557c50ffcbf068a52cebf3e7c776f253d32f Mon Sep 17 00:00:00 2001 From: Manuel Alejandro de Brito Fontes Date: Sun, 23 Jun 2019 22:34:07 -0400 Subject: [PATCH 4/4] In case of errors running e2e tests, print the generated nginx.conf file --- test/e2e/framework/framework.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/e2e/framework/framework.go b/test/e2e/framework/framework.go index cc784497da..ca8aaf9ed4 100644 --- a/test/e2e/framework/framework.go +++ b/test/e2e/framework/framework.go @@ -128,6 +128,20 @@ func (f *Framework) AfterEach() { Expect(err).ToNot(HaveOccurred()) By("Dumping NGINX logs after a failure running a test") Logf("%v", log) + + pod, err := getIngressNGINXPod(f.Namespace, f.KubeClientSet) + if err != nil { + return + } + + cmd := fmt.Sprintf("cat /etc/nginx/nginx.conf") + o, err := f.ExecCommand(pod, cmd) + if err != nil { + return + } + + By("Dumping NGINX configuration after a failure running a test") + Logf("%v", o) } }