From cbf041fc3e1050e58f829af10e2312aae87292d7 Mon Sep 17 00:00:00 2001 From: Henry Tran Date: Tue, 5 Jun 2018 09:51:22 -0400 Subject: [PATCH] Add Lua module to serve SSL Certificates dynamically --- internal/ingress/controller/nginx.go | 2 +- rootfs/etc/nginx/lua/certificate.lua | 54 +++ .../etc/nginx/lua/test/certificate_test.lua | 94 ++++ rootfs/etc/nginx/template/nginx.tmpl | 15 + test/e2e/lua/dynamic_certificates.go | 413 ++++++++++++++++++ test/e2e/lua/dynamic_configuration.go | 2 +- 6 files changed, 578 insertions(+), 2 deletions(-) create mode 100644 rootfs/etc/nginx/lua/certificate.lua create mode 100644 rootfs/etc/nginx/lua/test/certificate_test.lua create mode 100644 test/e2e/lua/dynamic_certificates.go diff --git a/internal/ingress/controller/nginx.go b/internal/ingress/controller/nginx.go index 18c40eccd3..22ad3d8e77 100644 --- a/internal/ingress/controller/nginx.go +++ b/internal/ingress/controller/nginx.go @@ -731,7 +731,7 @@ func clearCertificates(config *ingress.Configuration) { var clearedServers []*ingress.Server for _, server := range config.Servers { copyOfServer := *server - copyOfServer.SSLCert = ingress.SSLCert{} + copyOfServer.SSLCert = ingress.SSLCert{PemFileName: copyOfServer.SSLCert.PemFileName} clearedServers = append(clearedServers, ©OfServer) } config.Servers = clearedServers diff --git a/rootfs/etc/nginx/lua/certificate.lua b/rootfs/etc/nginx/lua/certificate.lua new file mode 100644 index 0000000000..7904a8d330 --- /dev/null +++ b/rootfs/etc/nginx/lua/certificate.lua @@ -0,0 +1,54 @@ +local ssl = require("ngx.ssl") +local configuration = require("configuration") + +local _M = {} + +local function set_pem_cert_key(pem_cert_key) + local der_cert, der_cert_err = ssl.cert_pem_to_der(pem_cert_key) + if not der_cert then + return "failed to convert certificate chain from PEM to DER: " .. der_cert_err + end + + local set_cert_ok, set_cert_err = ssl.set_der_cert(der_cert) + if not set_cert_ok then + return "failed to set DER cert: " .. set_cert_err + end + + local der_priv_key, dev_priv_key_err = ssl.priv_key_pem_to_der(pem_cert_key) + if not der_priv_key then + return "failed to convert private key from PEM to DER: " .. dev_priv_key_err + end + + local set_priv_key_ok, set_priv_key_err = ssl.set_der_priv_key(der_priv_key) + if not set_priv_key_ok then + return "failed to set DER private key: " .. set_priv_key_err + end +end + +function _M.call() + local hostname, hostname_err = ssl.server_name() + if hostname_err then + ngx.log(ngx.ERR, "Error getting the hostname, falling back on default certificate: " .. hostname_err) + return + end + + local pem_cert_key = configuration.get_pem_cert_key(hostname) + if not pem_cert_key or pem_cert_key == "" then + ngx.log(ngx.ERR, "Certificate not found, falling back on default certificate for hostname: " .. tostring(hostname)) + return + end + + local clear_ok, clear_err = ssl.clear_certs() + if not clear_ok then + ngx.log(ngx.ERR, "failed to clear existing (fallback) certificates: " .. clear_err) + return ngx.exit(ngx.ERROR) + end + + local set_pem_cert_key_err = set_pem_cert_key(pem_cert_key) + if set_pem_cert_key_err then + ngx.log(ngx.ERR, set_pem_cert_key_err) + return ngx.exit(ngx.ERROR) + end +end + +return _M diff --git a/rootfs/etc/nginx/lua/test/certificate_test.lua b/rootfs/etc/nginx/lua/test/certificate_test.lua new file mode 100644 index 0000000000..413854d069 --- /dev/null +++ b/rootfs/etc/nginx/lua/test/certificate_test.lua @@ -0,0 +1,94 @@ +local certificate = require("certificate") + +local unmocked_ngx = _G.ngx + +describe("Certificate", function() + describe("call", function() + local ssl = require("ngx.ssl") + local match = require("luassert.match") + + ssl.server_name = function() return "hostname", nil end + ssl.clear_certs = function() return true, "" end + ssl.set_der_cert = function(cert) return true, "" end + ssl.set_der_priv_key = function(priv_key) return true, "" end + + before_each(function() + ngx.exit = function(status) end + end) + + after_each(function() + ngx = unmocked_ngx + end) + + it("does not clear fallback certificates and logs error message when host is not in dictionary", function() + ngx.shared.certificate_data:set("hostname", "") + + spy.on(ngx, "log") + spy.on(ssl, "clear_certs") + spy.on(ssl, "set_der_cert") + spy.on(ssl, "set_der_priv_key") + + assert.has_no.errors(certificate.call) + assert.spy(ngx.log).was_called_with(ngx.ERR, "Certificate not found, falling back on default certificate for hostname: hostname") + assert.spy(ssl.clear_certs).was_not_called() + assert.spy(ssl.set_der_cert).was_not_called() + assert.spy(ssl.set_der_priv_key).was_not_called() + end) + + it("does not clear fallback certificates and logs error message when the cert is empty for given host", function() + spy.on(ngx, "log") + spy.on(ssl, "clear_certs") + spy.on(ssl, "set_der_cert") + spy.on(ssl, "set_der_priv_key") + + assert.has_no.errors(certificate.call) + assert.spy(ngx.log).was_called_with(ngx.ERR, "Certificate not found, falling back on default certificate for hostname: hostname") + assert.spy(ssl.clear_certs).was_not_called() + assert.spy(ssl.set_der_cert).was_not_called() + assert.spy(ssl.set_der_priv_key).was_not_called() + end) + + it("successfully sets SSL certificate and key when hostname is found in dictionary", function() + local _ = match._ + local pem_cert_key = "-----BEGIN CERTIFICATE-----\nMIID6DCCAlCgAwIBAgIQcfG0mA7BIFqhlnr/Zwh6TzANBgkqhkiG9w0BAQsFADBC\nMR4wHAYDVQQKExVta2NlcnQgZGV2ZWxvcG1lbnQgQ0ExIDAeBgNVBAsMF2hlbnJ5\ndHJhbkBPVFQtSGVucnlUcmFuMB4XDTE4MDcwOTIwMTY1MloXDTI4MDcwOTIwMTY1\nMlowSzEnMCUGA1UEChMebWtjZXJ0IGRldmVsb3BtZW50IGNlcnRpZmljYXRlMSAw\nHgYDVQQLDBdoZW5yeXRyYW5AT1RULUhlbnJ5VHJhbjCCASIwDQYJKoZIhvcNAQEB\nBQADggEPADCCAQoCggEBALIrsgHzjZZyKWPn3rGzTkaj9jADYAMhM+0wY3iky2Dx\ndr2YbKnZbbGxKLfVukYRsUUOK0SnBMTX15fsGanirj2hflMHfGvHilaVkVAkPJgD\nBTf2PkxFff99hS7/Ncz20MR6+E/vqp7Hx7IKDrg9lC9u1n82aotfN3gPhif8HyQu\n+P9cltsr9PewyPe4573WQmzXhTKaFm9+U9xZ2qS1J0DMEizRs45vuM040hxtiwVz\nM4Lm8DVpaYxMBWNI/zo9EZzoSJZH1sYUpTMwhNj+caEX+LK9PCM4Sht/yhPUc6aD\nnIEqraz+bS8dNFH5Ehp7n1SZL7YH6xT6da4F3ci7jEECAwEAAaNRME8wDgYDVR0P\nAQH/BAQDAgWgMBMGA1UdJQQMMAoGCCsGAQUFBwMBMAwGA1UdEwEB/wQCMAAwGgYD\nVR0RBBMwEYIPbXltaW5pa3ViZS5pbmZvMA0GCSqGSIb3DQEBCwUAA4IBgQC/TEpx\nJkL/ek37L2XKwGq96hNT10IZ9yE+RlndNGNY6eAc8y313sXBHTFbDCWQ2s0pWZZS\n+va20dnTQNyWzJAxFpdNvcKOCUGat4RPD/j+pBTEYk5n/oo7s2FWkG8kW6tKDilf\njWHpk7m9uYCO2sOZFiQPR81idR5PLox46SpJmIhDVfCi6VS4N+8fAT8Tbt9xkPmS\nmODhpnuIUt0NVTi62eqnxeO185qAt73xhz9Gj1KHntAK1ebcx0k3UxKRXQp9WY76\nF39sSz8OuhEhvv9ayl6uS6ZdSLvvb6kJrRRneKr01ridCOtiYB7cuXykDL1c6PUk\nugxDgTyCjiuPnRl0CLwxWT659PVozA2SO1YCW6UcoGdj2KMvsXezeWKpNGx3NHXO\nufdlxSbzWlamn+sPunWP3v2tfV0J8sHG3n1roeBO2N52197/ennGuCZfnF8C5MoG\n9YfMjKg9Z03G8sDpk9g5bHp9p28TO1X+Ht30PQzkUNhx3fjTO2DDvCyGk2k=\n-----END CERTIFICATE-----\n\n-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCyK7IB842Wcilj\n596xs05Go/YwA2ADITPtMGN4pMtg8Xa9mGyp2W2xsSi31bpGEbFFDitEpwTE19eX\n7Bmp4q49oX5TB3xrx4pWlZFQJDyYAwU39j5MRX3/fYUu/zXM9tDEevhP76qex8ey\nCg64PZQvbtZ/NmqLXzd4D4Yn/B8kLvj/XJbbK/T3sMj3uOe91kJs14UymhZvflPc\nWdqktSdAzBIs0bOOb7jNONIcbYsFczOC5vA1aWmMTAVjSP86PRGc6EiWR9bGFKUz\nMITY/nGhF/iyvTwjOEobf8oT1HOmg5yBKq2s/m0vHTRR+RIae59UmS+2B+sU+nWu\nBd3Iu4xBAgMBAAECggEBAIbuUIDp0fB9xJrEnwI0qLMWuPrjk3LLUmfunWZgZyWj\nuCkdpi17XHeVkyCl28v02itR77KuSg5I6B1F0Km34f0KsIBwyulU1I999e6bgsgc\ngXdAJS3d8u3qQVK2NChlQvWJq0PeXXiiE7nhpAQjnnXNmuP8cfPayEdEenUNmwfq\nxjEh2/oDzUTPD/4z5Hpw8n728SItgBolMNgGvmv5cC4JNLqujCUPwkLiZ3a2YbTY\nrmOO1xDkZnQWqyNP+baOwYwpu/kISPM3IveP5GGBNQsUsDxs6t80HNW/w8Ry0f50\n+gNTIuJVOLXfpVLIo87wTMEtRAqMzT4vxQIi+vj2XYECgYEA6QftSRKqur1G4P6Y\n9cseDnljJFWIjqex2q3NrvMaHbnlXp6AtPROoNz6L8H+PnBy8o1yZJwWnhKTvPaD\nsi+a1g7dqQIM4TjKLlidV57lt5ENw4ueW1o7Jbk75gawLhrBPSCFxR5xSqF+kQxn\nmWGjLnZoomD6fM2CG7EF1fg+wjsCgYEAw7t6db9tjGGM9J0rUe+jVtxMWiG9ArT0\nhmaLZQlKrOSFeEf8c4ZYBNxp/X+/jg0GWBX8P4KRubAz6bbn0A+07K9TClrvMFWq\nveqnK1JUsMGWsYQPp8dX8VS/jOFzYdiji9Ekyzs9RiXW8jzp0wzrccSjEr0de8HK\niEa9CH7cZ7MCgYB1V5mD51NzbyZG281YT+yVq0hiHnQCKa1kiYp+I0ouV9KJP9Vd\nyXvigwO0ksIc3PD09Ib65KJ6/K3KRHPygQg97ARwO2kS7E7a4aJxYcEZG4DLy/10\n0M3h5BGmdg23WZ+e0UarCPZRd1rNXWq5kLHkDpoH0j+wIqf2m8Bti3DGywKBgBEK\nn6zkz9rrG2So0n69yJDleVhXm6dCrg+NmhFf77qB4wUH73j3d25k6m2B0+HATI8a\nyu2Upq9uIfb1T9WTqIL6+NXr+OtSah1C8u8YqfsBv+cQwnQvLP78C/luH6ejPwoL\nWZLAQ6N54+8PUqRneZBcOH6HLKv7wXCACDFXKkV1AoGAfDb5GJ0NsovWhLfU5WEB\nSfdzHBplbp72q08S0aqTNm0wlTiCGYgm2Lle4IaOGoJ+7ipirL0KzuwisAZJFTvJ\nhsMqOmH/Ckledmf2JpLxyg8KB5KVA+RVQkrfVEv8yhqLcKQU6Z2n4jSTon7hXb1T\nf8neDpZ8DwO0W9cOdYLYyTg=\n-----END PRIVATE KEY-----" + ngx.shared.certificate_data:set("hostname", pem_cert_key) + + spy.on(ngx, "log") + spy.on(ssl, "set_der_cert") + spy.on(ssl, "set_der_priv_key") + + assert.has_no.errors(certificate.call) + assert.spy(ngx.log).was_not_called_with(ngx.ERR, _) + assert.spy(ssl.set_der_cert).was_called_with(ssl.cert_pem_to_der(pem_cert_key)) + assert.spy(ssl.set_der_priv_key).was_called_with(ssl.priv_key_pem_to_der(pem_cert_key)) + end) + + it("logs error message when certificate in dictionary is invalid", function() + ngx.shared.certificate_data:set("hostname", "something invalid") + + spy.on(ngx, "log") + spy.on(ssl, "set_der_cert") + spy.on(ssl, "set_der_priv_key") + + assert.has_no.errors(certificate.call) + assert.spy(ngx.log).was_called_with(ngx.ERR, "failed to convert certificate chain from PEM to DER: PEM_read_bio_X509_AUX() failed") + assert.spy(ssl.set_der_cert).was_not_called() + assert.spy(ssl.set_der_priv_key).was_not_called() + end) + + it("does not clear fallback certificates and logs error message when hostname could not be fetched", function() + ssl.server_name = function() return nil, "error" end + + spy.on(ngx, "log") + spy.on(ssl, "clear_certs") + spy.on(ssl, "set_der_cert") + spy.on(ssl, "set_der_priv_key") + + assert.has_no.errors(certificate.call) + assert.spy(ngx.log).was_called_with(ngx.ERR, "Error getting the hostname, falling back on default certificate: error") + assert.spy(ssl.clear_certs).was_not_called() + assert.spy(ssl.set_der_cert).was_not_called() + assert.spy(ssl.set_der_priv_key).was_not_called() + end) + end) +end) diff --git a/rootfs/etc/nginx/template/nginx.tmpl b/rootfs/etc/nginx/template/nginx.tmpl index ba2d2df688..ac1adb9c1e 100644 --- a/rootfs/etc/nginx/template/nginx.tmpl +++ b/rootfs/etc/nginx/template/nginx.tmpl @@ -85,6 +85,15 @@ http { else monitor = res end + + {{ if $all.DynamicCertificatesEnabled }} + ok, res = pcall(require, "certificate") + if not ok then + error("require failed: " .. tostring(res)) + else + certificate = res + end + {{ end }} } {{ if $all.DynamicConfigurationEnabled }} @@ -775,6 +784,12 @@ stream { ssl_stapling on; ssl_stapling_verify on; {{ end }} + + {{ if and (not $all.DisableLua) $all.DynamicCertificatesEnabled}} + ssl_certificate_by_lua_block { + certificate.call() + } + {{ end }} {{ end }} {{ if not (empty $server.AuthTLSError) }} diff --git a/test/e2e/lua/dynamic_certificates.go b/test/e2e/lua/dynamic_certificates.go new file mode 100644 index 0000000000..a8889bf01c --- /dev/null +++ b/test/e2e/lua/dynamic_certificates.go @@ -0,0 +1,413 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package lua + +import ( + "crypto/tls" + "fmt" + "net/http" + "strings" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/parnurzeal/gorequest" + + appsv1beta1 "k8s.io/api/apps/v1beta1" + extensions "k8s.io/api/extensions/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + + "k8s.io/ingress-nginx/test/e2e/framework" +) + +var _ = framework.IngressNginxDescribe("Dynamic Certificate", func() { + f := framework.NewDefaultFramework("dynamic-certificate") + host := "foo.com" + + BeforeEach(func() { + err := enableDynamicCertificates(f.IngressController.Namespace, f.KubeClientSet) + Expect(err).NotTo(HaveOccurred()) + + err = f.WaitForNginxConfiguration( + func(cfg string) bool { + return strings.Contains(cfg, "ok, res = pcall(require, \"certificate\")") + }) + Expect(err).NotTo(HaveOccurred()) + + err = f.NewEchoDeploymentWithReplicas(1) + Expect(err).NotTo(HaveOccurred()) + }) + + It("picks up the certificate when we add TLS spec to existing ingress", func() { + ing, err := f.EnsureIngress(framework.NewSingleIngress(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil)) + Expect(err).NotTo(HaveOccurred()) + Expect(ing).NotTo(BeNil()) + time.Sleep(waitForLuaSync) + resp, _, errs := gorequest.New(). + Get(f.IngressController.HTTPURL). + Set("Host", host). + End() + Expect(len(errs)).Should(BeNumerically("==", 0)) + Expect(resp.StatusCode).Should(Equal(http.StatusOK)) + + ing, err = f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.IngressController.Namespace).Get("foo.com", metav1.GetOptions{}) + Expect(err).ToNot(HaveOccurred()) + ing.Spec.TLS = []extensions.IngressTLS{ + { + Hosts: []string{host}, + SecretName: host, + }, + } + _, err = framework.CreateIngressTLSSecret(f.KubeClientSet, + ing.Spec.TLS[0].Hosts, + ing.Spec.TLS[0].SecretName, + ing.Namespace) + Expect(err).ToNot(HaveOccurred()) + _, err = f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.IngressController.Namespace).Update(ing) + Expect(err).ToNot(HaveOccurred()) + + By("configuring HTTPS endpoint") + err = f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, "server_name "+host) && + strings.Contains(server, "listen 443") + }) + Expect(err).ToNot(HaveOccurred()) + + time.Sleep(waitForLuaSync) + + By("serving the configured certificate on HTTPS endpoint") + resp, _, errs = gorequest.New(). + Get(f.IngressController.HTTPSURL). + Set("Host", ing.Spec.TLS[0].Hosts[0]). + TLSClientConfig(&tls.Config{ + InsecureSkipVerify: true, + ServerName: ing.Spec.TLS[0].Hosts[0], + }). + End() + Expect(len(errs)).Should(BeNumerically("==", 0)) + Expect(resp.StatusCode).Should(Equal(http.StatusOK)) + Expect(len(resp.TLS.PeerCertificates)).Should(BeNumerically("==", 1)) + Expect(resp.TLS.PeerCertificates[0].DNSNames[0]).Should(Equal(host)) + }) + + It("picks up the previously missing secret for a given ingress without reloading", func() { + ing, err := f.EnsureIngress(framework.NewSingleIngressWithTLS(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil)) + Expect(err).NotTo(HaveOccurred()) + Expect(ing).NotTo(BeNil()) + time.Sleep(waitForLuaSync) + resp, _, errs := gorequest.New(). + Get(fmt.Sprintf("%s?id=dummy_log_splitter_foo_bar", f.IngressController.HTTPSURL)). + Set("Host", host). + TLSClientConfig(&tls.Config{ + InsecureSkipVerify: true, + ServerName: ing.Spec.TLS[0].Hosts[0], + }). + End() + Expect(len(errs)).Should(BeNumerically("==", 0)) + Expect(resp.StatusCode).Should(Equal(http.StatusOK)) + + _, err = framework.CreateIngressTLSSecret(f.KubeClientSet, + ing.Spec.TLS[0].Hosts, + ing.Spec.TLS[0].SecretName, + ing.Namespace) + Expect(err).ToNot(HaveOccurred()) + + By("configuring certificate_by_lua and skipping Nginx configuration of the new certificate") + err = f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, "ssl_certificate_by_lua_block") && + !strings.Contains(server, fmt.Sprintf("ssl_certificate /etc/ingress-controller/ssl/%s-%s.pem;", ing.Namespace, host)) && + !strings.Contains(server, fmt.Sprintf("ssl_certificate_key /etc/ingress-controller/ssl/%s-%s.pem;", ing.Namespace, host)) && + strings.Contains(server, "listen 443") + }) + Expect(err).ToNot(HaveOccurred()) + + time.Sleep(waitForLuaSync) + + By("serving the configured certificate on HTTPS endpoint") + resp, _, errs = gorequest.New(). + Get(f.IngressController.HTTPSURL). + Set("Host", ing.Spec.TLS[0].Hosts[0]). + TLSClientConfig(&tls.Config{ + InsecureSkipVerify: true, + ServerName: ing.Spec.TLS[0].Hosts[0], + }). + End() + Expect(len(errs)).Should(BeNumerically("==", 0)) + Expect(resp.StatusCode).Should(Equal(http.StatusOK)) + Expect(len(resp.TLS.PeerCertificates)).Should(BeNumerically("==", 1)) + Expect(resp.TLS.PeerCertificates[0].DNSNames[0]).Should(Equal(host)) + + log, err := f.NginxLogs() + Expect(err).ToNot(HaveOccurred()) + Expect(log).ToNot(BeEmpty()) + index := strings.Index(log, "id=dummy_log_splitter_foo_bar") + restOfLogs := log[index:] + + By("skipping Nginx reload") + Expect(restOfLogs).ToNot(ContainSubstring(logRequireBackendReload)) + Expect(restOfLogs).ToNot(ContainSubstring(logBackendReloadSuccess)) + Expect(restOfLogs).To(ContainSubstring(logSkipBackendReload)) + }) + + Context("given an ingress with TLS correctly configured", func() { + BeforeEach(func() { + ing, err := f.EnsureIngress(framework.NewSingleIngressWithTLS(host, "/", host, f.IngressController.Namespace, "http-svc", 80, nil)) + Expect(err).NotTo(HaveOccurred()) + Expect(ing).NotTo(BeNil()) + time.Sleep(waitForLuaSync) + + resp, _, errs := gorequest.New(). + Get(f.IngressController.HTTPSURL). + Set("Host", host). + TLSClientConfig(&tls.Config{ + InsecureSkipVerify: true, + ServerName: host, + }). + End() + Expect(len(errs)).Should(BeNumerically("==", 0)) + Expect(resp.StatusCode).Should(Equal(http.StatusOK)) + + By("configuring HTTPS endpoint") + err = f.WaitForNginxServer(host, + func(server string) bool { + return strings.Contains(server, "server_name "+host) && + strings.Contains(server, "listen 443") + }) + Expect(err).ToNot(HaveOccurred()) + + _, err = framework.CreateIngressTLSSecret(f.KubeClientSet, + ing.Spec.TLS[0].Hosts, + ing.Spec.TLS[0].SecretName, + ing.Namespace) + Expect(err).ToNot(HaveOccurred()) + time.Sleep(waitForLuaSync) + + By("configuring certificate_by_lua and skipping Nginx configuration of the new certificate") + err = f.WaitForNginxServer(ing.Spec.TLS[0].Hosts[0], + func(server string) bool { + return strings.Contains(server, "ssl_certificate_by_lua_block") && + !strings.Contains(server, fmt.Sprintf("ssl_certificate /etc/ingress-controller/ssl/%s-%s.pem;", ing.Namespace, host)) && + !strings.Contains(server, fmt.Sprintf("ssl_certificate_key /etc/ingress-controller/ssl/%s-%s.pem;", ing.Namespace, host)) && + strings.Contains(server, "listen 443") + }) + Expect(err).ToNot(HaveOccurred()) + + time.Sleep(waitForLuaSync) + + By("serving the configured certificate on HTTPS endpoint") + resp, _, errs = gorequest.New(). + Get(f.IngressController.HTTPSURL). + Set("Host", ing.Spec.TLS[0].Hosts[0]). + TLSClientConfig(&tls.Config{ + InsecureSkipVerify: true, + ServerName: ing.Spec.TLS[0].Hosts[0], + }). + End() + Expect(len(errs)).Should(BeNumerically("==", 0)) + Expect(resp.StatusCode).Should(Equal(http.StatusOK)) + Expect(len(resp.TLS.PeerCertificates)).Should(BeNumerically("==", 1)) + Expect(resp.TLS.PeerCertificates[0].DNSNames[0]).Should(Equal(host)) + }) + + It("picks up the updated certificate without reloading", func() { + ing, err := f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.IngressController.Namespace).Get("foo.com", metav1.GetOptions{}) + + resp, _, errs := gorequest.New(). + Get(fmt.Sprintf("%s?id=dummy_log_splitter_foo_bar", f.IngressController.HTTPSURL)). + Set("Host", host). + TLSClientConfig(&tls.Config{ + InsecureSkipVerify: true, + ServerName: ing.Spec.TLS[0].Hosts[0], + }). + End() + Expect(len(errs)).Should(BeNumerically("==", 0)) + Expect(resp.StatusCode).Should(Equal(http.StatusOK)) + + _, err = framework.CreateIngressTLSSecret(f.KubeClientSet, + ing.Spec.TLS[0].Hosts, + ing.Spec.TLS[0].SecretName, + ing.Namespace) + Expect(err).ToNot(HaveOccurred()) + time.Sleep(waitForLuaSync) + + By("configuring certificate_by_lua and skipping Nginx configuration of the new certificate") + err = f.WaitForNginxServer(ing.Spec.TLS[0].Hosts[0], + func(server string) bool { + return strings.Contains(server, "ssl_certificate_by_lua_block") && + !strings.Contains(server, fmt.Sprintf("ssl_certificate /etc/ingress-controller/ssl/%s-%s.pem;", ing.Namespace, host)) && + !strings.Contains(server, fmt.Sprintf("ssl_certificate_key /etc/ingress-controller/ssl/%s-%s.pem;", ing.Namespace, host)) && + strings.Contains(server, "listen 443") + }) + Expect(err).ToNot(HaveOccurred()) + + time.Sleep(waitForLuaSync) + + By("serving the configured certificate on HTTPS endpoint") + resp, _, errs = gorequest.New(). + Get(f.IngressController.HTTPSURL). + Set("Host", ing.Spec.TLS[0].Hosts[0]). + TLSClientConfig(&tls.Config{ + InsecureSkipVerify: true, + ServerName: ing.Spec.TLS[0].Hosts[0], + }). + End() + Expect(len(errs)).Should(BeNumerically("==", 0)) + Expect(resp.StatusCode).Should(Equal(http.StatusOK)) + Expect(len(resp.TLS.PeerCertificates)).Should(BeNumerically("==", 1)) + Expect(resp.TLS.PeerCertificates[0].DNSNames[0]).Should(Equal(host)) + + log, err := f.NginxLogs() + Expect(err).ToNot(HaveOccurred()) + Expect(log).ToNot(BeEmpty()) + index := strings.Index(log, "id=dummy_log_splitter_foo_bar") + restOfLogs := log[index:] + + By("skipping Nginx reload") + Expect(restOfLogs).ToNot(ContainSubstring(logRequireBackendReload)) + Expect(restOfLogs).ToNot(ContainSubstring(logBackendReloadSuccess)) + Expect(restOfLogs).To(ContainSubstring(logSkipBackendReload)) + }) + + It("falls back to using default certificate when secret gets deleted without reloading", func() { + ing, err := f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.IngressController.Namespace).Get("foo.com", metav1.GetOptions{}) + + resp, _, errs := gorequest.New(). + Get(fmt.Sprintf("%s?id=dummy_log_splitter_foo_bar", f.IngressController.HTTPSURL)). + Set("Host", host). + TLSClientConfig(&tls.Config{ + InsecureSkipVerify: true, + ServerName: ing.Spec.TLS[0].Hosts[0], + }). + End() + Expect(len(errs)).Should(BeNumerically("==", 0)) + Expect(resp.StatusCode).Should(Equal(http.StatusOK)) + + f.KubeClientSet.CoreV1().Secrets(ing.Namespace).Delete(ing.Spec.TLS[0].SecretName, nil) + Expect(err).ToNot(HaveOccurred()) + time.Sleep(waitForLuaSync) + + By("configuring certificate_by_lua and skipping Nginx configuration of the new certificate") + err = f.WaitForNginxServer(ing.Spec.TLS[0].Hosts[0], + func(server string) bool { + return strings.Contains(server, "ssl_certificate_by_lua_block") && + strings.Contains(server, "ssl_certificate /etc/ingress-controller/ssl/default-fake-certificate.pem;") && + strings.Contains(server, "ssl_certificate_key /etc/ingress-controller/ssl/default-fake-certificate.pem;") && + strings.Contains(server, "listen 443") + }) + Expect(err).ToNot(HaveOccurred()) + + time.Sleep(waitForLuaSync) + + By("serving the default certificate on HTTPS endpoint") + resp, _, errs = gorequest.New(). + Get(f.IngressController.HTTPSURL). + Set("Host", ing.Spec.TLS[0].Hosts[0]). + TLSClientConfig(&tls.Config{ + InsecureSkipVerify: true, + ServerName: ing.Spec.TLS[0].Hosts[0], + }). + End() + Expect(len(errs)).Should(BeNumerically("==", 0)) + Expect(resp.StatusCode).Should(Equal(http.StatusOK)) + Expect(len(resp.TLS.PeerCertificates)).Should(BeNumerically("==", 1)) + Expect(resp.TLS.PeerCertificates[0].Issuer.CommonName).Should(Equal("Kubernetes Ingress Controller Fake Certificate")) + + log, err := f.NginxLogs() + Expect(err).ToNot(HaveOccurred()) + Expect(log).ToNot(BeEmpty()) + index := strings.Index(log, "id=dummy_log_splitter_foo_bar") + restOfLogs := log[index:] + + By("skipping Nginx reload") + Expect(restOfLogs).ToNot(ContainSubstring(logRequireBackendReload)) + Expect(restOfLogs).ToNot(ContainSubstring(logBackendReloadSuccess)) + Expect(restOfLogs).To(ContainSubstring(logSkipBackendReload)) + }) + + It("picks up a non-certificate only change", func() { + newHost := "foo2.com" + ing, err := f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.IngressController.Namespace).Get("foo.com", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + ing.Spec.Rules[0].Host = newHost + _, err = f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.IngressController.Namespace).Update(ing) + Expect(err).ToNot(HaveOccurred()) + + By("configuring HTTPS endpoint") + err = f.WaitForNginxServer(newHost, + func(server string) bool { + return strings.Contains(server, "server_name "+newHost) && + strings.Contains(server, "listen 443") + }) + Expect(err).ToNot(HaveOccurred()) + + By("serving the configured certificate on HTTPS endpoint") + resp, _, errs := gorequest.New(). + Get(f.IngressController.HTTPSURL). + Set("Host", newHost). + TLSClientConfig(&tls.Config{ + InsecureSkipVerify: true, + ServerName: newHost, + }). + End() + Expect(len(errs)).Should(BeNumerically("==", 0)) + Expect(resp.StatusCode).Should(Equal(http.StatusOK)) + Expect(len(resp.TLS.PeerCertificates)).Should(BeNumerically("==", 1)) + Expect(resp.TLS.PeerCertificates[0].Issuer.CommonName).Should(Equal("Kubernetes Ingress Controller Fake Certificate")) + }) + + It("removes HTTPS configuration when we delete TLS spec", func() { + ing, err := f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.IngressController.Namespace).Get("foo.com", metav1.GetOptions{}) + Expect(err).NotTo(HaveOccurred()) + + ing.Spec.TLS = []extensions.IngressTLS{} + + _, err = f.KubeClientSet.ExtensionsV1beta1().Ingresses(f.IngressController.Namespace).Update(ing) + Expect(err).ToNot(HaveOccurred()) + By("configuring HTTP endpoint") + err = f.WaitForNginxServer(host, + func(server string) bool { + return !strings.Contains(server, "ssl_certificate_by_lua_block") && + !strings.Contains(server, "listen 443") + }) + Expect(err).ToNot(HaveOccurred()) + + resp, _, errs := gorequest.New(). + Get(f.IngressController.HTTPURL). + Set("Host", host). + End() + Expect(len(errs)).Should(BeNumerically("==", 0)) + Expect(resp.StatusCode).Should(Equal(http.StatusOK)) + }) + }) +}) + +func enableDynamicCertificates(namespace string, kubeClientSet kubernetes.Interface) error { + return framework.UpdateDeployment(kubeClientSet, namespace, "nginx-ingress-controller", 1, + func(deployment *appsv1beta1.Deployment) error { + args := deployment.Spec.Template.Spec.Containers[0].Args + args = append(args, "--enable-dynamic-certificates") + args = append(args, "--enable-ssl-chain-completion=false") + deployment.Spec.Template.Spec.Containers[0].Args = args + _, err := kubeClientSet.AppsV1beta1().Deployments(namespace).Update(deployment) + + return err + }) +} diff --git a/test/e2e/lua/dynamic_configuration.go b/test/e2e/lua/dynamic_configuration.go index 788a1bdbf5..87e2388d2d 100644 --- a/test/e2e/lua/dynamic_configuration.go +++ b/test/e2e/lua/dynamic_configuration.go @@ -41,7 +41,7 @@ const ( logBackendReloadSuccess = "Backend successfully reloaded" logSkipBackendReload = "Changes handled by the dynamic configuration, skipping backend reload" logInitialConfigSync = "Initial synchronization of the NGINX configuration" - waitForLuaSync = 2 * time.Second + waitForLuaSync = 5 * time.Second ) var _ = framework.IngressNginxDescribe("Dynamic Configuration", func() {