From 2125126b04bde727d59be6f0aac9975536f67ba3 Mon Sep 17 00:00:00 2001 From: elijah Date: Tue, 17 Jan 2023 17:27:23 +0800 Subject: [PATCH 01/15] feat(k8s-discovery): support mTLS --- .../discovery/kubernetes/informer_factory.lua | 16 +++++++-- apisix/discovery/kubernetes/init.lua | 33 +++++++++++++++++-- 2 files changed, 43 insertions(+), 6 deletions(-) diff --git a/apisix/discovery/kubernetes/informer_factory.lua b/apisix/discovery/kubernetes/informer_factory.lua index 3dca064039fb..bde17bd4bde8 100644 --- a/apisix/discovery/kubernetes/informer_factory.lua +++ b/apisix/discovery/kubernetes/informer_factory.lua @@ -23,6 +23,10 @@ local type = type local core = require("apisix.core") local http = require("resty.http") +if not http.tls_handshake then + error("Bad http library. Should use api7-lua-resty-http instead") +end + local function list_query(informer) local arguments = { limit = informer.limit, @@ -269,12 +273,18 @@ local function list_watch(informer, apiserver) informer.fetch_state = "connecting" core.log.info("begin to connect ", apiserver.host, ":", apiserver.port) - ok, message = httpc:connect({ + local opt = { scheme = apiserver.schema, host = apiserver.host, port = apiserver.port, - ssl_verify = false - }) + ssl_verify = apiserver.ssl_verify, + } + if apiserver.schema == "https" and apiserver.certificate ~= "" and apiserver.key ~= "" then + opt.ssl_cert_path = apiserver.certificate + opt.ssl_key_path = apiserver.key + opt.ssl_server_name = apiserver.host + end + ok, message = httpc:connect(opt) if not ok then informer.fetch_state = "connect failed" diff --git a/apisix/discovery/kubernetes/init.lua b/apisix/discovery/kubernetes/init.lua index d7258a55642b..b0949890d46c 100644 --- a/apisix/discovery/kubernetes/init.lua +++ b/apisix/discovery/kubernetes/init.lua @@ -270,15 +270,42 @@ local function get_apiserver(conf) if err then return nil, err end + elseif conf.client.certificate and conf.client.key then + apiserver.certificate, err = read_env(conf.client.certificate) + if err then + return nil, err + end + apiserver.key, err = read_env(conf.client.key) + if err then + return nil, err + end else - return nil, "one of [client.token,client.token_file] should be set but none" + return nil, "one of [client.token,client.token_file, (client.certificate, client.key)] should be set but none" + end + + apiserver.ssl_verify = false + if conf.client.ssl_verify then + apiserver.ssl_verify, err = read_env(conf.client.ssl_verify) + if err then + return nil, err + end + if apiserver.ssl_verify ~= "true" and apiserver.ssl_verify ~= "false" then + return nil, "client.ssl_verify should be set to one of [true,false] but " .. apiserver.ssl_verify + end + if apiserver.ssl_verify == "true" then + apiserver.ssl_verify = true + end end -- remove possible extra whitespace apiserver.token = apiserver.token:gsub("%s+", "") + apiserver.certificate = apiserver.certificate:gsub("%s+", "") + apiserver.key = apiserver.key:gsub("%s+", "") - if apiserver.schema == "https" and apiserver.token == "" then - return nil, "apiserver.token should set to non-empty string when service.schema is https" + if apiserver.schema == "https" then + if (apiserver.token == "" or apiserver.certificate == "" or apiserver.key == "") then + return nil, "apiserver.token or (apiserver.certificate and apiserver.key) should set to non-empty string when service.schema is https" + end end return apiserver From 9bca96e137c498c1448ff4e082d599cbec5d3872 Mon Sep 17 00:00:00 2001 From: elijah Date: Sat, 21 Jan 2023 22:52:55 +0800 Subject: [PATCH 02/15] fix: tlshandshake with the client cert and key --- .../discovery/kubernetes/informer_factory.lua | 22 +++++++++++------- apisix/discovery/kubernetes/init.lua | 23 ++++++------------- apisix/discovery/kubernetes/schema.lua | 19 ++++++++++++--- apisix/patch.lua | 3 +++ 4 files changed, 40 insertions(+), 27 deletions(-) diff --git a/apisix/discovery/kubernetes/informer_factory.lua b/apisix/discovery/kubernetes/informer_factory.lua index bde17bd4bde8..9a6883b078e3 100644 --- a/apisix/discovery/kubernetes/informer_factory.lua +++ b/apisix/discovery/kubernetes/informer_factory.lua @@ -22,6 +22,7 @@ local math = math local type = type local core = require("apisix.core") local http = require("resty.http") +local patch = require("apisix.patch") if not http.tls_handshake then error("Bad http library. Should use api7-lua-resty-http instead") @@ -49,15 +50,18 @@ end local function list(httpc, apiserver, informer) + local headers = { + ["Host"] = apiserver.host .. ":" .. apiserver.port, + ["Accept"] = "application/json", + ["Connection"] = "keep-alive" + } + if apiserver.token ~= "" then + headers["Authorization"] = "Bearer " .. apiserver.token + end local response, err = httpc:request({ path = informer.path, query = list_query(informer), - headers = { - ["Host"] = apiserver.host .. ":" .. apiserver.port, - ["Authorization"] = "Bearer " .. apiserver.token, - ["Accept"] = "application/json", - ["Connection"] = "keep-alive" - } + headers = headers }) core.log.info("--raw=", informer.path, "?", list_query(informer)) @@ -279,10 +283,12 @@ local function list_watch(informer, apiserver) port = apiserver.port, ssl_verify = apiserver.ssl_verify, } - if apiserver.schema == "https" and apiserver.certificate ~= "" and apiserver.key ~= "" then - opt.ssl_cert_path = apiserver.certificate + if apiserver.schema == "https" and apiserver.cert ~= "" and apiserver.key ~= "" then + opt.ssl_cert_path = apiserver.cert opt.ssl_key_path = apiserver.key opt.ssl_server_name = apiserver.host + -- replace tcp socket of http client to support mtls + httpc.sock = patch.lua_tcp_socket() end ok, message = httpc:connect(opt) diff --git a/apisix/discovery/kubernetes/init.lua b/apisix/discovery/kubernetes/init.lua index b0949890d46c..4af7b8f5d4dd 100644 --- a/apisix/discovery/kubernetes/init.lua +++ b/apisix/discovery/kubernetes/init.lua @@ -270,31 +270,22 @@ local function get_apiserver(conf) if err then return nil, err end - elseif conf.client.certificate and conf.client.key then - apiserver.certificate, err = read_env(conf.client.certificate) + elseif conf.client.cert_file and conf.client.key_file then + apiserver.cert, err = read_env(conf.client.cert_file) if err then return nil, err end - apiserver.key, err = read_env(conf.client.key) + apiserver.key, err = read_env(conf.client.key_file) if err then return nil, err end else - return nil, "one of [client.token,client.token_file, (client.certificate, client.key)] should be set but none" + return nil, "one of [client.token,client.token_file,(client.cert_file,client.key_file)] should be set but none" end apiserver.ssl_verify = false if conf.client.ssl_verify then - apiserver.ssl_verify, err = read_env(conf.client.ssl_verify) - if err then - return nil, err - end - if apiserver.ssl_verify ~= "true" and apiserver.ssl_verify ~= "false" then - return nil, "client.ssl_verify should be set to one of [true,false] but " .. apiserver.ssl_verify - end - if apiserver.ssl_verify == "true" then - apiserver.ssl_verify = true - end + apiserver.ssl_verify = conf.client.ssl_verify end -- remove possible extra whitespace @@ -303,8 +294,8 @@ local function get_apiserver(conf) apiserver.key = apiserver.key:gsub("%s+", "") if apiserver.schema == "https" then - if (apiserver.token == "" or apiserver.certificate == "" or apiserver.key == "") then - return nil, "apiserver.token or (apiserver.certificate and apiserver.key) should set to non-empty string when service.schema is https" + if apiserver.token == "" and (apiserver.cert == "" or apiserver.key == "") then + return nil, "apiserver.token or (apiserver.cert and apiserver.key) should set to non-empty string when service.schema is https" end end diff --git a/apisix/discovery/kubernetes/schema.lua b/apisix/discovery/kubernetes/schema.lua index 170608f553b9..ba0249aee13e 100644 --- a/apisix/discovery/kubernetes/schema.lua +++ b/apisix/discovery/kubernetes/schema.lua @@ -41,13 +41,18 @@ local token_schema = { oneOf = token_patterns, } -local token_file_schema = { +local file_schema = { type = "string", pattern = [[^[^\:*?"<>|]*$]], minLength = 1, maxLength = 500, } +local default_ssl_verify_schema = { + type = "boolean", + default = false, +} + local namespace_pattern = [[^[a-z0-9]([-a-z0-9_.]*[a-z0-9])?$]] local namespace_regex_pattern = [[^[\x21-\x7e]*$]] @@ -135,7 +140,10 @@ return { type = "object", properties = { token = token_schema, - token_file = token_file_schema, + token_file = file_schema, + cert_file = file_schema, + key_file = file_schema, + ssl_verify = default_ssl_verify_schema, }, default = { token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" @@ -145,6 +153,7 @@ return { anyOf = { { required = { "token" } }, { required = { "token_file" } }, + { required = { "cert_file", "key_file" } }, } } }, @@ -191,11 +200,15 @@ return { type = "object", properties = { token = token_schema, - token_file = token_file_schema, + token_file = file_schema, + cert_file = file_schema, + key_file = file_schema, + ssl_verify = default_ssl_verify_schema, }, oneOf = { { required = { "token" } }, { required = { "token_file" } }, + { required = { "cert_file", "key_file" } }, }, }, namespace_selector = namespace_selector_schema, diff --git a/apisix/patch.lua b/apisix/patch.lua index 2b191b2a83e1..5f1e4c56a2d0 100644 --- a/apisix/patch.lua +++ b/apisix/patch.lua @@ -380,5 +380,8 @@ function _M.patch() end end +function _M.lua_tcp_socket() + return luasocket_tcp() +end return _M From b00b9fea796f0fa831febcf5fd07ea12bf1eba30 Mon Sep 17 00:00:00 2001 From: elijah Date: Sat, 21 Jan 2023 23:24:27 +0800 Subject: [PATCH 03/15] chore: fixl int --- .../discovery/kubernetes/informer_factory.lua | 18 +++++++++--------- apisix/discovery/kubernetes/init.lua | 6 ++++-- 2 files changed, 13 insertions(+), 11 deletions(-) diff --git a/apisix/discovery/kubernetes/informer_factory.lua b/apisix/discovery/kubernetes/informer_factory.lua index 9a6883b078e3..a33d4276d1af 100644 --- a/apisix/discovery/kubernetes/informer_factory.lua +++ b/apisix/discovery/kubernetes/informer_factory.lua @@ -24,9 +24,6 @@ local core = require("apisix.core") local http = require("resty.http") local patch = require("apisix.patch") -if not http.tls_handshake then - error("Bad http library. Should use api7-lua-resty-http instead") -end local function list_query(informer) local arguments = { @@ -212,15 +209,18 @@ local function watch(httpc, apiserver, informer) local http_seconds = watch_seconds + 120 httpc:set_timeouts(2000, 3000, http_seconds * 1000) + local headers = { + ["Host"] = apiserver.host .. ":" .. apiserver.port, + ["Accept"] = "application/json", + ["Connection"] = "keep-alive" + } + if apiserver.token ~= "" then + headers["Authorization"] = "Bearer " .. apiserver.token + end local response, err = httpc:request({ path = informer.path, query = watch_query(informer), - headers = { - ["Host"] = apiserver.host .. ":" .. apiserver.port, - ["Authorization"] = "Bearer " .. apiserver.token, - ["Accept"] = "application/json", - ["Connection"] = "keep-alive" - } + headers = headers }) core.log.info("--raw=", informer.path, "?", watch_query(informer)) diff --git a/apisix/discovery/kubernetes/init.lua b/apisix/discovery/kubernetes/init.lua index 4af7b8f5d4dd..496d53adabf9 100644 --- a/apisix/discovery/kubernetes/init.lua +++ b/apisix/discovery/kubernetes/init.lua @@ -280,7 +280,8 @@ local function get_apiserver(conf) return nil, err end else - return nil, "one of [client.token,client.token_file,(client.cert_file,client.key_file)] should be set but none" + return nil, "one of [client.token,client.token_file,(client.cert_file,client.key_file)] ".. + "should be set but none" end apiserver.ssl_verify = false @@ -295,7 +296,8 @@ local function get_apiserver(conf) if apiserver.schema == "https" then if apiserver.token == "" and (apiserver.cert == "" or apiserver.key == "") then - return nil, "apiserver.token or (apiserver.cert and apiserver.key) should set to non-empty string when service.schema is https" + return nil, "apiserver.token or (apiserver.cert and apiserver.key) ".. + "should set to non-empty string when service.schema is https" end end From c5f5985393d937ec4f6d1103deaec7eda7a1e843 Mon Sep 17 00:00:00 2001 From: elijah Date: Mon, 23 Jan 2023 00:33:27 +0800 Subject: [PATCH 04/15] fix test --- apisix/discovery/kubernetes/init.lua | 2 -- 1 file changed, 2 deletions(-) diff --git a/apisix/discovery/kubernetes/init.lua b/apisix/discovery/kubernetes/init.lua index 496d53adabf9..1191a9d1c232 100644 --- a/apisix/discovery/kubernetes/init.lua +++ b/apisix/discovery/kubernetes/init.lua @@ -291,8 +291,6 @@ local function get_apiserver(conf) -- remove possible extra whitespace apiserver.token = apiserver.token:gsub("%s+", "") - apiserver.certificate = apiserver.certificate:gsub("%s+", "") - apiserver.key = apiserver.key:gsub("%s+", "") if apiserver.schema == "https" then if apiserver.token == "" and (apiserver.cert == "" or apiserver.key == "") then From 1ac52c06ce21e7d5f9320a4b2cdc6e58aceac30c Mon Sep 17 00:00:00 2001 From: elijah Date: Mon, 23 Jan 2023 23:22:30 +0800 Subject: [PATCH 05/15] chore: fix schema --- apisix/discovery/kubernetes/schema.lua | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/apisix/discovery/kubernetes/schema.lua b/apisix/discovery/kubernetes/schema.lua index ba0249aee13e..52926ebe5b64 100644 --- a/apisix/discovery/kubernetes/schema.lua +++ b/apisix/discovery/kubernetes/schema.lua @@ -48,9 +48,8 @@ local file_schema = { maxLength = 500, } -local default_ssl_verify_schema = { +local ssl_verify_schema = { type = "boolean", - default = false, } local namespace_pattern = [[^[a-z0-9]([-a-z0-9_.]*[a-z0-9])?$]] @@ -143,7 +142,7 @@ return { token_file = file_schema, cert_file = file_schema, key_file = file_schema, - ssl_verify = default_ssl_verify_schema, + ssl_verify = ssl_verify_schema, }, default = { token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token" @@ -203,7 +202,7 @@ return { token_file = file_schema, cert_file = file_schema, key_file = file_schema, - ssl_verify = default_ssl_verify_schema, + ssl_verify = ssl_verify_schema, }, oneOf = { { required = { "token" } }, From b0c32cdc7df760252f1862cd2a0bf5e3dd90f667 Mon Sep 17 00:00:00 2001 From: elijah Date: Tue, 24 Jan 2023 16:22:40 +0800 Subject: [PATCH 06/15] fix test --- apisix/discovery/kubernetes/informer_factory.lua | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apisix/discovery/kubernetes/informer_factory.lua b/apisix/discovery/kubernetes/informer_factory.lua index a33d4276d1af..458df68c4a95 100644 --- a/apisix/discovery/kubernetes/informer_factory.lua +++ b/apisix/discovery/kubernetes/informer_factory.lua @@ -283,7 +283,9 @@ local function list_watch(informer, apiserver) port = apiserver.port, ssl_verify = apiserver.ssl_verify, } - if apiserver.schema == "https" and apiserver.cert ~= "" and apiserver.key ~= "" then + if apiserver.schema == "https" and + apiserver.cert and apiserver.cert ~= "" and + apiserver.key and apiserver.key ~= "" then opt.ssl_cert_path = apiserver.cert opt.ssl_key_path = apiserver.key opt.ssl_server_name = apiserver.host From e5d634f9ede3d79732c18df4bbc6b78f60437a7d Mon Sep 17 00:00:00 2001 From: elijah Date: Thu, 26 Jan 2023 16:32:58 +0800 Subject: [PATCH 07/15] chore: add mtls test --- .github/workflows/kubernetes-ci.yml | 14 + .../discovery/kubernetes/informer_factory.lua | 4 +- t/certs/k8s_mtls.key | 51 ++ t/certs/k8s_mtls_csr.conf | 35 ++ t/certs/k8s_mtls_csr.yaml | 29 ++ t/kubernetes/discovery/kubernetes.t | 153 ++++++ t/kubernetes/discovery/kubernetes_mtls.t | 489 ++++++++++++++++++ 7 files changed, 773 insertions(+), 2 deletions(-) create mode 100644 t/certs/k8s_mtls.key create mode 100644 t/certs/k8s_mtls_csr.conf create mode 100644 t/certs/k8s_mtls_csr.yaml create mode 100644 t/kubernetes/discovery/kubernetes_mtls.t diff --git a/.github/workflows/kubernetes-ci.yml b/.github/workflows/kubernetes-ci.yml index ea72fe57144c..3ba0385a6070 100644 --- a/.github/workflows/kubernetes-ci.yml +++ b/.github/workflows/kubernetes-ci.yml @@ -68,6 +68,20 @@ jobs: echo 'KUBERNETES_CLIENT_TOKEN='"${KUBERNETES_CLIENT_TOKEN_CONTENT}" echo 'KUBERNETES_CLIENT_TOKEN_FILE='${KUBERNETES_CLIENT_TOKEN_FILE} + openssl req -new -key ./t/certs/k8s_mtls.key -config ./t/certs/k8s_mtls_csr.conf -out ./t/certs/k8s_mtls.csr -nodes + + BASE64_CSR=$(cat ./t/certs/k8s_mtls.csr | base64 | tr -d '\n') + + cat ./t/certs/k8s_mtls_csr.yaml | envsubst | kubectl apply -f - + + k certificate approve k8s-mtls-csr + + k get csr k8s-mtls-csr -o jsonpath='{.status.certificate}' | base64 --decode > ./t/certs/k8s_mtls.pem + + kubectl get secrets | grep apisix-test | awk '{system("kubectl get secret -o jsonpath=\"{.data.ca\.crt}\" "$1" | base64 -d")}' > ./t/certs/k8s_mtls_ca.pem + + export KUBERNETES_APISERVER_ADDR=$(kubectl -n kube-system get pod -l component=kube-apiserver -o=jsonpath="{.items[0].metadata.annotations.kubeadm\.kubernetes\.io/kube-apiserver\.advertise-address\.endpoint}" | awk -F: '{print $1}') + kubectl proxy -p 6445 & - name: Linux Install diff --git a/apisix/discovery/kubernetes/informer_factory.lua b/apisix/discovery/kubernetes/informer_factory.lua index 458df68c4a95..7d4923d6cffb 100644 --- a/apisix/discovery/kubernetes/informer_factory.lua +++ b/apisix/discovery/kubernetes/informer_factory.lua @@ -52,7 +52,7 @@ local function list(httpc, apiserver, informer) ["Accept"] = "application/json", ["Connection"] = "keep-alive" } - if apiserver.token ~= "" then + if apiserver.token and apiserver.token ~= "" then headers["Authorization"] = "Bearer " .. apiserver.token end local response, err = httpc:request({ @@ -214,7 +214,7 @@ local function watch(httpc, apiserver, informer) ["Accept"] = "application/json", ["Connection"] = "keep-alive" } - if apiserver.token ~= "" then + if apiserver.token and apiserver.token ~= "" then headers["Authorization"] = "Bearer " .. apiserver.token end local response, err = httpc:request({ diff --git a/t/certs/k8s_mtls.key b/t/certs/k8s_mtls.key new file mode 100644 index 000000000000..5203cbf6e62c --- /dev/null +++ b/t/certs/k8s_mtls.key @@ -0,0 +1,51 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIJKQIBAAKCAgEAsQ6RDAiCQjhaZuOxNNquBoKuTvfFTL/J40yvWy1JUpMuF8Dt +CDbt+tgoi7RwWm7QDM01JHAdPlwLYzuFDpRinodWI36DcHlnYXbFoOLvrsEJayB0 +WZgE67zWHIn0+UfT8rTm+IhcS0f4ifBidWmpVnT9WkOzI4agLUNI4YxfU+/88asA +nafOny/kFcsQqQDjjgv5iv0ORUK312Ydv7lVTIdjDb7WOHu08nt5PMq4SS3c1uOc +ypvI0O7uudAc3GUxtPp6VE2Ff2TZoMKI5fY80zYk8JEUkCrOKuMvB+JzN59qPqag +/bxQhL8nPMp12Id4SISHgEmQSuiTt+SUnj6Sp2FiAdMWlcPssucbXlw2yO7PfJNW +xiiTgrf30eGGcw2fkwBQmZkWPDR3snkI/lps2MZVbYDXUEGk0PYMW6wRzOXr1M7Y +MWhxCHlwKzNeb4dH3RMZiAPc3v0upDyzqxvlWI3Q+8pMEqO4e2HjmyDKlZ4fh7jY +rORSugfCh4Fb7q3AwKu1L2fyGkna63B9vYxUDBqgrEtYusK66oO1/CZv5c7xSfL9 +xEpGCQr4Tot60b4QTmmB7ZBF5FvlTXxZS2y49c8Bgdh+Z5pkYEKT0LnabvN6Jurb +HpjFeFxSMFk/WTiad7ovKylNknSAJalyFYvSc4Uq8uumq6VjxV70Btf3L/kCAwEA +AQKCAgA7yB9RYM4abcyYUOjUX8HCkP0pzJ7heyA/jvTmZDQNnYRCoPFnhKR1veZy +xUnqPuAgcmkvFGvLUmjF6KLUeWWTAT8kXOMkZtlxHxk7aWRkLOlxMlQWrC19KxVg +LXR5drcpi8swv1nqUTKp4HBeFH8IohepEpK5vAr5PW6uOSc2wCE3t/VNRguN1C2+ +7gGCDrrGi2SGeUsIN667G8+PnJQgctYPHhBWoe5GsUoOLKRskP+vzspx3oe/g+02 +7/3OrJxL7wa+0C3zBao1jXmLq8h9yBpQ36uBwfI6qNqIXv1oGntVDA3vnlXRfHlT +HUkNAzCpy5W0yyHdKQo3XQaRJ698JK4GgSYBrn0jen2kZ4MHxa1PQIUjGkOwx+Lj +0y4kRpOv0RtacTsZvmg2AtEyMpPJoiQKjsIaTHWYauuaCFjDdjAIw5vz8qAJOc3E +oZvc6ksOizwOfmkHMcK0wO3Mw1ZXAiFBRh3H0T/0UIxAZQUTZzL8ZvVEf1oUz7MC +1bmJ5sAHpeLCgVWycjasaMqoMOULa0D3ljgH59aSuF7xjDC7nLol4h29ks0GNhel +IRMjX94pp7bJJvT7kG5k970ZprL/IEQBgkT1mFYBDSt6U4hbNY4zBxM+Bb5c+FrE +y7lpaFen2XIZswRzsEdqRaanLYzYV8KvjllQWOh1/dVPt0QqwQKCAQEA1z77CVNF +fzarastUS8N/+/6+cqNSWPFr2yEAX68xYIdkLr0wE+0L2qVEyBMwBchXkH+cR1yc +M3ZDjG0bTELuOmIUuJKshHUc7ULnI4u/6RVDIhRGCb85o9ITlrXhkflcem+qnNhP +8zo01h6OLIA8/Z7abl8KISb7IFZqvBDUuTV8ZJHNsHia3wshy9t0FtXTMw4dwpmG +5LDeyD+DdK3c5NOJV4CNEwksKIEEfv9fxSgBQrGM0zYs0Mif3Bk+68jdviyKAymB +d5umQ3oAf5BspILgqMAYAG23Rsymh1qSOgge5Y2XFXrmXmRNet6XWHly8FPwTpyu +7HI6+T8bDBxtDwKCAQEA0pSNLw4/jpTfaijPU46wUkVNPSgGyo6GiDQpLTmlZKBY +mM308ZF2BoCPn3oZEcAhEoHRYJM3rnvequrm7EUzRm4ujWyshPqsILwvVlebvZLA +7QK9IsB5K1dOhh+wKo6Sd8vl6qPpYzbyIeIZ5ZqVkc4xtgBx110e3Dm8C310SHd6 +oVlV0pcSHSWxNye8deUu58ylltG8GyQ1ZKEdrdgoOgPOt/QOfIeKehd10VaO0FoH +2rsUr8FjcXPM1nZEu0jgfx4/xDonI5NOyLhcjHHSNJJ6ZZK6yIMJkURqF4OJtFUT +N+rX86LWa68Q26Ny3E1FweC8cly+gGJ3oP6fBHOidwKCAQEAkoSRl3hyjzTknI6J +9ATBIwT2GMzp6hd0ZrZhLcDwqJeg71vi1T1/2zayykqdXuJQENF8zOrSfQlc08MV +WOUaiu1m03ksLAIYTb1D32NqoDvT9ee9ATeFAemnYeivApGujL1kct2gFp4hhztH +Zi1o1u+CGq1hFM4XFZ0djKUcWHJ7GW5SiHSgX4GRYNYigMTOvis2wNeHBjcLf14U +mqEbAzynecPMxA9zWqLwL46gX3Fq4j4/8SooeXqHrugUd6XEo4Zn790Z/xQZqkbq +/eqOubFJQjV4YRpOLshx9yME9LHAblviY0HEfK5kO2Y3uYb6YXOR2cE4JUVgtXdq +buUOqwKCAQEAork+XFVjUbfC6RL+GzCRGhIJfK+4h8AZP5knFJaOMzMhI3tLw3xH +OwEPewnr42aXZ+i/NiStC30W9PauWXekBtHP0WNf8SaP6n/hCJSMebK0uLG8G+Wl +/JLr0AMeq9QeZMpAMdBV4mAsPXit0FpjKBexHFY9olGILl/C6xIX0IPbVGFnS2OI +ENP2nIttO6FNXLwsXV5XQJX7WHaUK6l0dFhK3qWOqDZJEM2eMpTzqRdZKHIWqzYC +tUHp0PP1atAHAtQKLIVPh1Mmtz6X7cNqz0s9n84B3j48H3BBl0gDGkH/pt8n31I+ +5VmT6lXkxU5FEGus06bNYwI0NUls8hC1QwKCAQBKidO71ay/NQQJGzaiZMSCQCsI +pAAu/5FCGRjkui1MoUr0Tu8tDIop8hEl6ri36++Me7rYTvI13P5QUrQCM6mSLBgh +m1kqFyGWSIqiTWgb37A/Dfxbahzs/Sg9IZzs3rFD/kgvU3vsF5d2eBuVRN/xr//7 +teugW/xh1O7DUN3cdeInxnFLmII07+wtEKNO+Cu0dLBzAe4GCKOWCgmsW7NOKbt7 +sPuVoV4KgkxEdm4SBUt8A3q1jFunB8ESymBKx9PqiwbhthCpd0jfx9zAOtv4KnEg +a7IxAZtTc9eLlDEGn1G+BZJCZT1A0DgDFiipPCeJxDTLQI4gle2VUuDKYuwN +-----END RSA PRIVATE KEY----- diff --git a/t/certs/k8s_mtls_csr.conf b/t/certs/k8s_mtls_csr.conf new file mode 100644 index 000000000000..ecf4842a5836 --- /dev/null +++ b/t/certs/k8s_mtls_csr.conf @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +[req] +default_bits = 2048 +default_md = sha256 +distinguished_name = dn +prompt = no + +[dn] +CN = system:serviceaccount:default:apisix-test +O = system:serviceaccounts + +[v3_ext] +authorityKeyIdentifier = keyid,issuer:always +basicConstraints = CA:TRUE +keyUsage = keyEncipherment,dataEncipherment +extendedKeyUsage = clientAuth + +## openssl genrsa -out k8s_mtls.key 4096 +## openssl req -new -key k8s_mtls.key -config k8s_mtls_csr.conf -out k8s_mtls.csr -nodes + diff --git a/t/certs/k8s_mtls_csr.yaml b/t/certs/k8s_mtls_csr.yaml new file mode 100644 index 000000000000..ec8a3e438743 --- /dev/null +++ b/t/certs/k8s_mtls_csr.yaml @@ -0,0 +1,29 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +apiVersion: certificates.k8s.io/v1 +kind: CertificateSigningRequest +metadata: + name: k8s-mtls-csr +spec: + groups: + - system:authenticated + request: ${BASE64_CSR} + signerName: kubernetes.io/kube-apiserver-client + usages: + - digital signature + - key encipherment + - client auth diff --git a/t/kubernetes/discovery/kubernetes.t b/t/kubernetes/discovery/kubernetes.t index 24bf6f426a01..6c8196f67cea 100644 --- a/t/kubernetes/discovery/kubernetes.t +++ b/t/kubernetes/discovery/kubernetes.t @@ -24,6 +24,9 @@ workers(4); our $token_file = "/tmp/var/run/secrets/kubernetes.io/serviceaccount/token"; our $token_value = eval {`cat $token_file 2>/dev/null`}; +our $cert_file = "./t/certs/k8s_mtls.pem"; +our $key_file = "./t/certs/k8s_mtls.key"; +our $ca_cert_file = "./t/certs/k8s_mtls_ca.pem"; add_block_preprocessor(sub { my ($block) = @_; @@ -40,6 +43,9 @@ env MyPort=6443; env KUBERNETES_SERVICE_HOST=127.0.0.1; env KUBERNETES_SERVICE_PORT=6443; env KUBERNETES_CLIENT_TOKEN=$::token_value; +env KUBERNETES_CLIENT_CERT=$::cert_file; +env KUBERNETES_CLIENT_KEY=$::key_file; +env KUBERNETES_CERTIFICATE_AUTHORITY=$::ca_cert_file; _EOC_ $block->set_value("main_config", $main_config); @@ -304,3 +310,150 @@ GET /compare Content-type: application/json --- response_body true + + + +=== TEST 6: default value with minimal client tls configuration +--- yaml_config +apisix: + node_listen: 1984 + config_center: yaml +deployment: + role: data_plane + role_data_plane: + config_provider: yaml +discovery: + kubernetes: + client: + cert_file: ${KUBERNETES_CLIENT_CERT} + key_file: ${KUBERNETES_CLIENT_KEY} +--- request +GET /compare +{ + "service": { + "schema": "https", + "host": "${KUBERNETES_SERVICE_HOST}", + "port": "${KUBERNETES_SERVICE_PORT}" + }, + "client": { + "cert_file": "${KUBERNETES_CLIENT_CERT}", + "key_file": "${KUBERNETES_CLIENT_KEY}" + }, + "shared_size": "1m", + "default_weight": 50 +} +--- more_headers +Content-type: application/json +--- response_body +true + + + +=== TEST 7: client tls configuration with ca cert +--- yaml_config +apisix: + node_listen: 1984 + config_center: yaml +deployment: + role: data_plane + role_data_plane: + config_provider: yaml +ssl: + ssl_trusted_certificate: ${KUBERNETES_CERTIFICATE_AUTHORITY} + ssl_protocols: TLSv1.2 TLSv1.3 +discovery: + kubernetes: + client: + cert_file: ${KUBERNETES_CLIENT_CERT} + key_file: ${KUBERNETES_CLIENT_KEY} + ssl_verify: true +--- request +GET /compare +{ + "service": { + "schema": "https", + "host": "${KUBERNETES_SERVICE_HOST}", + "port": "${KUBERNETES_SERVICE_PORT}" + }, + "client": { + "cert_file": "${KUBERNETES_CLIENT_CERT}", + "key_file": "${KUBERNETES_CLIENT_KEY}", + "ssl_verify": true + }, + "shared_size": "1m", + "default_weight": 50 +} +--- more_headers +Content-type: application/json +--- response_body +true + + + +=== TEST 8: multi cluster mode client tls configuration +--- http_config +lua_shared_dict kubernetes-debug 1m; +lua_shared_dict kubernetes-release 1m; +--- yaml_config +apisix: + node_listen: 1984 +deployment: + role: data_plane + role_data_plane: + config_provider: yaml +discovery: + kubernetes: + - id: "debug" + service: + schema: "http" + host: "1.cluster.com" + port: "6445" + client: + token: ${KUBERNETES_CLIENT_TOKEN} + - id: "release" + service: + schema: "https" + host: "2.cluster.com" + port: "${MyPort}" + client: + cert_file: ${KUBERNETES_CLIENT_CERT} + key_file: ${KUBERNETES_CLIENT_KEY} + ssl_verify: false + default_weight: 33 + shared_size: "2m" +--- request +GET /compare +[ + { + "id": "debug", + "service": { + "schema": "http", + "host": "1.cluster.com", + "port": "6445" + }, + "client": { + "token": "${KUBERNETES_CLIENT_TOKEN}" + }, + "default_weight": 50, + "shared_size": "1m" + }, + { + "id": "release", + "service": { + "schema": "https", + "host": "2.cluster.com", + "port": "${MyPort}" + }, + "client": { + "cert_file": "${KUBERNETES_CLIENT_CERT}", + "key_file": "${KUBERNETES_CLIENT_KEY}", + "ssl_verify": false + }, + "default_weight": 33, + "shared_size": "2m" + } +] +--- more_headers +Content-type: application/json +--- response_body +true diff --git a/t/kubernetes/discovery/kubernetes_mtls.t b/t/kubernetes/discovery/kubernetes_mtls.t new file mode 100644 index 000000000000..118147c75ef8 --- /dev/null +++ b/t/kubernetes/discovery/kubernetes_mtls.t @@ -0,0 +1,489 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +BEGIN { + our $apiserver_addr = $ENV{'KUBERNETES_APISERVER_ADDR'}; + our $cert_file = "./t/certs/k8s_mtls.pem"; + our $key_file = "./t/certs/k8s_mtls.key"; + our $ca_cert_file = "./t/certs/k8s_mtls_ca.pem"; + our $token_file = "/tmp/var/run/secrets/kubernetes.io/serviceaccount/token"; + our $token_value = eval {`cat $token_file 2>/dev/null`}; + + our $yaml_config = <<_EOC_; +apisix: + node_listen: 1984 +deployment: + role: data_plane + role_data_plane: + config_provider: yaml +ssl: + ssl_trusted_certificate: $::ca_cert_file + ssl_protocols: TLSv1.2 TLSv1.3 +discovery: + kubernetes: + - id: first + service: + schema: "https" + host: $::apiserver_addr + port: "6443" + client: + cert_file: $::cert_file + key_file: $::key_file + ssl_verify: true + - id: second + service: + schema: "http" + host: "127.0.0.1" + port: "6445" + client: + token_file: "/tmp/var/run/secrets/kubernetes.io/serviceaccount/token" +_EOC_ + + our $scale_ns_c = <<_EOC_; +[ + { + "op": "replace_subsets", + "name": "ep", + "namespace": "ns-c", + "subsets": [ + { + "addresses": [ + { + "ip": "10.0.0.1" + } + ], + "ports": [ + { + "name": "p1", + "port": 5001 + } + ] + } + ] + } +] +_EOC_ + +} + +use t::APISIX 'no_plan'; + +repeat_each(1); +log_level('warn'); +no_root_location(); +no_shuffle(); +workers(4); + +add_block_preprocessor(sub { + my ($block) = @_; + + my $apisix_yaml = $block->apisix_yaml // <<_EOC_; +routes: [] +#END +_EOC_ + + $block->set_value("apisix_yaml", $apisix_yaml); + + my $main_config = $block->main_config // <<_EOC_; +env KUBERNETES_SERVICE_HOST=$::apiserver_addr; +env KUBERNETES_SERVICE_PORT=6443; +env KUBERNETES_CLIENT_CERT=$::cert_file; +env KUBERNETES_CLIENT_KEY=$::key_file; +env KUBERNETES_CERTIFICATE_AUTHORITY=$::ca_cert_file; +env KUBERNETES_CLIENT_TOKEN=$::token_value; +_EOC_ + + $block->set_value("main_config", $main_config); + + my $config = $block->config // <<_EOC_; + location /queries { + content_by_lua_block { + local core = require("apisix.core") + local d = require("apisix.discovery.kubernetes") + + ngx.sleep(1) + + ngx.req.read_body() + local request_body = ngx.req.get_body_data() + local queries = core.json.decode(request_body) + local response_body = "{" + for _,query in ipairs(queries) do + local nodes = d.nodes(query) + if nodes==nil or #nodes==0 then + response_body=response_body.." "..0 + else + response_body=response_body.." "..#nodes + end + end + ngx.say(response_body.." }") + } + } + + location /operators { + content_by_lua_block { + local http = require("resty.http") + local core = require("apisix.core") + local ipairs = ipairs + + ngx.req.read_body() + local request_body = ngx.req.get_body_data() + local operators = core.json.decode(request_body) + + core.log.info("get body ", request_body) + core.log.info("get operators ", #operators) + for _, op in ipairs(operators) do + local method, path, body + local headers = { + ["Host"] = "127.0.0.1:6445" + } + + if op.op == "replace_subsets" then + method = "PATCH" + path = "/api/v1/namespaces/" .. op.namespace .. "/endpoints/" .. op.name + if #op.subsets == 0 then + body = '[{"path":"/subsets","op":"replace","value":[]}]' + else + local t = { { op = "replace", path = "/subsets", value = op.subsets } } + body = core.json.encode(t, true) + end + headers["Content-Type"] = "application/json-patch+json" + end + + if op.op == "replace_labels" then + method = "PATCH" + path = "/api/v1/namespaces/" .. op.namespace .. "/endpoints/" .. op.name + local t = { { op = "replace", path = "/metadata/labels", value = op.labels } } + body = core.json.encode(t, true) + headers["Content-Type"] = "application/json-patch+json" + end + + local httpc = http.new() + core.log.info("begin to connect ", "127.0.0.1:6445") + local ok, message = httpc:connect({ + scheme = "http", + host = "127.0.0.1", + port = 6445, + }) + if not ok then + core.log.error("connect 127.0.0.1:6445 failed, message : ", message) + ngx.say("FAILED") + end + local res, err = httpc:request({ + method = method, + path = path, + headers = headers, + body = body, + }) + if err ~= nil then + core.log.err("operator k8s cluster error: ", err) + return 500 + end + if res.status ~= 200 and res.status ~= 201 and res.status ~= 409 then + return res.status + end + end + ngx.say("DONE") + } + } + +_EOC_ + + $block->set_value("config", $config); + +}); + +run_tests(); + +__DATA__ + +=== TEST 1: create namespace and endpoints +--- yaml_config eval: $::yaml_config +--- request +POST /operators +[ + { + "op": "replace_subsets", + "namespace": "ns-a", + "name": "ep", + "subsets": [ + { + "addresses": [ + { + "ip": "10.0.0.1" + }, + { + "ip": "10.0.0.2" + } + ], + "ports": [ + { + "name": "p1", + "port": 5001 + } + ] + }, + { + "addresses": [ + { + "ip": "20.0.0.1" + }, + { + "ip": "20.0.0.2" + } + ], + "ports": [ + { + "name": "p2", + "port": 5002 + } + ] + } + ] + }, + { + "op": "create_namespace", + "name": "ns-b" + }, + { + "op": "replace_subsets", + "namespace": "ns-b", + "name": "ep", + "subsets": [ + { + "addresses": [ + { + "ip": "10.0.0.1" + }, + { + "ip": "10.0.0.2" + } + ], + "ports": [ + { + "name": "p1", + "port": 5001 + } + ] + }, + { + "addresses": [ + { + "ip": "20.0.0.1" + }, + { + "ip": "20.0.0.2" + } + ], + "ports": [ + { + "name": "p2", + "port": 5002 + } + ] + } + ] + }, + { + "op": "create_namespace", + "name": "ns-c" + }, + { + "op": "replace_subsets", + "namespace": "ns-c", + "name": "ep", + "subsets": [ + { + "addresses": [ + { + "ip": "10.0.0.1" + }, + { + "ip": "10.0.0.2" + } + ], + "ports": [ + { + "port": 5001 + } + ] + }, + { + "addresses": [ + { + "ip": "20.0.0.1" + }, + { + "ip": "20.0.0.2" + } + ], + "ports": [ + { + "port": 5002 + } + ] + } + ] + } +] +--- more_headers +Content-type: application/json + + + +=== TEST 2: use default parameters +--- yaml_config eval: $::yaml_config +--- request +GET /queries +[ + "first/ns-a/ep:p1","first/ns-a/ep:p2","first/ns-b/ep:p1","first/ns-b/ep:p2","first/ns-c/ep:5001","first/ns-c/ep:5002", + "second/ns-a/ep:p1","second/ns-a/ep:p2","second/ns-b/ep:p1","second/ns-b/ep:p2","second/ns-c/ep:5001","second/ns-c/ep:5002" +] +--- more_headers +Content-type: application/json +--- response_body eval +qr{ 2 2 2 2 2 2 2 2 2 2 2 2 } + + + +=== TEST 3: use client tls without ca certificate +--- yaml_config +apisix: + node_listen: 1984 +deployment: + role: data_plane + role_data_plane: + config_provider: yaml +ssl: + ssl_protocols: TLSv1.2 TLSv1.3 +discovery: + kubernetes: + - id: first + service: + schema: "https" + host: ${KUBERNETES_SERVICE_HOST} + port: ${KUBERNETES_SERVICE_PORT} + client: + cert_file: ${KUBERNETES_CLIENT_CERT} + key_file: ${KUBERNETES_CLIENT_KEY} + ssl_verify: false + +--- request +GET /queries +[ + "first/ns-a/ep:p1","first/ns-a/ep:p2","first/ns-b/ep:p1","first/ns-b/ep:p2","first/ns-c/ep:5001","first/ns-c/ep:5002" +] +--- more_headers +Content-type: application/json +--- response_body eval +qr{ 2 2 2 2 2 2 } + + + +=== TEST 4: use client tls without specify ssl protocol +--- yaml_config +apisix: + node_listen: 1984 +deployment: + role: data_plane + role_data_plane: + config_provider: yaml +discovery: + kubernetes: + - id: first + service: + schema: "https" + host: ${KUBERNETES_SERVICE_HOST} + port: ${KUBERNETES_SERVICE_PORT} + client: + cert_file: ${KUBERNETES_CLIENT_CERT} + key_file: ${KUBERNETES_CLIENT_KEY} + ssl_verify: true + +--- request +GET /queries +[ + "first/ns-a/ep:p1","first/ns-a/ep:p2","first/ns-b/ep:p1","first/ns-b/ep:p2","first/ns-c/ep:5001","first/ns-c/ep:5002" +] +--- more_headers +Content-type: application/json +--- response_body eval +qr{ 2 2 2 2 2 2 } + + + +=== TEST 5: scale endpoints +--- yaml_config +apisix: + node_listen: 1984 +deployment: + role: data_plane + role_data_plane: + config_provider: yaml +ssl: + ssl_trusted_certificate: ${KUBERNETES_CERTIFICATE_AUTHORITY} + ssl_protocols: TLSv1.2 TLSv1.3 +discovery: + kubernetes: + - id: first + service: + schema: "https" + host: ${KUBERNETES_SERVICE_HOST} + port: ${KUBERNETES_SERVICE_PORT} + client: + cert_file: ${KUBERNETES_CLIENT_CERT} + key_file: ${KUBERNETES_CLIENT_KEY} + ssl_verify: true + +--- request eval +[ + +"GET /queries +[ + \"first/ns-a/ep:p1\",\"first/ns-a/ep:p2\" +]", + +"POST /operators +[{\"op\":\"replace_subsets\",\"name\":\"ep\",\"namespace\":\"ns-a\",\"subsets\":[]}]", + +"GET /queries +[ + \"first/ns-a/ep:p1\",\"first/ns-a/ep:p2\" +]", + +"GET /queries +[ + \"first/ns-c/ep:5001\",\"first/ns-c/ep:5002\",\"first/ns-c/ep:p1\" +]", + +"POST /operators +$::scale_ns_c", + +"GET /queries +[ + \"first/ns-c/ep:5001\",\"first/ns-c/ep:5002\",\"first/ns-c/ep:p1\" +]" + +] +--- response_body eval +[ + "{ 2 2 }\n", + "DONE\n", + "{ 0 0 }\n", + "{ 2 2 0 }\n", + "DONE\n", + "{ 0 0 1 }\n", +] From 49f75aed9e0058981d311eebdcdbe68af9bd95f6 Mon Sep 17 00:00:00 2001 From: elijah Date: Thu, 26 Jan 2023 17:05:45 +0800 Subject: [PATCH 08/15] docs: update mtls document --- docs/zh/latest/discovery/kubernetes.md | 86 +++++++++++++++++++++++--- 1 file changed, 76 insertions(+), 10 deletions(-) diff --git a/docs/zh/latest/discovery/kubernetes.md b/docs/zh/latest/discovery/kubernetes.md index 17342882082a..a48c8391d1c6 100644 --- a/docs/zh/latest/discovery/kubernetes.md +++ b/docs/zh/latest/discovery/kubernetes.md @@ -278,16 +278,6 @@ nodes("release/default/plat-dev:port") 调用会得到如下的返回值： ## Q&A -**Q: 为什么只支持配置 token 来访问 Kubernetes APIServer?** - -A: 一般情况下，我们有三种方式可以完成与 Kubernetes APIServer 的认证： - -- mTLS -- Token -- Basic authentication - -因为 lua-resty-http 目前不支持 mTLS, Basic authentication 不被推荐使用，所以当前只实现了 Token 认证方式。 - **Q: APISIX 继承了 NGINX 的多进程模型，是否意味着每个 APISIX 工作进程都会监听 Kubernetes Endpoints？** A: Kubernetes 服务发现只使用特权进程监听 Kubernetes Endpoints，然后将其值存储到 `ngx.shared.DICT` 中，工作进程通过查询 `ngx.shared.DICT` 来获取结果。 @@ -343,3 +333,79 @@ A: 假定你指定的 [_ServiceAccount_](https://kubernetes.io/docs/tasks/config ```shell kubectl -n apisix get secret kubernetes-discovery-token-c64cv -o jsonpath={.data.token} | base64 -d ``` + +**Q: 如何为 [_ServiceAccount_](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/) 创建 TLS 证书进而使用 mTLS 认证? ** + +A: 假定你使用以上的 [_ServiceAccount_](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/) 声明式定义来创建 `ServiceAccount`，请按如下步骤来创建 TLS 证书。 + + 1. 创建 rsa 私钥： + +```shell +openssl genrsa -out k8s_mtls.key 4096 +``` + + 2. 创建证书签名请求（CSR）对象发送到 Kubernetes API + + 使用以下配置文件生成 CSR： + +``` +[req] +default_bits = 2048 +default_md = sha256 +distinguished_name = dn +prompt = no + +[dn] +CN = system:serviceaccount:default:apisix-test +O = system:serviceaccounts + +[v3_ext] +authorityKeyIdentifier = keyid,issuer:always +basicConstraints = CA:TRUE +keyUsage = keyEncipherment,dataEncipherment +extendedKeyUsage = clientAuth +``` + + 创建 CSR： + +```shell +openssl req -new -key k8s_mtls.key -config k8s_mtls_csr.cnf -out k8s_mtls.csr -nodes +``` + + Kubernetes CSR 资源声明式定义如下： + +```yaml +apiVersion: certificates.k8s.io/v1 +kind: CertificateSigningRequest +metadata: + name: k8s-mtls-csr +spec: + groups: + - system:authenticated + request: ${BASE64_CSR} + signerName: kubernetes.io/kube-apiserver-client + usages: + - digital signature + - key encipherment + - client auth +``` + + 获取 csr 文件内容并替换 `BASE64_CSR` 环境变量，创建 Kubernetes CSR 资源： +```shell +export BASE64_CSR=$(cat ./k8s_mtls.csr | base64 | tr -d '\n') +cat k8s_mtls_csr.yaml | envsubst | kubectl apply -f - +``` + + 3. 手动批准证书签名请求（CSR）: + +```shell +kubectl certificate approve k8s-mtls-csr +``` + + 4. 下载颁发的证书并将其保存本地文件 `k8s_mtls.pem`： + +```shell +kubectl get csr k8s-mtls-csr -o jsonpath='{.status.certificate}' | base64 --decode > ./t/certs/k8s_mtls.pem +``` + +现在你可以使用 `k8s_mtls.key` 和 `k8s_mtls.pem` 文件进行 mTLS 认证，如果你想要进一步了解更多 TLS 证书详细内容，请参阅 [_Manage TLS Certificates in a Cluster_](https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster/)。 From c287ceb8366c38484960d1c9c7227d31c8529733 Mon Sep 17 00:00:00 2001 From: elijah Date: Thu, 26 Jan 2023 17:10:49 +0800 Subject: [PATCH 09/15] chore: remove unnecessary file --- .github/workflows/kubernetes-ci.yml | 2 ++ t/certs/k8s_mtls.key | 51 ----------------------------- 2 files changed, 2 insertions(+), 51 deletions(-) delete mode 100644 t/certs/k8s_mtls.key diff --git a/.github/workflows/kubernetes-ci.yml b/.github/workflows/kubernetes-ci.yml index 3ba0385a6070..c9352582ecfa 100644 --- a/.github/workflows/kubernetes-ci.yml +++ b/.github/workflows/kubernetes-ci.yml @@ -68,6 +68,8 @@ jobs: echo 'KUBERNETES_CLIENT_TOKEN='"${KUBERNETES_CLIENT_TOKEN_CONTENT}" echo 'KUBERNETES_CLIENT_TOKEN_FILE='${KUBERNETES_CLIENT_TOKEN_FILE} + openssl genrsa -out ./t/certs/k8s_mtls.key 4096 + openssl req -new -key ./t/certs/k8s_mtls.key -config ./t/certs/k8s_mtls_csr.conf -out ./t/certs/k8s_mtls.csr -nodes BASE64_CSR=$(cat ./t/certs/k8s_mtls.csr | base64 | tr -d '\n') diff --git a/t/certs/k8s_mtls.key b/t/certs/k8s_mtls.key deleted file mode 100644 index 5203cbf6e62c..000000000000 --- a/t/certs/k8s_mtls.key +++ /dev/null @@ -1,51 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIJKQIBAAKCAgEAsQ6RDAiCQjhaZuOxNNquBoKuTvfFTL/J40yvWy1JUpMuF8Dt -CDbt+tgoi7RwWm7QDM01JHAdPlwLYzuFDpRinodWI36DcHlnYXbFoOLvrsEJayB0 -WZgE67zWHIn0+UfT8rTm+IhcS0f4ifBidWmpVnT9WkOzI4agLUNI4YxfU+/88asA -nafOny/kFcsQqQDjjgv5iv0ORUK312Ydv7lVTIdjDb7WOHu08nt5PMq4SS3c1uOc -ypvI0O7uudAc3GUxtPp6VE2Ff2TZoMKI5fY80zYk8JEUkCrOKuMvB+JzN59qPqag -/bxQhL8nPMp12Id4SISHgEmQSuiTt+SUnj6Sp2FiAdMWlcPssucbXlw2yO7PfJNW -xiiTgrf30eGGcw2fkwBQmZkWPDR3snkI/lps2MZVbYDXUEGk0PYMW6wRzOXr1M7Y -MWhxCHlwKzNeb4dH3RMZiAPc3v0upDyzqxvlWI3Q+8pMEqO4e2HjmyDKlZ4fh7jY -rORSugfCh4Fb7q3AwKu1L2fyGkna63B9vYxUDBqgrEtYusK66oO1/CZv5c7xSfL9 -xEpGCQr4Tot60b4QTmmB7ZBF5FvlTXxZS2y49c8Bgdh+Z5pkYEKT0LnabvN6Jurb -HpjFeFxSMFk/WTiad7ovKylNknSAJalyFYvSc4Uq8uumq6VjxV70Btf3L/kCAwEA -AQKCAgA7yB9RYM4abcyYUOjUX8HCkP0pzJ7heyA/jvTmZDQNnYRCoPFnhKR1veZy -xUnqPuAgcmkvFGvLUmjF6KLUeWWTAT8kXOMkZtlxHxk7aWRkLOlxMlQWrC19KxVg -LXR5drcpi8swv1nqUTKp4HBeFH8IohepEpK5vAr5PW6uOSc2wCE3t/VNRguN1C2+ -7gGCDrrGi2SGeUsIN667G8+PnJQgctYPHhBWoe5GsUoOLKRskP+vzspx3oe/g+02 -7/3OrJxL7wa+0C3zBao1jXmLq8h9yBpQ36uBwfI6qNqIXv1oGntVDA3vnlXRfHlT -HUkNAzCpy5W0yyHdKQo3XQaRJ698JK4GgSYBrn0jen2kZ4MHxa1PQIUjGkOwx+Lj -0y4kRpOv0RtacTsZvmg2AtEyMpPJoiQKjsIaTHWYauuaCFjDdjAIw5vz8qAJOc3E -oZvc6ksOizwOfmkHMcK0wO3Mw1ZXAiFBRh3H0T/0UIxAZQUTZzL8ZvVEf1oUz7MC -1bmJ5sAHpeLCgVWycjasaMqoMOULa0D3ljgH59aSuF7xjDC7nLol4h29ks0GNhel -IRMjX94pp7bJJvT7kG5k970ZprL/IEQBgkT1mFYBDSt6U4hbNY4zBxM+Bb5c+FrE -y7lpaFen2XIZswRzsEdqRaanLYzYV8KvjllQWOh1/dVPt0QqwQKCAQEA1z77CVNF -fzarastUS8N/+/6+cqNSWPFr2yEAX68xYIdkLr0wE+0L2qVEyBMwBchXkH+cR1yc -M3ZDjG0bTELuOmIUuJKshHUc7ULnI4u/6RVDIhRGCb85o9ITlrXhkflcem+qnNhP -8zo01h6OLIA8/Z7abl8KISb7IFZqvBDUuTV8ZJHNsHia3wshy9t0FtXTMw4dwpmG -5LDeyD+DdK3c5NOJV4CNEwksKIEEfv9fxSgBQrGM0zYs0Mif3Bk+68jdviyKAymB -d5umQ3oAf5BspILgqMAYAG23Rsymh1qSOgge5Y2XFXrmXmRNet6XWHly8FPwTpyu -7HI6+T8bDBxtDwKCAQEA0pSNLw4/jpTfaijPU46wUkVNPSgGyo6GiDQpLTmlZKBY -mM308ZF2BoCPn3oZEcAhEoHRYJM3rnvequrm7EUzRm4ujWyshPqsILwvVlebvZLA -7QK9IsB5K1dOhh+wKo6Sd8vl6qPpYzbyIeIZ5ZqVkc4xtgBx110e3Dm8C310SHd6 -oVlV0pcSHSWxNye8deUu58ylltG8GyQ1ZKEdrdgoOgPOt/QOfIeKehd10VaO0FoH -2rsUr8FjcXPM1nZEu0jgfx4/xDonI5NOyLhcjHHSNJJ6ZZK6yIMJkURqF4OJtFUT -N+rX86LWa68Q26Ny3E1FweC8cly+gGJ3oP6fBHOidwKCAQEAkoSRl3hyjzTknI6J -9ATBIwT2GMzp6hd0ZrZhLcDwqJeg71vi1T1/2zayykqdXuJQENF8zOrSfQlc08MV -WOUaiu1m03ksLAIYTb1D32NqoDvT9ee9ATeFAemnYeivApGujL1kct2gFp4hhztH -Zi1o1u+CGq1hFM4XFZ0djKUcWHJ7GW5SiHSgX4GRYNYigMTOvis2wNeHBjcLf14U -mqEbAzynecPMxA9zWqLwL46gX3Fq4j4/8SooeXqHrugUd6XEo4Zn790Z/xQZqkbq -/eqOubFJQjV4YRpOLshx9yME9LHAblviY0HEfK5kO2Y3uYb6YXOR2cE4JUVgtXdq -buUOqwKCAQEAork+XFVjUbfC6RL+GzCRGhIJfK+4h8AZP5knFJaOMzMhI3tLw3xH -OwEPewnr42aXZ+i/NiStC30W9PauWXekBtHP0WNf8SaP6n/hCJSMebK0uLG8G+Wl -/JLr0AMeq9QeZMpAMdBV4mAsPXit0FpjKBexHFY9olGILl/C6xIX0IPbVGFnS2OI -ENP2nIttO6FNXLwsXV5XQJX7WHaUK6l0dFhK3qWOqDZJEM2eMpTzqRdZKHIWqzYC -tUHp0PP1atAHAtQKLIVPh1Mmtz6X7cNqz0s9n84B3j48H3BBl0gDGkH/pt8n31I+ -5VmT6lXkxU5FEGus06bNYwI0NUls8hC1QwKCAQBKidO71ay/NQQJGzaiZMSCQCsI -pAAu/5FCGRjkui1MoUr0Tu8tDIop8hEl6ri36++Me7rYTvI13P5QUrQCM6mSLBgh -m1kqFyGWSIqiTWgb37A/Dfxbahzs/Sg9IZzs3rFD/kgvU3vsF5d2eBuVRN/xr//7 -teugW/xh1O7DUN3cdeInxnFLmII07+wtEKNO+Cu0dLBzAe4GCKOWCgmsW7NOKbt7 -sPuVoV4KgkxEdm4SBUt8A3q1jFunB8ESymBKx9PqiwbhthCpd0jfx9zAOtv4KnEg -a7IxAZtTc9eLlDEGn1G+BZJCZT1A0DgDFiipPCeJxDTLQI4gle2VUuDKYuwN ------END RSA PRIVATE KEY----- From 24cee3e371e8940787ca8a2e0461a88644091069 Mon Sep 17 00:00:00 2001 From: elijah Date: Thu, 26 Jan 2023 17:13:25 +0800 Subject: [PATCH 10/15] chore: refine test --- t/kubernetes/discovery/kubernetes_mtls.t | 34 +----------------------- 1 file changed, 1 insertion(+), 33 deletions(-) diff --git a/t/kubernetes/discovery/kubernetes_mtls.t b/t/kubernetes/discovery/kubernetes_mtls.t index 118147c75ef8..940231faa5fd 100644 --- a/t/kubernetes/discovery/kubernetes_mtls.t +++ b/t/kubernetes/discovery/kubernetes_mtls.t @@ -393,39 +393,7 @@ qr{ 2 2 2 2 2 2 } -=== TEST 4: use client tls without specify ssl protocol ---- yaml_config -apisix: - node_listen: 1984 -deployment: - role: data_plane - role_data_plane: - config_provider: yaml -discovery: - kubernetes: - - id: first - service: - schema: "https" - host: ${KUBERNETES_SERVICE_HOST} - port: ${KUBERNETES_SERVICE_PORT} - client: - cert_file: ${KUBERNETES_CLIENT_CERT} - key_file: ${KUBERNETES_CLIENT_KEY} - ssl_verify: true - ---- request -GET /queries -[ - "first/ns-a/ep:p1","first/ns-a/ep:p2","first/ns-b/ep:p1","first/ns-b/ep:p2","first/ns-c/ep:5001","first/ns-c/ep:5002" -] ---- more_headers -Content-type: application/json ---- response_body eval -qr{ 2 2 2 2 2 2 } - - - -=== TEST 5: scale endpoints +=== TEST 4: scale endpoints --- yaml_config apisix: node_listen: 1984 From 3286feb1f4e460ae97bf816ff607d26aadb97f44 Mon Sep 17 00:00:00 2001 From: elijah Date: Thu, 26 Jan 2023 17:23:06 +0800 Subject: [PATCH 11/15] doc: improve the document --- .github/workflows/kubernetes-ci.yml | 4 ++-- docs/zh/latest/discovery/kubernetes.md | 14 ++++++++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/kubernetes-ci.yml b/.github/workflows/kubernetes-ci.yml index c9352582ecfa..9e11ea0bee51 100644 --- a/.github/workflows/kubernetes-ci.yml +++ b/.github/workflows/kubernetes-ci.yml @@ -76,9 +76,9 @@ jobs: cat ./t/certs/k8s_mtls_csr.yaml | envsubst | kubectl apply -f - - k certificate approve k8s-mtls-csr + kubectl certificate approve k8s-mtls-csr - k get csr k8s-mtls-csr -o jsonpath='{.status.certificate}' | base64 --decode > ./t/certs/k8s_mtls.pem + kubectl get csr k8s-mtls-csr -o jsonpath='{.status.certificate}' | base64 --decode > ./t/certs/k8s_mtls.pem kubectl get secrets | grep apisix-test | awk '{system("kubectl get secret -o jsonpath=\"{.data.ca\.crt}\" "$1" | base64 -d")}' > ./t/certs/k8s_mtls_ca.pem diff --git a/docs/zh/latest/discovery/kubernetes.md b/docs/zh/latest/discovery/kubernetes.md index a48c8391d1c6..a93d29d05588 100644 --- a/docs/zh/latest/discovery/kubernetes.md +++ b/docs/zh/latest/discovery/kubernetes.md @@ -334,7 +334,7 @@ A: 假定你指定的 [_ServiceAccount_](https://kubernetes.io/docs/tasks/config kubectl -n apisix get secret kubernetes-discovery-token-c64cv -o jsonpath={.data.token} | base64 -d ``` -**Q: 如何为 [_ServiceAccount_](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/) 创建 TLS 证书进而使用 mTLS 认证? ** +**Q: 如何为 [_ServiceAccount_](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/) 创建 TLS 证书进而使用 mTLS 认证?** A: 假定你使用以上的 [_ServiceAccount_](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/) 声明式定义来创建 `ServiceAccount`，请按如下步骤来创建 TLS 证书。 @@ -346,7 +346,7 @@ openssl genrsa -out k8s_mtls.key 4096 2. 创建证书签名请求（CSR）对象发送到 Kubernetes API - 使用以下配置文件生成 CSR： + CSR 配置文件如下： ``` [req] @@ -405,7 +405,13 @@ kubectl certificate approve k8s-mtls-csr 4. 下载颁发的证书并将其保存本地文件 `k8s_mtls.pem`： ```shell -kubectl get csr k8s-mtls-csr -o jsonpath='{.status.certificate}' | base64 --decode > ./t/certs/k8s_mtls.pem +kubectl get csr k8s-mtls-csr -o jsonpath='{.status.certificate}' | base64 --decode > k8s_mtls.pem ``` -现在你可以使用 `k8s_mtls.key` 和 `k8s_mtls.pem` 文件进行 mTLS 认证，如果你想要进一步了解更多 TLS 证书详细内容，请参阅 [_Manage TLS Certificates in a Cluster_](https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster/)。 + 5. 下载 CA 证书文件： + +```shell +kubectl get secrets | grep apisix-test | awk '{system("kubectl get secret -o jsonpath=\"{.data.ca\.crt}\" "$1" | base64 -d")}' > k8s_mtls_ca.pem +``` + +现在你可以使用证书文件 `k8s_mtls.pem`，证书私钥 `k8s_mtls.key` 及 CA 证书 `k8s_mtls_ca.pem` 文件进行 mTLS 认证，如果你想要进一步了解更多 TLS 证书详细内容，请参阅 [_Manage TLS Certificates in a Cluster_](https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster/)。 From 839d924460f976ee5170a31abf6d50c96b1c4b06 Mon Sep 17 00:00:00 2001 From: elijah Date: Fri, 27 Jan 2023 15:30:50 +0800 Subject: [PATCH 12/15] chore: move the document part into a new PR --- docs/zh/latest/discovery/kubernetes.md | 92 +++----------------------- 1 file changed, 10 insertions(+), 82 deletions(-) diff --git a/docs/zh/latest/discovery/kubernetes.md b/docs/zh/latest/discovery/kubernetes.md index a93d29d05588..17342882082a 100644 --- a/docs/zh/latest/discovery/kubernetes.md +++ b/docs/zh/latest/discovery/kubernetes.md @@ -278,6 +278,16 @@ nodes("release/default/plat-dev:port") 调用会得到如下的返回值： ## Q&A +**Q: 为什么只支持配置 token 来访问 Kubernetes APIServer?** + +A: 一般情况下，我们有三种方式可以完成与 Kubernetes APIServer 的认证： + +- mTLS +- Token +- Basic authentication + +因为 lua-resty-http 目前不支持 mTLS, Basic authentication 不被推荐使用，所以当前只实现了 Token 认证方式。 + **Q: APISIX 继承了 NGINX 的多进程模型，是否意味着每个 APISIX 工作进程都会监听 Kubernetes Endpoints？** A: Kubernetes 服务发现只使用特权进程监听 Kubernetes Endpoints，然后将其值存储到 `ngx.shared.DICT` 中，工作进程通过查询 `ngx.shared.DICT` 来获取结果。 @@ -333,85 +343,3 @@ A: 假定你指定的 [_ServiceAccount_](https://kubernetes.io/docs/tasks/config ```shell kubectl -n apisix get secret kubernetes-discovery-token-c64cv -o jsonpath={.data.token} | base64 -d ``` - -**Q: 如何为 [_ServiceAccount_](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/) 创建 TLS 证书进而使用 mTLS 认证?** - -A: 假定你使用以上的 [_ServiceAccount_](https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/) 声明式定义来创建 `ServiceAccount`，请按如下步骤来创建 TLS 证书。 - - 1. 创建 rsa 私钥： - -```shell -openssl genrsa -out k8s_mtls.key 4096 -``` - - 2. 创建证书签名请求（CSR）对象发送到 Kubernetes API - - CSR 配置文件如下： - -``` -[req] -default_bits = 2048 -default_md = sha256 -distinguished_name = dn -prompt = no - -[dn] -CN = system:serviceaccount:default:apisix-test -O = system:serviceaccounts - -[v3_ext] -authorityKeyIdentifier = keyid,issuer:always -basicConstraints = CA:TRUE -keyUsage = keyEncipherment,dataEncipherment -extendedKeyUsage = clientAuth -``` - - 创建 CSR： - -```shell -openssl req -new -key k8s_mtls.key -config k8s_mtls_csr.cnf -out k8s_mtls.csr -nodes -``` - - Kubernetes CSR 资源声明式定义如下： - -```yaml -apiVersion: certificates.k8s.io/v1 -kind: CertificateSigningRequest -metadata: - name: k8s-mtls-csr -spec: - groups: - - system:authenticated - request: ${BASE64_CSR} - signerName: kubernetes.io/kube-apiserver-client - usages: - - digital signature - - key encipherment - - client auth -``` - - 获取 csr 文件内容并替换 `BASE64_CSR` 环境变量，创建 Kubernetes CSR 资源： -```shell -export BASE64_CSR=$(cat ./k8s_mtls.csr | base64 | tr -d '\n') -cat k8s_mtls_csr.yaml | envsubst | kubectl apply -f - -``` - - 3. 手动批准证书签名请求（CSR）: - -```shell -kubectl certificate approve k8s-mtls-csr -``` - - 4. 下载颁发的证书并将其保存本地文件 `k8s_mtls.pem`： - -```shell -kubectl get csr k8s-mtls-csr -o jsonpath='{.status.certificate}' | base64 --decode > k8s_mtls.pem -``` - - 5. 下载 CA 证书文件： - -```shell -kubectl get secrets | grep apisix-test | awk '{system("kubectl get secret -o jsonpath=\"{.data.ca\.crt}\" "$1" | base64 -d")}' > k8s_mtls_ca.pem -``` - -现在你可以使用证书文件 `k8s_mtls.pem`，证书私钥 `k8s_mtls.key` 及 CA 证书 `k8s_mtls_ca.pem` 文件进行 mTLS 认证，如果你想要进一步了解更多 TLS 证书详细内容，请参阅 [_Manage TLS Certificates in a Cluster_](https://kubernetes.io/docs/tasks/tls/managing-tls-in-a-cluster/)。 From 97c996d08d363d62d01d978552710eba6416e826 Mon Sep 17 00:00:00 2001 From: elijah Date: Sat, 28 Jan 2023 11:41:14 +0800 Subject: [PATCH 13/15] chore: fix kubernetes ci --- .github/workflows/kubernetes-ci.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/kubernetes-ci.yml b/.github/workflows/kubernetes-ci.yml index 9e11ea0bee51..a33502fa358e 100644 --- a/.github/workflows/kubernetes-ci.yml +++ b/.github/workflows/kubernetes-ci.yml @@ -72,15 +72,17 @@ jobs: openssl req -new -key ./t/certs/k8s_mtls.key -config ./t/certs/k8s_mtls_csr.conf -out ./t/certs/k8s_mtls.csr -nodes - BASE64_CSR=$(cat ./t/certs/k8s_mtls.csr | base64 | tr -d '\n') + export BASE64_CSR=$(cat ./t/certs/k8s_mtls.csr | base64 | tr -d '\n') - cat ./t/certs/k8s_mtls_csr.yaml | envsubst | kubectl apply -f - + envsubst < ./t/certs/k8s_mtls_csr.yaml | kubectl apply -f - kubectl certificate approve k8s-mtls-csr kubectl get csr k8s-mtls-csr -o jsonpath='{.status.certificate}' | base64 --decode > ./t/certs/k8s_mtls.pem - kubectl get secrets | grep apisix-test | awk '{system("kubectl get secret -o jsonpath=\"{.data.ca\.crt}\" "$1" | base64 -d")}' > ./t/certs/k8s_mtls_ca.pem + kubectl get secrets | grep mtls-client | awk '{system("kubectl get secret -o jsonpath=\"{.items[0].data}\" $1")}'| jq -r '."ca.crt"' | base64 -d > ./t/certs/k8s_mtls_ca.pem + + cat ./t/certs/k8s_mtls_ca.pem export KUBERNETES_APISERVER_ADDR=$(kubectl -n kube-system get pod -l component=kube-apiserver -o=jsonpath="{.items[0].metadata.annotations.kubeadm\.kubernetes\.io/kube-apiserver\.advertise-address\.endpoint}" | awk -F: '{print $1}') From 3d4207298e4cc472beb7bb784d2e8361459b2bfc Mon Sep 17 00:00:00 2001 From: elijah Date: Sat, 28 Jan 2023 11:58:23 +0800 Subject: [PATCH 14/15] chore: fix kubernetes test chore: fix kubernetes ci environment variables chore: fix kubernetes ci environment variables --- .github/workflows/kubernetes-ci.yml | 4 ---- ci/kubernetes-ci.sh | 1 + t/kubernetes/discovery/kubernetes.t | 3 ++- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/kubernetes-ci.yml b/.github/workflows/kubernetes-ci.yml index a33502fa358e..84b396cb0b81 100644 --- a/.github/workflows/kubernetes-ci.yml +++ b/.github/workflows/kubernetes-ci.yml @@ -82,10 +82,6 @@ jobs: kubectl get secrets | grep mtls-client | awk '{system("kubectl get secret -o jsonpath=\"{.items[0].data}\" $1")}'| jq -r '."ca.crt"' | base64 -d > ./t/certs/k8s_mtls_ca.pem - cat ./t/certs/k8s_mtls_ca.pem - - export KUBERNETES_APISERVER_ADDR=$(kubectl -n kube-system get pod -l component=kube-apiserver -o=jsonpath="{.items[0].metadata.annotations.kubeadm\.kubernetes\.io/kube-apiserver\.advertise-address\.endpoint}" | awk -F: '{print $1}') - kubectl proxy -p 6445 & - name: Linux Install diff --git a/ci/kubernetes-ci.sh b/ci/kubernetes-ci.sh index c40b8c78c897..6341a66fc2c3 100755 --- a/ci/kubernetes-ci.sh +++ b/ci/kubernetes-ci.sh @@ -21,6 +21,7 @@ run_case() { export_or_prefix export PERL5LIB=.:$PERL5LIB + export KUBERNETES_APISERVER_ADDR=$(kubectl -n kube-system get pod -l component=kube-apiserver -o=jsonpath="{.items[0].metadata.annotations.kubeadm\.kubernetes\.io/kube-apiserver\.advertise-address\.endpoint}" | awk -F: '{print $1}') prove -Itest-nginx/lib -I./ -r t/kubernetes | tee test-result rerun_flaky_tests test-result } diff --git a/t/kubernetes/discovery/kubernetes.t b/t/kubernetes/discovery/kubernetes.t index 6c8196f67cea..5e818722b69d 100644 --- a/t/kubernetes/discovery/kubernetes.t +++ b/t/kubernetes/discovery/kubernetes.t @@ -22,6 +22,7 @@ no_root_location(); no_shuffle(); workers(4); +our $apiserver_addr = $ENV{'KUBERNETES_APISERVER_ADDR'}; our $token_file = "/tmp/var/run/secrets/kubernetes.io/serviceaccount/token"; our $token_value = eval {`cat $token_file 2>/dev/null`}; our $cert_file = "./t/certs/k8s_mtls.pem"; @@ -40,7 +41,7 @@ _EOC_ my $main_config = $block->main_config // <<_EOC_; env MyPort=6443; -env KUBERNETES_SERVICE_HOST=127.0.0.1; +env KUBERNETES_SERVICE_HOST=$::apiserver_addr; env KUBERNETES_SERVICE_PORT=6443; env KUBERNETES_CLIENT_TOKEN=$::token_value; env KUBERNETES_CLIENT_CERT=$::cert_file; From 8ef44e2d6a5962a74c6554218e9f7a05ce0dae50 Mon Sep 17 00:00:00 2001 From: elijah Date: Sat, 28 Jan 2023 16:22:10 +0800 Subject: [PATCH 15/15] chore: fix test --- .github/workflows/kubernetes-ci.yml | 14 -------------- t/kubernetes/discovery/kubernetes.t | 8 ++++++-- t/kubernetes/discovery/kubernetes_mtls.t | 5 ++--- 3 files changed, 8 insertions(+), 19 deletions(-) diff --git a/.github/workflows/kubernetes-ci.yml b/.github/workflows/kubernetes-ci.yml index 84b396cb0b81..ea72fe57144c 100644 --- a/.github/workflows/kubernetes-ci.yml +++ b/.github/workflows/kubernetes-ci.yml @@ -68,20 +68,6 @@ jobs: echo 'KUBERNETES_CLIENT_TOKEN='"${KUBERNETES_CLIENT_TOKEN_CONTENT}" echo 'KUBERNETES_CLIENT_TOKEN_FILE='${KUBERNETES_CLIENT_TOKEN_FILE} - openssl genrsa -out ./t/certs/k8s_mtls.key 4096 - - openssl req -new -key ./t/certs/k8s_mtls.key -config ./t/certs/k8s_mtls_csr.conf -out ./t/certs/k8s_mtls.csr -nodes - - export BASE64_CSR=$(cat ./t/certs/k8s_mtls.csr | base64 | tr -d '\n') - - envsubst < ./t/certs/k8s_mtls_csr.yaml | kubectl apply -f - - - kubectl certificate approve k8s-mtls-csr - - kubectl get csr k8s-mtls-csr -o jsonpath='{.status.certificate}' | base64 --decode > ./t/certs/k8s_mtls.pem - - kubectl get secrets | grep mtls-client | awk '{system("kubectl get secret -o jsonpath=\"{.items[0].data}\" $1")}'| jq -r '."ca.crt"' | base64 -d > ./t/certs/k8s_mtls_ca.pem - kubectl proxy -p 6445 & - name: Linux Install diff --git a/t/kubernetes/discovery/kubernetes.t b/t/kubernetes/discovery/kubernetes.t index 5e818722b69d..365e377b7119 100644 --- a/t/kubernetes/discovery/kubernetes.t +++ b/t/kubernetes/discovery/kubernetes.t @@ -22,7 +22,11 @@ no_root_location(); no_shuffle(); workers(4); -our $apiserver_addr = $ENV{'KUBERNETES_APISERVER_ADDR'}; +system('grep client-cert ~/.kube/config |cut -d" " -f 6 | base64 -d > ./t/certs/k8s_mtls.pem'); +system('grep client-key-data ~/.kube/config |cut -d" " -f 6 | base64 -d > ./t/certs/k8s_mtls.key'); +system('grep certificate-authority-data ~/.kube/config |cut -d" " -f 6 | base64 -d > ./t/certs/k8s_mtls_ca.pem'); + +our $apiserver_addr = $ENV{'KUBERNETES_APISERVER_ADDR'} || "127.0.0.1"; our $token_file = "/tmp/var/run/secrets/kubernetes.io/serviceaccount/token"; our $token_value = eval {`cat $token_file 2>/dev/null`}; our $cert_file = "./t/certs/k8s_mtls.pem"; @@ -360,7 +364,7 @@ deployment: role_data_plane: config_provider: yaml ssl: - ssl_trusted_certificate: ${KUBERNETES_CERTIFICATE_AUTHORITY} + ssl_trusted_certificate: t/certs/k8s_mtls_ca.pem ssl_protocols: TLSv1.2 TLSv1.3 discovery: kubernetes: diff --git a/t/kubernetes/discovery/kubernetes_mtls.t b/t/kubernetes/discovery/kubernetes_mtls.t index 940231faa5fd..df7212c6a878 100644 --- a/t/kubernetes/discovery/kubernetes_mtls.t +++ b/t/kubernetes/discovery/kubernetes_mtls.t @@ -349,13 +349,12 @@ Content-type: application/json --- request GET /queries [ - "first/ns-a/ep:p1","first/ns-a/ep:p2","first/ns-b/ep:p1","first/ns-b/ep:p2","first/ns-c/ep:5001","first/ns-c/ep:5002", - "second/ns-a/ep:p1","second/ns-a/ep:p2","second/ns-b/ep:p1","second/ns-b/ep:p2","second/ns-c/ep:5001","second/ns-c/ep:5002" + "first/ns-a/ep:p1","first/ns-a/ep:p2","first/ns-b/ep:p1","first/ns-b/ep:p2","first/ns-c/ep:5001","first/ns-c/ep:5002" ] --- more_headers Content-type: application/json --- response_body eval -qr{ 2 2 2 2 2 2 2 2 2 2 2 2 } +qr{ 2 2 2 2 2 2 }