feat(k8s-discovery): support mTLS

apisix/discovery/kubernetes/informer_factory.lua

@@ -22,6 +22,8 @@ local math = math
 local type = type
 local core = require("apisix.core")
 local http = require("resty.http")
+local patch = require("apisix.patch")
+
 
 local function list_query(informer)
     local arguments = {
@@ -45,15 +47,18 @@ end
 
 
 local function list(httpc, apiserver, informer)
+    local headers = {
+        ["Host"] = apiserver.host .. ":" .. apiserver.port,
+        ["Accept"] = "application/json",
+        ["Connection"] = "keep-alive"
+    }
+    if apiserver.token and apiserver.token ~= "" then
+        headers["Authorization"] = "Bearer " .. apiserver.token
+    end
     local response, err = httpc:request({
         path = informer.path,
         query = list_query(informer),
-        headers = {
-            ["Host"] = apiserver.host .. ":" .. apiserver.port,
-            ["Authorization"] = "Bearer " .. apiserver.token,
-            ["Accept"] = "application/json",
-            ["Connection"] = "keep-alive"
-        }
+        headers = headers
     })
 
     core.log.info("--raw=", informer.path, "?", list_query(informer))
@@ -204,15 +209,18 @@ local function watch(httpc, apiserver, informer)
         local http_seconds = watch_seconds + 120
         httpc:set_timeouts(2000, 3000, http_seconds * 1000)
 
+        local headers = {
+            ["Host"] = apiserver.host .. ":" .. apiserver.port,
+            ["Accept"] = "application/json",
+            ["Connection"] = "keep-alive"
+        }
+        if apiserver.token and apiserver.token ~= "" then
+            headers["Authorization"] = "Bearer " .. apiserver.token
+        end
         local response, err = httpc:request({
             path = informer.path,
             query = watch_query(informer),
-            headers = {
-                ["Host"] = apiserver.host .. ":" .. apiserver.port,
-                ["Authorization"] = "Bearer " .. apiserver.token,
-                ["Accept"] = "application/json",
-                ["Connection"] = "keep-alive"
-            }
+            headers = headers
         })
 
         core.log.info("--raw=", informer.path, "?", watch_query(informer))
@@ -269,12 +277,22 @@ local function list_watch(informer, apiserver)
     informer.fetch_state = "connecting"
     core.log.info("begin to connect ", apiserver.host, ":", apiserver.port)
 
-    ok, message = httpc:connect({
+    local opt = {
         scheme = apiserver.schema,
         host = apiserver.host,
         port = apiserver.port,
-        ssl_verify = false
-    })
+        ssl_verify = apiserver.ssl_verify,
+    }
+    if apiserver.schema == "https" and
+            apiserver.cert and apiserver.cert ~= "" and
+            apiserver.key and apiserver.key ~= "" then
+        opt.ssl_cert_path = apiserver.cert
+        opt.ssl_key_path = apiserver.key
+        opt.ssl_server_name = apiserver.host
+        -- replace tcp socket of http client to support mtls
+        httpc.sock = patch.lua_tcp_socket()
+    end
+    ok, message = httpc:connect(opt)
 
     if not ok then
         informer.fetch_state = "connect failed"

apisix/discovery/kubernetes/init.lua

@@ -270,15 +270,33 @@ local function get_apiserver(conf)
         if err then
             return nil, err
         end
+    elseif conf.client.cert_file and conf.client.key_file then
+        apiserver.cert, err = read_env(conf.client.cert_file)
+        if err then
+            return nil, err
+        end
+        apiserver.key, err = read_env(conf.client.key_file)
+        if err then
+            return nil, err
+        end
     else
-        return nil, "one of [client.token,client.token_file] should be set but none"
+        return nil, "one of [client.token,client.token_file,(client.cert_file,client.key_file)] "..
+                "should be set but none"
+    end
+
+    apiserver.ssl_verify = false
+    if conf.client.ssl_verify then
+        apiserver.ssl_verify = conf.client.ssl_verify
     end
 
     -- remove possible extra whitespace
     apiserver.token = apiserver.token:gsub("%s+", "")
 
-    if apiserver.schema == "https" and apiserver.token == "" then
-        return nil, "apiserver.token should set to non-empty string when service.schema is https"
+    if apiserver.schema == "https" then
+        if apiserver.token == "" and (apiserver.cert == "" or apiserver.key == "") then
+            return nil, "apiserver.token or (apiserver.cert and apiserver.key) "..
+                    "should set to non-empty string when service.schema is https"
+        end
     end
 
     return apiserver

apisix/discovery/kubernetes/schema.lua

@@ -41,13 +41,17 @@ local token_schema = {
     oneOf = token_patterns,
 }
 
-local token_file_schema = {
+local file_schema = {
     type = "string",
     pattern = [[^[^\:*?"<>|]*$]],
     minLength = 1,
     maxLength = 500,
 }
 
+local ssl_verify_schema = {
+    type = "boolean",
+}
+
 local namespace_pattern = [[^[a-z0-9]([-a-z0-9_.]*[a-z0-9])?$]]
 
 local namespace_regex_pattern = [[^[\x21-\x7e]*$]]
@@ -135,7 +139,10 @@ return {
                     type = "object",
                     properties = {
                         token = token_schema,
-                        token_file = token_file_schema,
+                        token_file = file_schema,
+                        cert_file = file_schema,
+                        key_file = file_schema,
+                        ssl_verify = ssl_verify_schema,
                     },
                     default = {
                         token_file = "/var/run/secrets/kubernetes.io/serviceaccount/token"
@@ -145,6 +152,7 @@ return {
                             anyOf = {
                                 { required = { "token" } },
                                 { required = { "token_file" } },
+                                { required = { "cert_file", "key_file" } },
                             }
                         }
                     },
@@ -191,11 +199,15 @@ return {
                         type = "object",
                         properties = {
                             token = token_schema,
-                            token_file = token_file_schema,
+                            token_file = file_schema,
+                            cert_file = file_schema,
+                            key_file = file_schema,
+                            ssl_verify = ssl_verify_schema,
                         },
                         oneOf = {
                             { required = { "token" } },
                             { required = { "token_file" } },
+                            { required = { "cert_file", "key_file" } },
                         },
                     },
                     namespace_selector = namespace_selector_schema,

apisix/patch.lua

@@ -380,5 +380,8 @@ function _M.patch()
     end
 end
 
+function _M.lua_tcp_socket()
+    return luasocket_tcp()
+end
 
 return _M

ci/kubernetes-ci.sh

@@ -21,6 +21,7 @@
 run_case() {
     export_or_prefix
     export PERL5LIB=.:$PERL5LIB
+    export KUBERNETES_APISERVER_ADDR=$(kubectl -n kube-system get pod -l component=kube-apiserver -o=jsonpath="{.items[0].metadata.annotations.kubeadm\.kubernetes\.io/kube-apiserver\.advertise-address\.endpoint}" | awk -F: '{print $1}')
     prove -Itest-nginx/lib -I./ -r t/kubernetes | tee test-result
     rerun_flaky_tests test-result
 }

t/certs/k8s_mtls_csr.conf

@@ -0,0 +1,35 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+[req]
+default_bits = 2048
+default_md = sha256
+distinguished_name = dn
+prompt = no
+
+[dn]
+CN = system:serviceaccount:default:apisix-test
+O = system:serviceaccounts
+
+[v3_ext]
+authorityKeyIdentifier = keyid,issuer:always
+basicConstraints = CA:TRUE
+keyUsage = keyEncipherment,dataEncipherment
+extendedKeyUsage = clientAuth
+
+## openssl genrsa -out k8s_mtls.key 4096
+## openssl req -new -key k8s_mtls.key -config k8s_mtls_csr.conf -out k8s_mtls.csr -nodes
+

t/certs/k8s_mtls_csr.yaml

@@ -0,0 +1,29 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+apiVersion: certificates.k8s.io/v1
+kind: CertificateSigningRequest
+metadata:
+  name: k8s-mtls-csr
+spec:
+  groups:
+  - system:authenticated
+  request: ${BASE64_CSR}
+  signerName: kubernetes.io/kube-apiserver-client
+  usages:
+  - digital signature
+  - key encipherment
+  - client auth