feat(deployment): provide conf server in control_plane role (#7365)

Signed-off-by: spacewander <spacewanderlzx@gmail.com>

apisix/cli/schema.lua

@@ -272,9 +272,43 @@ local deployment_schema = {
     traditional = {
         properties = {
             etcd = etcd_schema,
+            role_traditional = {
+                properties = {
+                    config_provider = {
+                        enum = {"etcd"}
+                    },
+                },
+                required = {"config_provider"}
+            }
         },
         required = {"etcd"}
     },
+    control_plane = {
+        properties = {
+            etcd = etcd_schema,
+            role_control_plane = {
+                properties = {
+                    config_provider = {
+                        enum = {"etcd"}
+                    },
+                    conf_server = {
+                        properties = {
+                            listen = {
+                                type = "string",
+                                default = "0.0.0.0:9280",
+                            },
+                            cert = { type = "string" },
+                            cert_key = { type = "string" },
+                            client_ca_cert = { type = "string" },
+                        },
+                        required = {"cert", "cert_key"}
+                    },
+                },
+                required = {"config_provider", "conf_server"}
+            }
+        },
+        required = {"etcd", "role_control_plane"}
+    }
 }
 
 

apisix/cli/snippet.lua

@@ -24,7 +24,10 @@ local _M = {}
 
 
 function _M.generate_conf_server(env, conf)
-    if not (conf.deployment and conf.deployment.role == "traditional") then
+    if not (conf.deployment and (
+        conf.deployment.role == "traditional" or
+        conf.deployment.role == "control_plane"))
+    then
         return nil, nil
     end
 
@@ -49,6 +52,17 @@ function _M.generate_conf_server(env, conf)
         end
     end
 
+    local control_plane
+    if conf.deployment.role == "control_plane" then
+        control_plane = conf.deployment.role_control_plane.conf_server
+        control_plane.cert = pl_path.abspath(control_plane.cert)
+        control_plane.cert_key = pl_path.abspath(control_plane.cert_key)
+
+        if control_plane.client_ca_cert then
+            control_plane.client_ca_cert = pl_path.abspath(control_plane.client_ca_cert)
+        end
+    end
+
     local conf_render = template.compile([[
     upstream apisix_conf_backend {
         server 0.0.0.0:80;
@@ -58,7 +72,20 @@ function _M.generate_conf_server(env, conf)
         }
     }
     server {
+        {% if control_plane then %}
+        listen {* control_plane.listen *} ssl;
+        ssl_certificate {* control_plane.cert *};
+        ssl_certificate_key {* control_plane.cert_key *};
+
+        {% if control_plane.client_ca_cert then %}
+        ssl_verify_client on;
+        ssl_client_certificate {* control_plane.client_ca_cert *};
+        {% end %}
+
+        {% else %}
         listen unix:{* home *}/conf/config_listen.sock;
+        {% end %}
+
         access_log off;
 
         set $upstream_host '';
@@ -71,17 +98,20 @@ function _M.generate_conf_server(env, conf)
         location / {
             {% if enable_https then %}
             proxy_pass https://apisix_conf_backend;
+            proxy_ssl_protocols TLSv1.2 TLSv1.3;
             proxy_ssl_server_name on;
+
             {% if sni then %}
             proxy_ssl_name {* sni *};
             {% else %}
             proxy_ssl_name $upstream_host;
             {% end %}
-            proxy_ssl_protocols TLSv1.2 TLSv1.3;
+
             {% if client_cert then %}
             proxy_ssl_certificate {* client_cert *};
             proxy_ssl_certificate_key {* client_cert_key *};
             {% end %}
+
             {% else %}
             proxy_pass http://apisix_conf_backend;
             {% end %}
@@ -107,9 +137,10 @@ function _M.generate_conf_server(env, conf)
     end
 
     return conf_render({
-        sni = etcd.tls and etcd.tls.sni,
-        enable_https = enable_https,
+        sni = tls and tls.sni,
         home = env.apisix_home or ".",
+        control_plane = control_plane,
+        enable_https = enable_https,
         client_cert = client_cert,
         client_cert_key = client_cert_key,
     })

t/cli/test_deployment_control_plane.sh

@@ -19,6 +19,27 @@
 
 . ./t/cli/common.sh
 
+echo '
+deployment:
+    role: control_plane
+    role_control_plane:
+        config_provider: etcd
+        conf_server:
+            cert: t/certs/mtls_server.crt
+    etcd:
+        prefix: "/apisix"
+        host:
+            - http://127.0.0.1:2379
+' > conf/config.yaml
+
+out=$(make init 2>&1 || true)
+if ! echo "$out" | grep 'property "cert_key" is required'; then
+    echo "failed: should check deployment schema during init"
+    exit 1
+fi
+
+echo "passed: should check deployment schema during init"
+
 echo '
 apisix:
     enable_admin: false
@@ -49,3 +70,32 @@ if [ ! $code -eq 200 ]; then
 fi
 
 echo "passed: control_plane should enable Admin API"
+
+echo '
+deployment:
+    role: control_plane
+    role_control_plane:
+        config_provider: etcd
+        conf_server:
+            listen: 0.0.0.0:12345
+            cert: t/certs/mtls_server.crt
+            cert_key: t/certs/mtls_server.key
+            client_ca_cert: t/certs/mtls_ca.crt
+    etcd:
+        prefix: "/apisix"
+        host:
+            - http://127.0.0.1:2379
+' > conf/config.yaml
+
+make run
+sleep 1
+
+code=$(curl -o /dev/null -s -w %{http_code} http://127.0.0.1:9080/apisix/admin/routes -H 'X-API-KEY: edd1c9f034335f136f87ad84b625c8f1')
+make stop
+
+if [ ! $code -eq 200 ]; then
+    echo "failed: could not work with etcd"
+    exit 1
+fi
+
+echo "passed: work well with etcd in control plane"

t/deployment/conf_server.t

@@ -427,3 +427,69 @@ deployment:
             - http://localhost:12345
 --- error_log
 Receive Host: localhost
+
+
+
+=== TEST 10: mTLS for control plane
+--- exec
+curl --cert t/certs/mtls_client.crt --key t/certs/mtls_client.key -k https://localhost:12345/version
+--- response_body eval
+qr/"etcdserver":/
+--- extra_yaml_config
+deployment:
+    role: control_plane
+    role_control_plane:
+        config_provider: etcd
+        conf_server:
+            listen: 0.0.0.0:12345
+            cert: t/certs/mtls_server.crt
+            cert_key: t/certs/mtls_server.key
+            client_ca_cert: t/certs/mtls_ca.crt
+    etcd:
+        prefix: "/apisix"
+        host:
+            - http://127.0.0.1:2379
+
+
+
+=== TEST 11: no client certificate
+--- exec
+curl -k https://localhost:12345/version
+--- response_body eval
+qr/No required SSL certificate was sent/
+--- extra_yaml_config
+deployment:
+    role: control_plane
+    role_control_plane:
+        config_provider: etcd
+        conf_server:
+            listen: 0.0.0.0:12345
+            cert: t/certs/mtls_server.crt
+            cert_key: t/certs/mtls_server.key
+            client_ca_cert: t/certs/mtls_ca.crt
+    etcd:
+        prefix: "/apisix"
+        host:
+            - http://127.0.0.1:2379
+
+
+
+=== TEST 12: wrong client certificate
+--- exec
+curl --cert t/certs/apisix.crt --key t/certs/apisix.key -k https://localhost:12345/version
+--- response_body eval
+qr/The SSL certificate error/
+--- extra_yaml_config
+deployment:
+    role: control_plane
+    role_control_plane:
+        config_provider: etcd
+        conf_server:
+            listen: 0.0.0.0:12345
+            cert: t/certs/mtls_server.crt
+            cert_key: t/certs/mtls_server.key
+            client_ca_cert: t/certs/mtls_ca.crt
+    etcd:
+        prefix: "/apisix"
+        host:
+            - http://127.0.0.1:2379