From 54d9e9ce5b612535b876fd6a18c6fe787ad1dc1a Mon Sep 17 00:00:00 2001
From: tzssangglass <tzssangglass@gmail.com>
Date: Mon, 5 Sep 2022 10:04:17 +0800
Subject: [PATCH 01/11] change: move etcd conf under deployment

---
 conf/config-default.yaml             |  55 ++++++-------
 conf/config.yaml                     |  18 ++++-
 docs/en/latest/certificate.md        |  22 +++---
 docs/en/latest/installation-guide.md |   9 ++-
 docs/en/latest/mtls.md               |  12 ++-
 docs/zh/latest/certificate.md        |  22 +++---
 docs/zh/latest/installation-guide.md |   9 ++-
 docs/zh/latest/mtls.md               |  12 ++-
 t/APISIX.pm                          |  16 +++-
 t/chaos/utils/setup_chaos_utils.sh   |  10 ++-
 t/cli/test_ci_only.sh                |  12 ++-
 t/cli/test_etcd.sh                   |  60 ++++++++------
 t/cli/test_etcd_healthcheck.sh       |  14 ++--
 t/cli/test_etcd_mtls.sh              | 113 ++++++++++++++++-----------
 t/cli/test_etcd_tls.sh               |  31 +++++---
 t/cli/test_main.sh                   |  24 ++++--
 t/cli/test_validate_config.sh        |   8 +-
 t/core/config.t                      |  15 ++--
 t/core/config_etcd.t                 | 109 +++++++++++++++++---------
 t/core/etcd-auth-fail.t              |  16 ++--
 t/core/etcd-auth.t                   |  16 ++--
 t/core/etcd-mtls.t                   | 104 +++++++++++++++---------
 t/core/etcd-sync.t                   |  10 ++-
 t/plugin/example.t                   |  15 ++--
 24 files changed, 467 insertions(+), 265 deletions(-)

diff --git a/conf/config-default.yaml b/conf/config-default.yaml
index 89287004f256..6c194064cbf7 100755
--- a/conf/config-default.yaml
+++ b/conf/config-default.yaml
@@ -275,27 +275,6 @@ nginx_config:                     # config for render the template to generate n
       kubernetes: 1m
       tars: 1m
 
-etcd:
-  host:                           # it's possible to define multiple etcd hosts addresses of the same etcd cluster.
-    - "http://127.0.0.1:2379"     # multiple etcd address, if your etcd cluster enables TLS, please use https scheme,
-                                  # e.g. https://127.0.0.1:2379.
-  prefix: /apisix                 # apisix configurations prefix
-  #timeout: 30                    # 30 seconds
-  #resync_delay: 5                # when sync failed and a rest is needed, resync after the configured seconds plus 50% random jitter
-  #health_check_timeout: 10       # etcd retry the unhealthy nodes after the configured seconds
-  startup_retry: 2                # the number of retry to etcd during the startup, default to 2
-  #user: root                     # root username for etcd
-  #password: 5tHkHhYkjr6cQY       # root password for etcd
-  tls:
-    # To enable etcd client certificate you need to build APISIX-Base, see
-    # https://apisix.apache.org/docs/apisix/FAQ#how-do-i-build-the-apisix-base-environment
-    #cert: /path/to/cert          # path of certificate used by the etcd client
-    #key: /path/to/key            # path of key used by the etcd client
-
-    verify: true                  # whether to verify the etcd endpoint certificate when setup a TLS connection to etcd,
-                                  # the default value is true, e.g. the certificate will be verified strictly.
-    #sni:                         # the SNI for etcd TLS requests. If missed, the host part of the URL will be used.
-
 # HashiCorp Vault storage backend for sensitive data retrieval. The config shows an example of what APISIX expects if you
 # wish to integrate Vault for secret (sensetive string, public private keys etc.) retrieval. APISIX communicates with Vault
 # server HTTP APIs. By default, APISIX doesn't need this configuration.
@@ -558,13 +537,27 @@ plugin_attr:
 #  redirect:
 #    https_port: 8443   # the default port for use by HTTP redirects to HTTPS
 
-#deployment:
-#    role: traditional
-#    role_traditional:
-#        config_provider: etcd
-#    etcd:
-#        host:                           # it's possible to define multiple etcd hosts addresses of the same etcd cluster.
-#          - "http://127.0.0.1:2379"     # multiple etcd address, if your etcd cluster enables TLS, please use https scheme,
-#                                        # e.g. https://127.0.0.1:2379.
-#        prefix: /apisix                 # configuration prefix in etcd
-#        timeout: 30                     # 30 seconds
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:                           # it's possible to define multiple etcd hosts addresses of the same etcd cluster.
+      - "http://127.0.0.1:2379"     # multiple etcd address, if your etcd cluster enables TLS, please use https scheme,
+                                    # e.g. https://127.0.0.1:2379.
+    prefix: /apisix                 # configuration prefix in etcd
+    timeout: 30                     # 30 seconds
+    #resync_delay: 5                # when sync failed and a rest is needed, resync after the configured seconds plus 50% random jitter
+    #health_check_timeout: 10       # etcd retry the unhealthy nodes after the configured seconds
+    startup_retry: 2                # the number of retry to etcd during the startup, default to 2
+    #user: root                     # root username for etcd
+    #password: 5tHkHhYkjr6cQY       # root password for etcd
+    tls:
+      # To enable etcd client certificate you need to build APISIX-Base, see
+      # https://apisix.apache.org/docs/apisix/FAQ#how-do-i-build-the-apisix-base-environment
+      #cert: /path/to/cert          # path of certificate used by the etcd client
+      #key: /path/to/key            # path of key used by the etcd client
+
+      verify: true                  # whether to verify the etcd endpoint certificate when setup a TLS connection to etcd,
+                                    # the default value is true, e.g. the certificate will be verified strictly.
+      #sni:                         # the SNI for etcd TLS requests. If missed, the host part of the URL will be used.
diff --git a/conf/config.yaml b/conf/config.yaml
index 421ac0912aa6..6a5f56205a36 100644
--- a/conf/config.yaml
+++ b/conf/config.yaml
@@ -17,13 +17,21 @@
 # If you want to set the specified configuration value, you can set the new
 # in this file. For example if you want to specify the etcd address:
 #
-# etcd:
+# deployment:
+#   role: traditional
+#   role_traditional:
+#     config_provider: etcd
+#   etcd:
 #     host:
 #       - http://127.0.0.1:2379
 #
 # To configure via environment variables, you can use `${{VAR}}` syntax. For instance:
 #
-# etcd:
+# deployment:
+#   role: traditional
+#   role_traditional:
+#     config_provider: etcd
+#   etcd:
 #     host:
 #       - http://${{ETCD_HOST}}:2379
 #
@@ -34,7 +42,11 @@
 # Also, If you want to use default value when the environment variable not set,
 # Use `${{VAR:=default_value}}` instead. For instance:
 #
-# etcd:
+# deployment:
+#   role: traditional
+#   role_traditional:
+#     config_provider: etcd
+#   etcd:
 #     host:
 #       - http://${{ETCD_HOST:=localhost}}:2379
 #
diff --git a/docs/en/latest/certificate.md b/docs/en/latest/certificate.md
index ab02eadcea77..94e74d722d11 100644
--- a/docs/en/latest/certificate.md
+++ b/docs/en/latest/certificate.md
@@ -246,15 +246,19 @@ apisix:
   ssl:
     ssl_trusted_certificate: /path/to/apisix.ca-bundle
 
-etcd:
-  host:
-    - "https://127.0.0.1:12379"
-    - "https://127.0.0.1:22379"
-    - "https://127.0.0.1:32379"
-  tls:
-    cert: /path/to/bar_apisix.crt
-    key: /path/to/bar_apisix.key
-    sni: etcd.cluster.dev
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - "https://127.0.0.1:12379"
+      - "https://127.0.0.1:22379"
+      - "https://127.0.0.1:32379"
+    tls:
+      cert: /path/to/bar_apisix.crt
+      key: /path/to/bar_apisix.key
+      sni: etcd.cluster.dev
 ```
 
 4. Test APISIX Admin API
diff --git a/docs/en/latest/installation-guide.md b/docs/en/latest/installation-guide.md
index fcddb6c4c5e9..a383d2505196 100644
--- a/docs/en/latest/installation-guide.md
+++ b/docs/en/latest/installation-guide.md
@@ -240,8 +240,13 @@ Now, if you decide you want to change the etcd address to `http://foo:2379`, you
 apisix:
   node_listen: 8000
 
-etcd:
-  host: "http://foo:2379"
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - "http://foo:2379"
 ```
 
 :::warning
diff --git a/docs/en/latest/mtls.md b/docs/en/latest/mtls.md
index 124b8e2f08a6..2cf7a87efd4b 100644
--- a/docs/en/latest/mtls.md
+++ b/docs/en/latest/mtls.md
@@ -71,10 +71,14 @@ curl --cacert /data/certs/mtls_ca.crt --key /data/certs/mtls_client.key --cert /
 You need to build [APISIX-Base](./FAQ.md#how-do-i-build-the-apisix-base-environment) and configure `etcd.tls` section if you want APISIX to work on an etcd cluster with mTLS enabled.
 
 ```yaml
-etcd:
-  tls:
-    cert: /data/certs/etcd_client.pem       # path of certificate used by the etcd client
-    key: /data/certs/etcd_client.key        # path of key used by the etcd client
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    tls:
+      cert: /data/certs/etcd_client.pem       # path of certificate used by the etcd client
+      key: /data/certs/etcd_client.key        # path of key used by the etcd client
 ```
 
 If APISIX does not trust the CA certificate that used by etcd server, we need to set up the CA certificate.
diff --git a/docs/zh/latest/certificate.md b/docs/zh/latest/certificate.md
index 5ff5813d9bab..06f7933c2d84 100644
--- a/docs/zh/latest/certificate.md
+++ b/docs/zh/latest/certificate.md
@@ -243,15 +243,19 @@ apisix:
   ssl:
     ssl_trusted_certificate: /path/to/apisix.ca-bundle
 
-etcd:
-  host:
-    - "https://127.0.0.1:12379"
-    - "https://127.0.0.1:22379"
-    - "https://127.0.0.1:32379"
-  tls:
-    cert: /path/to/bar_apisix.crt
-    key: /path/to/bar_apisix.key
-    sni: etcd.cluster.dev
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - "https://127.0.0.1:12379"
+      - "https://127.0.0.1:22379"
+      - "https://127.0.0.1:32379"
+    tls:
+      cert: /path/to/bar_apisix.crt
+      key: /path/to/bar_apisix.key
+      sni: etcd.cluster.dev
 ```
 
 4. 测试 Admin API
diff --git a/docs/zh/latest/installation-guide.md b/docs/zh/latest/installation-guide.md
index 2359045afefe..5b0ea80e3aed 100644
--- a/docs/zh/latest/installation-guide.md
+++ b/docs/zh/latest/installation-guide.md
@@ -234,8 +234,13 @@ apisix:
 apisix:
   node_listen: 8000 # APISIX listening port
 
-etcd:
-  host: "http://foo:2379" # etcd address
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - "http://foo:2379"
 ```
 
 :::warning
diff --git a/docs/zh/latest/mtls.md b/docs/zh/latest/mtls.md
index cc1ad0fcba42..414adb87684f 100644
--- a/docs/zh/latest/mtls.md
+++ b/docs/zh/latest/mtls.md
@@ -71,10 +71,14 @@ curl --cacert /data/certs/mtls_ca.crt --key /data/certs/mtls_client.key --cert /
 你需要构建 [APISIX-Base](./FAQ.md#如何构建-APISIX-Base-环境？)，并且需要在配置文件中设定 `etcd.tls` 来使 ETCD 的双向认证功能正常工作。
 
 ```yaml
-etcd:
-  tls:
-    cert: /data/certs/etcd_client.pem       # path of certificate used by the etcd client
-    key: /data/certs/etcd_client.key        # path of key used by the etcd client
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    tls:
+      cert: /data/certs/etcd_client.pem       # path of certificate used by the etcd client
+      key: /data/certs/etcd_client.key        # path of key used by the etcd client
 ```
 
 如果 APISIX 不信任 etcd server 使用的 CA 证书，我们需要设置 CA 证书。
diff --git a/t/APISIX.pm b/t/APISIX.pm
index 999bcd8a1994..be4f2614286b 100644
--- a/t/APISIX.pm
+++ b/t/APISIX.pm
@@ -90,6 +90,8 @@ my $ssl_ecc_crt = read_file("t/certs/apisix_ecc.crt");
 my $ssl_ecc_key = read_file("t/certs/apisix_ecc.key");
 my $test2_crt = read_file("t/certs/test2.crt");
 my $test2_key = read_file("t/certs/test2.key");
+my $etcd_pem = read_file("t/certs/etcd.pem");
+my $etcd_key = read_file("t/certs/etcd.key");
 $user_yaml_config = <<_EOC_;
 apisix:
   node_listen: 1984
@@ -104,9 +106,13 @@ my $etcd_enable_auth = $ENV{"ETCD_ENABLE_AUTH"} || "false";
 
 if ($etcd_enable_auth eq "true") {
     $user_yaml_config .= <<_EOC_;
-etcd:
-  user: root
-  password: 5tHkHhYkjr6cQY
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    user: root
+    password: 5tHkHhYkjr6cQY
 _EOC_
 }
 
@@ -845,6 +851,10 @@ $ssl_ecc_key
 $test2_crt
 >>> ../conf/cert/test2.key
 $test2_key
+>>> ../conf/cert/etcd.pem
+$etcd_pem
+>>> ../conf/cert/etcd.key
+$etcd_key
 $user_apisix_yaml
 _EOC_
 
diff --git a/t/chaos/utils/setup_chaos_utils.sh b/t/chaos/utils/setup_chaos_utils.sh
index bffc8598e20f..4b41bb6e3fd8 100755
--- a/t/chaos/utils/setup_chaos_utils.sh
+++ b/t/chaos/utils/setup_chaos_utils.sh
@@ -34,9 +34,13 @@ modify_config() {
     DNS_IP=$(kubectl get svc -n kube-system -l k8s-app=kube-dns -o 'jsonpath={..spec.clusterIP}')
     echo "dns_resolver:
   - ${DNS_IP}
-etcd:
-  host:
-  - \"http://etcd.default.svc.cluster.local:2379\"
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - \"http://etcd.default.svc.cluster.local:2379\"
 plugin_attr:
   prometheus:
     enable_export_server: false
diff --git a/t/cli/test_ci_only.sh b/t/cli/test_ci_only.sh
index a440cf255ac2..d7d9f5bd1bbc 100755
--- a/t/cli/test_ci_only.sh
+++ b/t/cli/test_ci_only.sh
@@ -26,10 +26,14 @@
 git checkout conf/config.yaml
 
 echo '
-etcd:
-  host:
-    - "http://127.0.0.1:3379"
-  prefix: "/apisix"
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - "http://127.0.0.1:3379"
+    prefix: "/apisix"
 ' > conf/config.yaml
 
 out=$(make init 2>&1 || true)
diff --git a/t/cli/test_etcd.sh b/t/cli/test_etcd.sh
index 23b5882e946d..033cab5beb0f 100755
--- a/t/cli/test_etcd.sh
+++ b/t/cli/test_etcd.sh
@@ -32,13 +32,17 @@ etcdctl --endpoints=127.0.0.1:2379 auth enable
 etcdctl --endpoints=127.0.0.1:2379 --user=root:apache-api6 del /apisix --prefix
 
 echo '
-etcd:
-  host:
-    - http://127.0.0.1:2379
-  prefix: /apisix
-  timeout: 30
-  user: root
-  password: apache-api6
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - http://127.0.0.1:2379
+    prefix: /apisix
+    timeout: 30
+    user: root
+    password: apache-api6
 ' > conf/config.yaml
 
 make init
@@ -84,10 +88,14 @@ echo "passed: properly handle the error when connecting to etcd without auth"
 git checkout conf/config.yaml
 
 echo '
-etcd:
-  host:
-    - http://127.0.0.1:2389
-  prefix: /apisix
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - http://127.0.0.1:2389
+    prefix: /apisix
 ' > conf/config.yaml
 
 out=$(make init 2>&1 || true)
@@ -102,10 +110,14 @@ echo "passed: Show retry time info successfully"
 git checkout conf/config.yaml
 
 echo '
-etcd:
-  host:
-    - http://127.0.0.1:2389
-  prefix: /apisix
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - http://127.0.0.1:2389
+    prefix: /apisix
 ' > conf/config.yaml
 
 out=$(make init 2>&1 || true)
@@ -129,13 +141,17 @@ etcdctl --endpoints=127.0.0.1:2379 auth enable
 etcdctl --endpoints=127.0.0.1:2379 --user=root:apache-api6 del /apisix --prefix
 
 echo '
-etcd:
-  host:
-    - http://127.0.0.1:2379
-  prefix: /apisix
-  timeout: 30
-  user: root
-  password: apache-api7
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - http://127.0.0.1:2379
+    prefix: /apisix
+    timeout: 30
+    user: root
+    password: apache-api7
 ' > conf/config.yaml
 
 out=$(make init 2>&1 || true)
diff --git a/t/cli/test_etcd_healthcheck.sh b/t/cli/test_etcd_healthcheck.sh
index 7b631afe0052..52b90bc908d2 100755
--- a/t/cli/test_etcd_healthcheck.sh
+++ b/t/cli/test_etcd_healthcheck.sh
@@ -30,11 +30,15 @@ if [ -z "logs/error.log" ]; then
 fi
 
 echo '
-etcd:
-  host:
-    - "http://127.0.0.1:23790"
-    - "http://127.0.0.1:23791"
-    - "http://127.0.0.1:23792"
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - "http://127.0.0.1:23790"
+      - "http://127.0.0.1:23791"
+      - "http://127.0.0.1:23792"
   health_check_timeout: '"$HEALTH_CHECK_RETRY_TIMEOUT"'
   timeout: 2
 ' > conf/config.yaml
diff --git a/t/cli/test_etcd_mtls.sh b/t/cli/test_etcd_mtls.sh
index 371330e939a2..d61d6d517c1f 100755
--- a/t/cli/test_etcd_mtls.sh
+++ b/t/cli/test_etcd_mtls.sh
@@ -25,14 +25,18 @@ exit_if_not_customed_nginx
 
 # etcd mTLS verify
 echo '
-etcd:
-  host:
-    - "https://admin.apisix.dev:22379"
-  prefix: "/apisix"
-  tls:
-    cert: t/certs/mtls_client.crt
-    key: t/certs/mtls_client.key
-    verify: false
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - "https://admin.apisix.dev:22379"
+    prefix: "/apisix"
+    tls:
+      cert: t/certs/mtls_client.crt
+      key: t/certs/mtls_client.key
+      verify: false
   ' > conf/config.yaml
 
 out=$(make init 2>&1 || echo "ouch")
@@ -44,12 +48,16 @@ fi
 echo "passed: certificate verify success expectedly"
 
 echo '
-etcd:
-  host:
-    - "https://admin.apisix.dev:22379"
-  prefix: "/apisix"
-  tls:
-    verify: false
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - "https://admin.apisix.dev:22379"
+    prefix: "/apisix"
+    tls:
+      verify: false
   ' > conf/config.yaml
 
 out=$(make init 2>&1 || echo "ouch")
@@ -65,13 +73,17 @@ echo '
 apisix:
   ssl:
     ssl_trusted_certificate: t/certs/mtls_ca.crt
-etcd:
-  host:
-    - "https://admin.apisix.dev:22379"
-  prefix: "/apisix"
-  tls:
-    cert: t/certs/mtls_client.crt
-    key: t/certs/mtls_client.key
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - "https://admin.apisix.dev:22379"
+    prefix: "/apisix"
+    tls:
+      cert: t/certs/mtls_client.crt
+      key: t/certs/mtls_client.key
   ' > conf/config.yaml
 
 out=$(make init 2>&1 || echo "ouch")
@@ -95,13 +107,17 @@ apisix:
       - addr: 9100
   ssl:
     ssl_trusted_certificate: t/certs/mtls_ca.crt
-etcd:
-  host:
-    - "https://admin.apisix.dev:22379"
-  prefix: "/apisix"
-  tls:
-    cert: t/certs/mtls_client.crt
-    key: t/certs/mtls_client.key
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - "https://admin.apisix.dev:22379"
+    prefix: "/apisix"
+    tls:
+      cert: t/certs/mtls_client.crt
+      key: t/certs/mtls_client.key
   ' > conf/config.yaml
 
 out=$(make init 2>&1 || echo "ouch")
@@ -132,13 +148,17 @@ echo '
 apisix:
   ssl:
     ssl_trusted_certificate: t/certs/mtls_ca.crt
-etcd:
-  host:
-    - "https://127.0.0.1:22379"
-  prefix: "/apisix"
-  tls:
-    cert: t/certs/mtls_client.crt
-    key: t/certs/mtls_client.key
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - "https://127.0.0.1:22379"
+    prefix: "/apisix"
+    tls:
+      cert: t/certs/mtls_client.crt
+      key: t/certs/mtls_client.key
   ' > conf/config.yaml
 
 rm logs/error.log || true
@@ -147,7 +167,7 @@ make run
 sleep 1
 make stop
 
-if ! grep -E 'certificate host mismatch' logs/error.log; then
+if ! grep -E 'upstream SSL certificate does not match \"127.0.0.1\" while SSL handshaking to upstream' logs/error.log; then
     echo "failed: should got certificate host mismatch when use host in etcd.host as sni"
     exit 1
 fi
@@ -161,14 +181,18 @@ echo '
 apisix:
   ssl:
     ssl_trusted_certificate: t/certs/mtls_ca.crt
-etcd:
-  host:
-    - "https://127.0.0.1:22379"
-  prefix: "/apisix"
-  tls:
-    cert: t/certs/mtls_client.crt
-    key: t/certs/mtls_client.key
-    sni: "admin.apisix.dev"
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - "https://127.0.0.1:22379"
+    prefix: "/apisix"
+    tls:
+      cert: t/certs/mtls_client.crt
+      key: t/certs/mtls_client.key
+      sni: "admin.apisix.dev"
   ' > conf/config.yaml
 
 rm logs/error.log || true
@@ -183,4 +207,3 @@ if grep -E 'certificate host mismatch' logs/error.log; then
 fi
 
 echo "passed: specify custom sni instead of using etcd.host"
-
diff --git a/t/cli/test_etcd_tls.sh b/t/cli/test_etcd_tls.sh
index 906a2b91d1b2..39db833f9674 100755
--- a/t/cli/test_etcd_tls.sh
+++ b/t/cli/test_etcd_tls.sh
@@ -27,10 +27,17 @@
 git checkout conf/config.yaml
 
 echo '
-etcd:
-  host:
-    - "https://127.0.0.1:12379"
-  prefix: "/apisix"
+apisix:
+  ssl:
+    ssl_trusted_certificate: t/certs/mtls_ca.crt
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - "https://127.0.0.1:12379"
+    prefix: "/apisix"
   ' > conf/config.yaml
 
 out=$(make init 2>&1 || true)
@@ -46,12 +53,16 @@ echo "passed: Show certificate verify failed info successfully"
 git checkout conf/config.yaml
 
 echo '
-etcd:
-  host:
-    - "https://127.0.0.1:12379"
-  tls:
-    verify: false
-  prefix: "/apisix"
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - "https://127.0.0.1:12379"
+    prefix: "/apisix"
+    tls:
+      verify: false
   ' > conf/config.yaml
 
 out=$(make init 2>&1 || true)
diff --git a/t/cli/test_main.sh b/t/cli/test_main.sh
index 4397b8228810..6a0358405889 100755
--- a/t/cli/test_main.sh
+++ b/t/cli/test_main.sh
@@ -256,9 +256,13 @@ echo "passed: resolve variables wrapped with whitespace"
 
 # support environment variables in local_conf
 echo '
-etcd:
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
     host:
-        - "http://${{ETCD_HOST}}:${{ETCD_PORT}}"
+      - "http://${{ETCD_HOST}}:${{ETCD_PORT}}"
 ' > conf/config.yaml
 
 ETCD_HOST=127.0.0.1 ETCD_PORT=2379 make init
@@ -270,9 +274,13 @@ fi
 
 # don't override user's envs configuration
 echo '
-etcd:
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
     host:
-        - "http://${{ETCD_HOST}}:${{ETCD_PORT}}"
+      - "http://${{ETCD_HOST}}:${{ETCD_PORT}}"
 nginx_config:
     envs:
         - ETCD_HOST
@@ -291,9 +299,13 @@ if ! grep "env ETCD_HOST;" conf/nginx.conf > /dev/null; then
 fi
 
 echo '
-etcd:
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
     host:
-        - "http://${{ETCD_HOST}}:${{ETCD_PORT}}"
+      - "http://${{ETCD_HOST}}:${{ETCD_PORT}}"
 nginx_config:
     envs:
         - ETCD_HOST=1.1.1.1
diff --git a/t/cli/test_validate_config.sh b/t/cli/test_validate_config.sh
index dc5730e829ec..2fe5d40666ae 100755
--- a/t/cli/test_validate_config.sh
+++ b/t/cli/test_validate_config.sh
@@ -205,9 +205,13 @@ fi
 echo "passed: check the realip configuration for batch-requests"
 
 echo '
-etcd:
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
     host:
-        - 127.0.0.1
+      - 127.0.0.1
 ' > conf/config.yaml
 
 out=$(make init 2>&1 || true)
diff --git a/t/core/config.t b/t/core/config.t
index b87fe1224e0c..29d1cc52dc07 100644
--- a/t/core/config.t
+++ b/t/core/config.t
@@ -55,12 +55,15 @@ first plugin: "real-ip"
         }
     }
 --- yaml_config
-etcd:
-  host:
-    - "http://127.0.0.1:2379" # etcd address
-  prefix: "/apisix"             # apisix configurations prefix
-  timeout: 1
-
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - "http://127.0.0.1:2379" # etcd address
+    prefix: "/apisix"           # apisix configurations prefix
+    timeout: 1
 plugins:
   - example-plugin
 
diff --git a/t/core/config_etcd.t b/t/core/config_etcd.t
index 3baefa275de4..1ed164fe40fb 100644
--- a/t/core/config_etcd.t
+++ b/t/core/config_etcd.t
@@ -21,6 +21,13 @@ no_long_string();
 no_root_location();
 log_level("info");
 
+Test::Nginx::Socket::set_http_config_filter(sub {
+    my $config = shift;
+    my $snippet = `./t/bin/gen_snippet.lua conf_server`;
+    $config .= $snippet;
+    return $config;
+});
+
 run_tests;
 
 __DATA__
@@ -29,10 +36,15 @@ __DATA__
 --- yaml_config
 apisix:
   node_listen: 1984
-etcd:
-  host:
-    - "http://127.0.0.1:7777"  -- wrong etcd port
-  timeout: 1
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    prefix: "/apisix"
+    host:
+      - "http://127.0.0.1:7777"  -- wrong etcd port
+    timeout: 1
 --- config
     location /t {
         content_by_lua_block {
@@ -54,9 +66,15 @@ qr/(connection refused){1,}/
 --- yaml_config
 apisix:
   node_listen: 1984
-etcd:
-  host:
-    - "https://127.0.0.1:2379"
+  ssl:
+    ssl_trusted_certificate: t/servroot/conf/cert/etcd.pem
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - "https://127.0.0.1:2379"
 --- extra_init_by_lua
 local health_check = require("resty.etcd.health_check")
 health_check.get_target_status = function()
@@ -73,9 +91,9 @@ end
 --- request
 GET /t
 --- grep_error_log chop
-handshake failed
+peer closed connection in SSL handshake while SSL handshaking to upstream
 --- grep_error_log_out eval
-qr/(handshake failed){1,}/
+qr/(peer closed connection in SSL handshake while SSL handshaking to upstream){1,}/
 
 
 
@@ -83,9 +101,13 @@ qr/(handshake failed){1,}/
 --- yaml_config
 apisix:
   node_listen: 1984
-etcd:
-  host:
-    - "http://127.0.0.1:12379"
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - "http://127.0.0.1:12379"
 --- config
     location /t {
         content_by_lua_block {
@@ -107,9 +129,15 @@ qr/(closed){1,}/
 --- yaml_config
 apisix:
   node_listen: 1984
-etcd:
-  host:
-    - "https://127.0.0.1:12379"
+  ssl:
+    ssl_trusted_certificate: t/servroot/conf/cert/etcd.pem
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - "https://127.0.0.1:12379"
 --- extra_init_by_lua
 local health_check = require("resty.etcd.health_check")
 health_check.get_target_status = function()
@@ -126,9 +154,9 @@ end
 --- request
 GET /t
 --- grep_error_log chop
-18: self signed certificate
+10:certificate has expired
 --- grep_error_log_out eval
-qr/(18: self signed certificate){1,}/
+qr/(10:certificate has expired){1,}/
 
 
 
@@ -137,11 +165,15 @@ qr/(18: self signed certificate){1,}/
 apisix:
   node_listen: 1984
   admin_key: null
-etcd:
-  host:
-    - "https://127.0.0.1:12379"
-  tls:
-    verify: false
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - "https://127.0.0.1:12379"
+    tls:
+      verify: false
 --- config
     location /t {
         content_by_lua_block {
@@ -159,9 +191,8 @@ etcd:
                     "desc": "new route",
                     "uri": "/index.html"
                 }]]
-                )
+            )
 
-            ngx.status = code
             ngx.say(body)
         }
     }
@@ -179,11 +210,15 @@ passed
 apisix:
   node_listen: 1984
   admin_key: null
-etcd:
-  host:
-    - "https://127.0.0.1:12379"
-  tls:
-    verify: false
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - "https://127.0.0.1:12379"
+    tls:
+      verify: false
 --- config
     location /t {
         content_by_lua_block {
@@ -210,12 +245,16 @@ passed
 --- yaml_config
 apisix:
   node_listen: 1984
-etcd:
-  host:
-    - "http://127.0.0.1:1980" -- fake server port
-  timeout: 1
-  user: root                    # root username for etcd
-  password: 5tHkHhYkjr6cQY      # root password for etcd
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - "http://127.0.0.1:1980" -- fake server port
+    timeout: 1
+    user: root                    # root username for etcd
+    password: 5tHkHhYkjr6cQY      # root password for etcd
 --- extra_init_by_lua
 local health_check = require("resty.etcd.health_check")
 health_check.get_target_status = function()
diff --git a/t/core/etcd-auth-fail.t b/t/core/etcd-auth-fail.t
index 3ac2bb82b52b..c85f660dc06d 100644
--- a/t/core/etcd-auth-fail.t
+++ b/t/core/etcd-auth-fail.t
@@ -79,12 +79,16 @@ qr /insufficient credentials code: 401/
         }
     }
 --- yaml_config
-etcd:
-  host:
-    - "http://127.0.0.1:2379"
-  prefix: "/apisix"
-  user: apisix
-  password: abc123
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - "http://127.0.0.1:2379"
+    prefix: "/apisix"
+    user: apisix
+    password: abc123
 --- request
 GET /t
 --- error_log eval
diff --git a/t/core/etcd-auth.t b/t/core/etcd-auth.t
index f2f322db9b47..448893b264ec 100644
--- a/t/core/etcd-auth.t
+++ b/t/core/etcd-auth.t
@@ -85,12 +85,16 @@ test_value
         }
     }
 --- yaml_config
-etcd:
-  host:
-    - "http://127.0.0.1:2379"
-  prefix: "/apisix"
-  user: apisix
-  password: abc123
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - "http://127.0.0.1:2379"
+    prefix: "/apisix"
+    user: apisix
+    password: abc123
 --- request
 GET /t
 --- no_error_log
diff --git a/t/core/etcd-mtls.t b/t/core/etcd-mtls.t
index a004aef04711..09a5c177ae49 100644
--- a/t/core/etcd-mtls.t
+++ b/t/core/etcd-mtls.t
@@ -24,6 +24,12 @@ if ($out !~ m/function:/) {
     plan('no_plan');
 }
 
+Test::Nginx::Socket::set_http_config_filter(sub {
+    my $config = shift;
+    my $snippet = `./t/bin/gen_snippet.lua conf_server`;
+    $config .= $snippet;
+    return $config;
+});
 
 add_block_preprocessor(sub {
     my ($block) = @_;
@@ -39,14 +45,18 @@ __DATA__
 
 === TEST 1: run etcd in init phase
 --- yaml_config
-etcd:
-  host:
-    - "https://127.0.0.1:22379"
-  prefix: "/apisix"
-  tls:
-    cert: t/certs/mtls_client.crt
-    key: t/certs/mtls_client.key
-    verify: false
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - "https://127.0.0.1:22379"
+    prefix: "/apisix"
+    tls:
+      cert: t/certs/mtls_client.crt
+      key: t/certs/mtls_client.key
+      verify: false
 --- init_by_lua_block
     local apisix = require("apisix")
     apisix.http_init()
@@ -90,14 +100,18 @@ init_by_lua:26: 404
 
 === TEST 2: run etcd in init phase (stream)
 --- yaml_config
-etcd:
-  host:
-    - "https://127.0.0.1:22379"
-  prefix: "/apisix"
-  tls:
-    cert: t/certs/mtls_client.crt
-    key: t/certs/mtls_client.key
-    verify: false
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - "https://127.0.0.1:22379"
+    prefix: "/apisix"
+    tls:
+      cert: t/certs/mtls_client.crt
+      key: t/certs/mtls_client.key
+      verify: false
 --- stream_init_by_lua_block
     apisix = require("apisix")
     apisix.stream_init()
@@ -140,14 +154,18 @@ init_by_lua:26: 404
 
 === TEST 3: sync
 --- extra_yaml_config
-etcd:
-  host:
-    - "https://127.0.0.1:22379"
-  prefix: "/apisix"
-  tls:
-    cert: t/certs/mtls_client.crt
-    key: t/certs/mtls_client.key
-    verify: false
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - "https://127.0.0.1:22379"
+    prefix: "/apisix"
+    tls:
+      cert: t/certs/mtls_client.crt
+      key: t/certs/mtls_client.key
+      verify: false
 --- config
     location /t {
         content_by_lua_block {
@@ -196,14 +214,18 @@ waitdir key
 
 === TEST 4: sync (stream)
 --- extra_yaml_config
-etcd:
-  host:
-    - "https://127.0.0.1:22379"
-  prefix: "/apisix"
-  tls:
-    cert: t/certs/mtls_client.crt
-    key: t/certs/mtls_client.key
-    verify: false
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - "https://127.0.0.1:22379"
+    prefix: "/apisix"
+    tls:
+      cert: t/certs/mtls_client.crt
+      key: t/certs/mtls_client.key
+      verify: false
 --- stream_server_config
     content_by_lua_block {
         local core = require("apisix.core")
@@ -245,13 +267,17 @@ waitdir key
 apisix:
   ssl:
     ssl_trusted_certificate: t/certs/mtls_ca.crt
-etcd:
-  host:
-    - "https://127.0.0.1:22379"
-  prefix: "/apisix"
-  tls:
-    cert: t/certs/mtls_client.crt
-    key: t/certs/mtls_client.key
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - "https://127.0.0.1:22379"
+    prefix: "/apisix"
+    tls:
+      cert: t/certs/mtls_client.crt
+      key: t/certs/mtls_client.key
 --- init_by_lua_block
     local apisix = require("apisix")
     apisix.http_init()
diff --git a/t/core/etcd-sync.t b/t/core/etcd-sync.t
index a1e674218f91..28a89b21f6cb 100644
--- a/t/core/etcd-sync.t
+++ b/t/core/etcd-sync.t
@@ -24,9 +24,13 @@ __DATA__
 
 === TEST 1: minus timeout to watch repeatedly
 --- extra_yaml_config
-etcd:
-  host:
-    - "http://127.0.0.1:2379"
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - "http://127.0.0.1:2379"
 --- config
     location /t {
         content_by_lua_block {
diff --git a/t/plugin/example.t b/t/plugin/example.t
index 985aa11f111b..21972d290a08 100644
--- a/t/plugin/example.t
+++ b/t/plugin/example.t
@@ -165,12 +165,15 @@ GET /t
 --- response_body
 plugin name: example-plugin priority: 0
 --- yaml_config
-etcd:
-  host:
-    - "http://127.0.0.1:2379" # etcd address
-  prefix: "/apisix"             # apisix configurations prefix
-  timeout: 1
-
+deployment:
+  role: traditional
+  role_traditional:
+    config_provider: etcd
+  etcd:
+    host:
+      - "http://127.0.0.1:2379" # etcd address
+    prefix: "/apisix"           # apisix configurations prefix
+    timeout: 1
 plugins:
   - example-plugin
   - not-exist-plugin

From d0f0fafa26ffef58e7d00ad13ba23a20bbe4671d Mon Sep 17 00:00:00 2001
From: tzssangglass <tzssangglass@gmail.com>
Date: Tue, 6 Sep 2022 11:06:35 +0800
Subject: [PATCH 02/11] fix CI

---
 apisix/cli/snippet.lua           |  16 +++++
 apisix/core/etcd.lua             |  73 ++++++++++++++++-----
 ci/pod/docker-compose.common.yml | 106 +++++++++++++++----------------
 t/APISIX.pm                      |   7 ++
 t/core/config.t                  |  25 +-------
 t/core/config_etcd.t             |   7 --
 t/core/etcd-mtls.t               |  20 +++---
 t/core/etcd.t                    |   8 +--
 8 files changed, 149 insertions(+), 113 deletions(-)

diff --git a/apisix/cli/snippet.lua b/apisix/cli/snippet.lua
index 6c2414c34311..d1be7dfc49b1 100644
--- a/apisix/cli/snippet.lua
+++ b/apisix/cli/snippet.lua
@@ -70,6 +70,15 @@ function _M.generate_conf_server(env, conf)
         end
     end
 
+    local ssl_trusted_certificate
+    local etcd_tls_verify = etcd.tls.verify
+    if enable_https and etcd_tls_verify then
+        if not conf.apisix.ssl.ssl_trusted_certificate then
+            return nil, "should set ssl_trusted_certificate if etcd tls verify is enabled"
+        end
+        ssl_trusted_certificate = pl_path.abspath(conf.apisix.ssl.ssl_trusted_certificate)
+    end
+
     local conf_render = template.compile([[
     upstream apisix_conf_backend {
         server 0.0.0.0:80;
@@ -113,6 +122,11 @@ function _M.generate_conf_server(env, conf)
             proxy_ssl_protocols TLSv1.2 TLSv1.3;
             proxy_ssl_server_name on;
 
+            {% if etcd_tls_verify then %}
+            proxy_ssl_verify on;
+            proxy_ssl_trusted_certificate {* ssl_trusted_certificate *};
+            {% end %}
+
             {% if sni then %}
             proxy_ssl_name {* sni *};
             {% else %}
@@ -157,6 +171,8 @@ function _M.generate_conf_server(env, conf)
         client_cert = client_cert,
         client_cert_key = client_cert_key,
         trusted_ca_cert = trusted_ca_cert,
+        etcd_tls_verify = etcd_tls_verify,
+        ssl_trusted_certificate = ssl_trusted_certificate,
     })
 end
 
diff --git a/apisix/core/etcd.lua b/apisix/core/etcd.lua
index 7ac08334e2d0..d9d577c0c503 100644
--- a/apisix/core/etcd.lua
+++ b/apisix/core/etcd.lua
@@ -157,7 +157,7 @@ _M.new = new
 -- @treturn table|nil the etcd client, or nil if failed.
 -- @treturn string|nil the configured prefix of etcd keys, or nil if failed.
 -- @treturn nil|string the error message.
-function _M.new_without_proxy()
+local function new_without_proxy()
     local local_conf, err = fetch_local_conf()
     if not local_conf then
         return nil, nil, err
@@ -166,6 +166,7 @@ function _M.new_without_proxy()
     local etcd_conf = clone_tab(local_conf.etcd)
     return _new(etcd_conf)
 end
+_M.new_without_proxy = new_without_proxy
 
 
 -- convert ETCD v3 entry to v2 one
@@ -280,8 +281,14 @@ function _M.watch_format(v3res)
 end
 
 
-function _M.get(key, is_dir)
-    local etcd_cli, prefix, err = new()
+function _M.get(key, is_dir, noproxy)
+    local etcd_cli, prefix, err
+    if noproxy then
+        etcd_cli, prefix, err = new_without_proxy()
+    else
+        etcd_cli, prefix, err = new()
+    end
+
     if not etcd_cli then
         return nil, err
     end
@@ -299,8 +306,14 @@ function _M.get(key, is_dir)
 end
 
 
-local function set(key, value, ttl)
-    local etcd_cli, prefix, err = new()
+local function set(key, value, ttl, noproxy)
+    local etcd_cli, prefix, err
+    if noproxy then
+        etcd_cli, prefix, err = new_without_proxy()
+    else
+        etcd_cli, prefix, err = new()
+    end
+
     if not etcd_cli then
         return nil, err
     end
@@ -343,8 +356,14 @@ end
 _M.set = set
 
 
-function _M.atomic_set(key, value, ttl, mod_revision)
-    local etcd_cli, prefix, err = new()
+function _M.atomic_set(key, value, ttl, mod_revision, noproxy)
+    local etcd_cli, prefix, err
+    if noproxy then
+        etcd_cli, prefix, err = new_without_proxy()
+    else
+        etcd_cli, prefix, err = new()
+    end
+
     if not etcd_cli then
         return nil, err
     end
@@ -402,8 +421,14 @@ function _M.atomic_set(key, value, ttl, mod_revision)
 end
 
 
-function _M.push(key, value, ttl)
-    local etcd_cli, _, err = new()
+function _M.push(key, value, ttl, noproxy)
+    local etcd_cli, _, err
+    if noproxy then
+        etcd_cli, _, err = new_without_proxy()
+    else
+        etcd_cli, _, err = new()
+    end
+
     if not etcd_cli then
         return nil, err
     end
@@ -434,8 +459,14 @@ function _M.push(key, value, ttl)
 end
 
 
-function _M.delete(key)
-    local etcd_cli, prefix, err = new()
+function _M.delete(key, noproxy)
+    local etcd_cli, prefix, err
+    if noproxy then
+        etcd_cli, prefix, err = new_without_proxy()
+    else
+        etcd_cli, prefix, err = new()
+    end
+
     if not etcd_cli then
         return nil, err
     end
@@ -472,8 +503,14 @@ end
 -- --   etcdcluster = "3.5.0",
 -- --   etcdserver = "3.5.0"
 -- -- }
-function _M.server_version()
-    local etcd_cli, _, err = new()
+function _M.server_version(noproxy)
+    local etcd_cli, _, err
+    if noproxy then
+        etcd_cli, _, err = new_without_proxy()
+    else
+        etcd_cli, _, err = new()
+    end
+
     if not etcd_cli then
         return nil, err
     end
@@ -482,8 +519,14 @@ function _M.server_version()
 end
 
 
-function _M.keepalive(id)
-    local etcd_cli, _, err = new()
+function _M.keepalive(id, noproxy)
+    local etcd_cli, _, err
+    if noproxy then
+        etcd_cli, _, err = new_without_proxy()
+    else
+        etcd_cli, _, err = new()
+    end
+
     if not etcd_cli then
         return nil, err
     end
diff --git a/ci/pod/docker-compose.common.yml b/ci/pod/docker-compose.common.yml
index 9e0394a48bd2..bf6783dfd13e 100644
--- a/ci/pod/docker-compose.common.yml
+++ b/ci/pod/docker-compose.common.yml
@@ -30,60 +30,60 @@ services:
       - "3379:2379"
       - "3380:2380"
 
-  etcd:
-    image: bitnami/etcd:3.5.4
-    restart: unless-stopped
-    env_file:
-      - ci/pod/etcd/env/common.env
-    environment:
-        ETCD_ADVERTISE_CLIENT_URLS: http://0.0.0.0:2379
-    ports:
-      - "2379:2379"
-      - "2380:2380"
+  # etcd:
+  #   image: bitnami/etcd:3.5.4
+  #   restart: unless-stopped
+  #   env_file:
+  #     - ci/pod/etcd/env/common.env
+  #   environment:
+  #       ETCD_ADVERTISE_CLIENT_URLS: http://0.0.0.0:2379
+  #   ports:
+  #     - "2379:2379"
+  #     - "2380:2380"
 
-  etcd_tls:
-    image: bitnami/etcd:3.5.4
-    restart: unless-stopped
-    env_file:
-      - ci/pod/etcd/env/common.env
-    environment:
-      ETCD_ADVERTISE_CLIENT_URLS: https://0.0.0.0:12379
-      ETCD_LISTEN_CLIENT_URLS: https://0.0.0.0:12379
-      ETCD_CERT_FILE: /certs/etcd.pem
-      ETCD_KEY_FILE: /certs/etcd.key
-    ports:
-      - "12379:12379"
-      - "12380:12380"
-    volumes:
-      - ./t/certs:/certs
+  # etcd_tls:
+  #   image: bitnami/etcd:3.5.4
+  #   restart: unless-stopped
+  #   env_file:
+  #     - ci/pod/etcd/env/common.env
+  #   environment:
+  #     ETCD_ADVERTISE_CLIENT_URLS: https://0.0.0.0:12379
+  #     ETCD_LISTEN_CLIENT_URLS: https://0.0.0.0:12379
+  #     ETCD_CERT_FILE: /certs/etcd.pem
+  #     ETCD_KEY_FILE: /certs/etcd.key
+  #   ports:
+  #     - "12379:12379"
+  #     - "12380:12380"
+  #   volumes:
+  #     - ./t/certs:/certs
 
-  etcd_mtls:
-    image: bitnami/etcd:3.5.4
-    restart: unless-stopped
-    env_file:
-      - ci/pod/etcd/env/common.env
-    environment:
-      ETCD_ADVERTISE_CLIENT_URLS: https://0.0.0.0:22379
-      ETCD_LISTEN_CLIENT_URLS: https://0.0.0.0:22379
-      ETCD_CERT_FILE: /certs/mtls_server.crt
-      ETCD_KEY_FILE: /certs/mtls_server.key
-      ETCD_CLIENT_CERT_AUTH: "true"
-      ETCD_TRUSTED_CA_FILE: /certs/mtls_ca.crt
-    ports:
-      - "22379:22379"
-      - "22380:22380"
-    volumes:
-      - ./t/certs:/certs
+  # etcd_mtls:
+  #   image: bitnami/etcd:3.5.4
+  #   restart: unless-stopped
+  #   env_file:
+  #     - ci/pod/etcd/env/common.env
+  #   environment:
+  #     ETCD_ADVERTISE_CLIENT_URLS: https://0.0.0.0:22379
+  #     ETCD_LISTEN_CLIENT_URLS: https://0.0.0.0:22379
+  #     ETCD_CERT_FILE: /certs/mtls_server.crt
+  #     ETCD_KEY_FILE: /certs/mtls_server.key
+  #     ETCD_CLIENT_CERT_AUTH: "true"
+  #     ETCD_TRUSTED_CA_FILE: /certs/mtls_ca.crt
+  #   ports:
+  #     - "22379:22379"
+  #     - "22380:22380"
+  #   volumes:
+  #     - ./t/certs:/certs
 
 
-  ## Redis cluster
-  redis-cluster:
-    image: vishnunair/docker-redis-cluster:latest
-    restart: unless-stopped
-    ports:
-      - "5000:6379"
-      - "5002:6380"
-      - "5003:6381"
-      - "5004:6382"
-      - "5005:6383"
-      - "5006:6384"
+  # ## Redis cluster
+  # redis-cluster:
+  #   image: vishnunair/docker-redis-cluster:latest
+  #   restart: unless-stopped
+  #   ports:
+  #     - "5000:6379"
+  #     - "5002:6380"
+  #     - "5003:6381"
+  #     - "5004:6382"
+  #     - "5005:6383"
+  #     - "5006:6384"
diff --git a/t/APISIX.pm b/t/APISIX.pm
index be4f2614286b..26bf7efe66ce 100644
--- a/t/APISIX.pm
+++ b/t/APISIX.pm
@@ -33,6 +33,13 @@ my $nginx_binary = $ENV{'TEST_NGINX_BINARY'} || 'nginx';
 $ENV{TEST_NGINX_HTML_DIR} ||= html_dir();
 $ENV{TEST_NGINX_FAST_SHUTDOWN} ||= 1;
 
+Test::Nginx::Socket::set_http_config_filter(sub {
+    my $config = shift;
+    my $snippet = `$apisix_home/t/bin/gen_snippet.lua conf_server`;
+    $config .= $snippet;
+    return $config;
+});
+
 sub read_file($) {
     my $infile = shift;
     open my $in, "$apisix_home/$infile"
diff --git a/t/core/config.t b/t/core/config.t
index 29d1cc52dc07..0ace5b7b3fa8 100644
--- a/t/core/config.t
+++ b/t/core/config.t
@@ -306,30 +306,7 @@ seq: {"Block style":["Mercury","Venus","Earth","Mars","Jupiter","Saturn","Uranus
 
 
 
-=== TEST 3: allow environment variable
---- config
-    location /t {
-        content_by_lua_block {
-            local config = require("apisix.core").config.local_conf()
-
-            ngx.say(config.apisix.id)
-        }
-    }
---- main_config
-env AID=3;
---- yaml_config
-#nginx_config:
-    #env: AID=3
-apisix:
-    id: ${{ AID }}
---- request
-GET /t
---- response_body
-3
-
-
-
-=== TEST 4: allow integer worker processes
+=== TEST 3: allow integer worker processes
 --- config
     location /t {
         content_by_lua_block {
diff --git a/t/core/config_etcd.t b/t/core/config_etcd.t
index 1ed164fe40fb..5c1d590a14f5 100644
--- a/t/core/config_etcd.t
+++ b/t/core/config_etcd.t
@@ -21,13 +21,6 @@ no_long_string();
 no_root_location();
 log_level("info");
 
-Test::Nginx::Socket::set_http_config_filter(sub {
-    my $config = shift;
-    my $snippet = `./t/bin/gen_snippet.lua conf_server`;
-    $config .= $snippet;
-    return $config;
-});
-
 run_tests;
 
 __DATA__
diff --git a/t/core/etcd-mtls.t b/t/core/etcd-mtls.t
index 09a5c177ae49..62a9f8745d0b 100644
--- a/t/core/etcd-mtls.t
+++ b/t/core/etcd-mtls.t
@@ -61,23 +61,23 @@ deployment:
     local apisix = require("apisix")
     apisix.http_init()
     local etcd = require("apisix.core.etcd")
-    assert(etcd.set("/a", "ab"))
+    assert(etcd.set("/a", "ab", nil, true))
 
-    local res, err = etcd.get("/a")
+    local res, err = etcd.get("/a", nil, true)
     if not res then
         ngx.log(ngx.ERR, err)
         return
     end
     ngx.log(ngx.WARN, res.body.node.value)
 
-    local res, err = etcd.delete("/a")
+    local res, err = etcd.delete("/a", true)
     if not res then
         ngx.log(ngx.ERR, err)
         return
     end
     ngx.log(ngx.WARN, res.status)
 
-    local res, err = etcd.get("/a")
+    local res, err = etcd.get("/a", nil, true)
     if not res then
         ngx.log(ngx.ERR, err)
         return
@@ -116,23 +116,23 @@ deployment:
     apisix = require("apisix")
     apisix.stream_init()
     local etcd = require("apisix.core.etcd")
-    assert(etcd.set("/a", "ab"))
+    assert(etcd.set("/a", "ab", nil, true))
 
-    local res, err = etcd.get("/a")
+    local res, err = etcd.get("/a", nil, true)
     if not res then
         ngx.log(ngx.ERR, err)
         return
     end
     ngx.log(ngx.WARN, res.body.node.value)
 
-    local res, err = etcd.delete("/a")
+    local res, err = etcd.delete("/a", true)
     if not res then
         ngx.log(ngx.ERR, err)
         return
     end
     ngx.log(ngx.WARN, res.status)
 
-    local res, err = etcd.get("/a")
+    local res, err = etcd.get("/a", nil, true)
     if not res then
         ngx.log(ngx.ERR, err)
         return
@@ -282,8 +282,8 @@ deployment:
     local apisix = require("apisix")
     apisix.http_init()
     local etcd = require("apisix.core.etcd")
-    assert(etcd.set("/a", "ab"))
-    local res, err = etcd.get("/a")
+    assert(etcd.set("/a", "ab", nil, true))
+    local res, err = etcd.get("/a", nil, true)
     if not res then
         ngx.log(ngx.ERR, err)
         return
diff --git a/t/core/etcd.t b/t/core/etcd.t
index 1b8f25db4ab2..16c006f79046 100644
--- a/t/core/etcd.t
+++ b/t/core/etcd.t
@@ -379,23 +379,23 @@ ab
     local apisix = require("apisix")
     apisix.http_init()
     local etcd = require("apisix.core.etcd")
-    assert(etcd.set("/a", "ab"))
+    assert(etcd.set("/a", "ab", nil, true))
 
-    local res, err = etcd.get("/a")
+    local res, err = etcd.get("/a", nil, true)
     if not res then
         ngx.log(ngx.ERR, err)
         return
     end
     ngx.log(ngx.WARN, res.body.node.value)
 
-    local res, err = etcd.delete("/a")
+    local res, err = etcd.delete("/a", true)
     if not res then
         ngx.log(ngx.ERR, err)
         return
     end
     ngx.log(ngx.WARN, res.status)
 
-    local res, err = etcd.get("/a")
+    local res, err = etcd.get("/a", nil, true)
     if not res then
         ngx.log(ngx.ERR, err)
         return

From 1372cfd1b7421a29b3660aeb07d5088e66a967b2 Mon Sep 17 00:00:00 2001
From: tzssangglass <tzssangglass@gmail.com>
Date: Tue, 6 Sep 2022 11:07:39 +0800
Subject: [PATCH 03/11] revert yaml

---
 ci/pod/docker-compose.common.yml | 106 +++++++++++++++----------------
 1 file changed, 53 insertions(+), 53 deletions(-)

diff --git a/ci/pod/docker-compose.common.yml b/ci/pod/docker-compose.common.yml
index bf6783dfd13e..9e0394a48bd2 100644
--- a/ci/pod/docker-compose.common.yml
+++ b/ci/pod/docker-compose.common.yml
@@ -30,60 +30,60 @@ services:
       - "3379:2379"
       - "3380:2380"
 
-  # etcd:
-  #   image: bitnami/etcd:3.5.4
-  #   restart: unless-stopped
-  #   env_file:
-  #     - ci/pod/etcd/env/common.env
-  #   environment:
-  #       ETCD_ADVERTISE_CLIENT_URLS: http://0.0.0.0:2379
-  #   ports:
-  #     - "2379:2379"
-  #     - "2380:2380"
+  etcd:
+    image: bitnami/etcd:3.5.4
+    restart: unless-stopped
+    env_file:
+      - ci/pod/etcd/env/common.env
+    environment:
+        ETCD_ADVERTISE_CLIENT_URLS: http://0.0.0.0:2379
+    ports:
+      - "2379:2379"
+      - "2380:2380"
 
-  # etcd_tls:
-  #   image: bitnami/etcd:3.5.4
-  #   restart: unless-stopped
-  #   env_file:
-  #     - ci/pod/etcd/env/common.env
-  #   environment:
-  #     ETCD_ADVERTISE_CLIENT_URLS: https://0.0.0.0:12379
-  #     ETCD_LISTEN_CLIENT_URLS: https://0.0.0.0:12379
-  #     ETCD_CERT_FILE: /certs/etcd.pem
-  #     ETCD_KEY_FILE: /certs/etcd.key
-  #   ports:
-  #     - "12379:12379"
-  #     - "12380:12380"
-  #   volumes:
-  #     - ./t/certs:/certs
+  etcd_tls:
+    image: bitnami/etcd:3.5.4
+    restart: unless-stopped
+    env_file:
+      - ci/pod/etcd/env/common.env
+    environment:
+      ETCD_ADVERTISE_CLIENT_URLS: https://0.0.0.0:12379
+      ETCD_LISTEN_CLIENT_URLS: https://0.0.0.0:12379
+      ETCD_CERT_FILE: /certs/etcd.pem
+      ETCD_KEY_FILE: /certs/etcd.key
+    ports:
+      - "12379:12379"
+      - "12380:12380"
+    volumes:
+      - ./t/certs:/certs
 
-  # etcd_mtls:
-  #   image: bitnami/etcd:3.5.4
-  #   restart: unless-stopped
-  #   env_file:
-  #     - ci/pod/etcd/env/common.env
-  #   environment:
-  #     ETCD_ADVERTISE_CLIENT_URLS: https://0.0.0.0:22379
-  #     ETCD_LISTEN_CLIENT_URLS: https://0.0.0.0:22379
-  #     ETCD_CERT_FILE: /certs/mtls_server.crt
-  #     ETCD_KEY_FILE: /certs/mtls_server.key
-  #     ETCD_CLIENT_CERT_AUTH: "true"
-  #     ETCD_TRUSTED_CA_FILE: /certs/mtls_ca.crt
-  #   ports:
-  #     - "22379:22379"
-  #     - "22380:22380"
-  #   volumes:
-  #     - ./t/certs:/certs
+  etcd_mtls:
+    image: bitnami/etcd:3.5.4
+    restart: unless-stopped
+    env_file:
+      - ci/pod/etcd/env/common.env
+    environment:
+      ETCD_ADVERTISE_CLIENT_URLS: https://0.0.0.0:22379
+      ETCD_LISTEN_CLIENT_URLS: https://0.0.0.0:22379
+      ETCD_CERT_FILE: /certs/mtls_server.crt
+      ETCD_KEY_FILE: /certs/mtls_server.key
+      ETCD_CLIENT_CERT_AUTH: "true"
+      ETCD_TRUSTED_CA_FILE: /certs/mtls_ca.crt
+    ports:
+      - "22379:22379"
+      - "22380:22380"
+    volumes:
+      - ./t/certs:/certs
 
 
-  # ## Redis cluster
-  # redis-cluster:
-  #   image: vishnunair/docker-redis-cluster:latest
-  #   restart: unless-stopped
-  #   ports:
-  #     - "5000:6379"
-  #     - "5002:6380"
-  #     - "5003:6381"
-  #     - "5004:6382"
-  #     - "5005:6383"
-  #     - "5006:6384"
+  ## Redis cluster
+  redis-cluster:
+    image: vishnunair/docker-redis-cluster:latest
+    restart: unless-stopped
+    ports:
+      - "5000:6379"
+      - "5002:6380"
+      - "5003:6381"
+      - "5004:6382"
+      - "5005:6383"
+      - "5006:6384"

From e3463a30624a96f8b44ec69b2a322d426a578137 Mon Sep 17 00:00:00 2001
From: tzssangglass <tzssangglass@gmail.com>
Date: Tue, 6 Sep 2022 18:57:22 +0800
Subject: [PATCH 04/11] fix CI

---
 apisix/core/config_etcd.lua |  2 +-
 apisix/core/etcd.lua        | 95 +++++++++++++++----------------------
 t/cli/test_access_log.sh    |  4 +-
 3 files changed, 42 insertions(+), 59 deletions(-)

diff --git a/apisix/core/config_etcd.lua b/apisix/core/config_etcd.lua
index 85cf8d7f4120..e432b05d950d 100644
--- a/apisix/core/config_etcd.lua
+++ b/apisix/core/config_etcd.lua
@@ -507,7 +507,7 @@ do
         end
 
         local err
-        etcd_cli, err = etcd_apisix.new()
+        etcd_cli, err = etcd_apisix.switch_proxy()
         return etcd_cli, err
     end
 end
diff --git a/apisix/core/etcd.lua b/apisix/core/etcd.lua
index d9d577c0c503..786797b7f65c 100644
--- a/apisix/core/etcd.lua
+++ b/apisix/core/etcd.lua
@@ -31,6 +31,7 @@ local string            = string
 local tonumber          = tonumber
 local ngx_config_prefix = ngx.config.prefix()
 local ngx_socket_tcp    = ngx.socket.tcp
+local ngx_get_phase     = ngx.get_phase
 
 
 local is_http = ngx.config.subsystem == "http"
@@ -169,6 +170,30 @@ end
 _M.new_without_proxy = new_without_proxy
 
 
+local function switch_proxy()
+    if ngx_get_phase() == "init" or ngx_get_phase() == "init_worker" then
+        return new_without_proxy()
+    end
+
+    local etcd_cli, prefix, err = new()
+    if not etcd_cli or err then
+        return etcd_cli, prefix, err
+    end
+
+    if not etcd_cli.unix_socket_proxy then
+        return etcd_cli, prefix, err
+    end
+    local sock = ngx_socket_tcp()
+    local res, _ = sock:connect(etcd_cli.unix_socket_proxy)
+
+    if not res then
+        return new_without_proxy()
+    end
+
+    return etcd_cli, prefix, err
+end
+_M.switch_proxy = switch_proxy
+
 -- convert ETCD v3 entry to v2 one
 local function kvs_to_node(kvs)
     local node = {}
@@ -281,14 +306,8 @@ function _M.watch_format(v3res)
 end
 
 
-function _M.get(key, is_dir, noproxy)
-    local etcd_cli, prefix, err
-    if noproxy then
-        etcd_cli, prefix, err = new_without_proxy()
-    else
-        etcd_cli, prefix, err = new()
-    end
-
+function _M.get(key, is_dir)
+    local etcd_cli, prefix, err = switch_proxy()
     if not etcd_cli then
         return nil, err
     end
@@ -306,14 +325,8 @@ function _M.get(key, is_dir, noproxy)
 end
 
 
-local function set(key, value, ttl, noproxy)
-    local etcd_cli, prefix, err
-    if noproxy then
-        etcd_cli, prefix, err = new_without_proxy()
-    else
-        etcd_cli, prefix, err = new()
-    end
-
+local function set(key, value, ttl)
+    local etcd_cli, prefix, err = switch_proxy()
     if not etcd_cli then
         return nil, err
     end
@@ -356,14 +369,8 @@ end
 _M.set = set
 
 
-function _M.atomic_set(key, value, ttl, mod_revision, noproxy)
-    local etcd_cli, prefix, err
-    if noproxy then
-        etcd_cli, prefix, err = new_without_proxy()
-    else
-        etcd_cli, prefix, err = new()
-    end
-
+function _M.atomic_set(key, value, ttl, mod_revision)
+    local etcd_cli, prefix, err = switch_proxy()
     if not etcd_cli then
         return nil, err
     end
@@ -421,14 +428,8 @@ function _M.atomic_set(key, value, ttl, mod_revision, noproxy)
 end
 
 
-function _M.push(key, value, ttl, noproxy)
-    local etcd_cli, _, err
-    if noproxy then
-        etcd_cli, _, err = new_without_proxy()
-    else
-        etcd_cli, _, err = new()
-    end
-
+function _M.push(key, value, ttl)
+    local etcd_cli, _, err = switch_proxy()
     if not etcd_cli then
         return nil, err
     end
@@ -459,14 +460,8 @@ function _M.push(key, value, ttl, noproxy)
 end
 
 
-function _M.delete(key, noproxy)
-    local etcd_cli, prefix, err
-    if noproxy then
-        etcd_cli, prefix, err = new_without_proxy()
-    else
-        etcd_cli, prefix, err = new()
-    end
-
+function _M.delete(key)
+    local etcd_cli, prefix, err = switch_proxy()
     if not etcd_cli then
         return nil, err
     end
@@ -503,14 +498,8 @@ end
 -- --   etcdcluster = "3.5.0",
 -- --   etcdserver = "3.5.0"
 -- -- }
-function _M.server_version(noproxy)
-    local etcd_cli, _, err
-    if noproxy then
-        etcd_cli, _, err = new_without_proxy()
-    else
-        etcd_cli, _, err = new()
-    end
-
+function _M.server_version()
+    local etcd_cli, _, err = switch_proxy()
     if not etcd_cli then
         return nil, err
     end
@@ -519,14 +508,8 @@ function _M.server_version(noproxy)
 end
 
 
-function _M.keepalive(id, noproxy)
-    local etcd_cli, _, err
-    if noproxy then
-        etcd_cli, _, err = new_without_proxy()
-    else
-        etcd_cli, _, err = new()
-    end
-
+function _M.keepalive(id)
+    local etcd_cli, _, err = switch_proxy()
     if not etcd_cli then
         return nil, err
     end
diff --git a/t/cli/test_access_log.sh b/t/cli/test_access_log.sh
index a71cf47e1f29..ad48dcb4c865 100755
--- a/t/cli/test_access_log.sh
+++ b/t/cli/test_access_log.sh
@@ -57,7 +57,7 @@ if [ $count_test_access_log -eq 0 ]; then
 fi
 
 count_access_log_off=`grep -c "access_log off;" conf/nginx.conf || true`
-if [ $count_access_log_off -eq 4 ]; then
+if [ $count_access_log_off -eq 5 ]; then
     echo "failed: nginx.conf file find access_log off; when enable access log"
     exit 1
 fi
@@ -92,7 +92,7 @@ if [ $count_test_access_log -eq 1 ]; then
 fi
 
 count_access_log_off=`grep -c "access_log off;" conf/nginx.conf || true`
-if [ $count_access_log_off -ne 4 ]; then
+if [ $count_access_log_off -ne 5 ]; then
     echo "failed: nginx.conf file doesn't find access_log off; when disable access log"
     exit 1
 fi

From 1d728ceed8d438cea960fa834c86419a826fb3fc Mon Sep 17 00:00:00 2001
From: tzssangglass <tzssangglass@gmail.com>
Date: Tue, 6 Sep 2022 19:08:59 +0800
Subject: [PATCH 05/11] revert

---
 t/core/etcd-mtls.t | 16 ++++++++--------
 t/core/etcd.t      |  6 +++---
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/t/core/etcd-mtls.t b/t/core/etcd-mtls.t
index 62a9f8745d0b..be472de1d105 100644
--- a/t/core/etcd-mtls.t
+++ b/t/core/etcd-mtls.t
@@ -61,9 +61,9 @@ deployment:
     local apisix = require("apisix")
     apisix.http_init()
     local etcd = require("apisix.core.etcd")
-    assert(etcd.set("/a", "ab", nil, true))
+    assert(etcd.set("/a", "ab"))
 
-    local res, err = etcd.get("/a", nil, true)
+    local res, err = etcd.get("/a")
     if not res then
         ngx.log(ngx.ERR, err)
         return
@@ -77,7 +77,7 @@ deployment:
     end
     ngx.log(ngx.WARN, res.status)
 
-    local res, err = etcd.get("/a", nil, true)
+    local res, err = etcd.get("/a")
     if not res then
         ngx.log(ngx.ERR, err)
         return
@@ -116,9 +116,9 @@ deployment:
     apisix = require("apisix")
     apisix.stream_init()
     local etcd = require("apisix.core.etcd")
-    assert(etcd.set("/a", "ab", nil, true))
+    assert(etcd.set("/a", "ab"))
 
-    local res, err = etcd.get("/a", nil, true)
+    local res, err = etcd.get("/a")
     if not res then
         ngx.log(ngx.ERR, err)
         return
@@ -132,7 +132,7 @@ deployment:
     end
     ngx.log(ngx.WARN, res.status)
 
-    local res, err = etcd.get("/a", nil, true)
+    local res, err = etcd.get("/a")
     if not res then
         ngx.log(ngx.ERR, err)
         return
@@ -282,8 +282,8 @@ deployment:
     local apisix = require("apisix")
     apisix.http_init()
     local etcd = require("apisix.core.etcd")
-    assert(etcd.set("/a", "ab", nil, true))
-    local res, err = etcd.get("/a", nil, true)
+    assert(etcd.set("/a", "ab"))
+    local res, err = etcd.get("/a")
     if not res then
         ngx.log(ngx.ERR, err)
         return
diff --git a/t/core/etcd.t b/t/core/etcd.t
index 16c006f79046..a5771781248d 100644
--- a/t/core/etcd.t
+++ b/t/core/etcd.t
@@ -379,9 +379,9 @@ ab
     local apisix = require("apisix")
     apisix.http_init()
     local etcd = require("apisix.core.etcd")
-    assert(etcd.set("/a", "ab", nil, true))
+    assert(etcd.set("/a", "ab"))
 
-    local res, err = etcd.get("/a", nil, true)
+    local res, err = etcd.get("/a")
     if not res then
         ngx.log(ngx.ERR, err)
         return
@@ -395,7 +395,7 @@ ab
     end
     ngx.log(ngx.WARN, res.status)
 
-    local res, err = etcd.get("/a", nil, true)
+    local res, err = etcd.get("/a")
     if not res then
         ngx.log(ngx.ERR, err)
         return

From 97c97b7ff6fe2befefaa86dd2af8e4984cab13fa Mon Sep 17 00:00:00 2001
From: tzssangglass <tzssangglass@gmail.com>
Date: Tue, 6 Sep 2022 23:28:39 +0800
Subject: [PATCH 06/11] fix CI

---
 apisix/core/etcd.lua                 | 12 +++++-------
 t/cli/test_deployment_traditional.sh | 15 ---------------
 t/deployment/conf_server.t           | 15 ++++++++++++---
 3 files changed, 17 insertions(+), 25 deletions(-)

diff --git a/apisix/core/etcd.lua b/apisix/core/etcd.lua
index 786797b7f65c..00473d79feee 100644
--- a/apisix/core/etcd.lua
+++ b/apisix/core/etcd.lua
@@ -78,7 +78,7 @@ local function _new(etcd_conf)
 end
 
 
-local function new()
+function _M.new()
     local local_conf, err = fetch_local_conf()
     if not local_conf then
         return nil, nil, err
@@ -147,7 +147,6 @@ local function new()
 
     return _new(etcd_conf)
 end
-_M.new = new
 
 
 ---
@@ -158,7 +157,7 @@ _M.new = new
 -- @treturn table|nil the etcd client, or nil if failed.
 -- @treturn string|nil the configured prefix of etcd keys, or nil if failed.
 -- @treturn nil|string the error message.
-local function new_without_proxy()
+function _M.new_without_proxy()
     local local_conf, err = fetch_local_conf()
     if not local_conf then
         return nil, nil, err
@@ -167,15 +166,14 @@ local function new_without_proxy()
     local etcd_conf = clone_tab(local_conf.etcd)
     return _new(etcd_conf)
 end
-_M.new_without_proxy = new_without_proxy
 
 
 local function switch_proxy()
     if ngx_get_phase() == "init" or ngx_get_phase() == "init_worker" then
-        return new_without_proxy()
+        return _M.new_without_proxy()
     end
 
-    local etcd_cli, prefix, err = new()
+    local etcd_cli, prefix, err = _M.new()
     if not etcd_cli or err then
         return etcd_cli, prefix, err
     end
@@ -187,7 +185,7 @@ local function switch_proxy()
     local res, _ = sock:connect(etcd_cli.unix_socket_proxy)
 
     if not res then
-        return new_without_proxy()
+        return _M.new_without_proxy()
     end
 
     return etcd_cli, prefix, err
diff --git a/t/cli/test_deployment_traditional.sh b/t/cli/test_deployment_traditional.sh
index ba6767b87815..1dead769bc10 100755
--- a/t/cli/test_deployment_traditional.sh
+++ b/t/cli/test_deployment_traditional.sh
@@ -19,21 +19,6 @@
 
 . ./t/cli/common.sh
 
-echo '
-deployment:
-    role: traditional
-    role_traditional:
-        config_provider: etcd
-' > conf/config.yaml
-
-out=$(make init 2>&1 || true)
-if ! echo "$out" | grep 'invalid deployment traditional configuration: property "etcd" is required'; then
-    echo "failed: should check deployment schema during init"
-    exit 1
-fi
-
-echo "passed: should check deployment schema during init"
-
 # HTTP
 echo '
 deployment:
diff --git a/t/deployment/conf_server.t b/t/deployment/conf_server.t
index ee52346644c5..d683b3c11446 100644
--- a/t/deployment/conf_server.t
+++ b/t/deployment/conf_server.t
@@ -169,6 +169,15 @@ localhost is resolved to: 127.0.0.2
 
 === TEST 4: update balancer if the DNS result changed
 --- extra_init_by_lua
+    local etcd = require("apisix.core.etcd")
+    package.loaded.proxy_by_conf_server = 0
+    local old_etcd_new = etcd.new
+    etcd.new = function ()
+        local proxy_by_conf_server = package.loaded.proxy_by_conf_server
+        package.loaded.proxy_by_conf_server = proxy_by_conf_server + 1
+        return old_etcd_new()
+    end
+
     local resolver = require("apisix.core.resolver")
     local old_f = resolver.parse_domain
     package.loaded.counter = 0
@@ -200,12 +209,12 @@ localhost is resolved to: 127.0.0.2
             assert(etcd.set("/apisix/test", "foo"))
             local res = assert(etcd.get("/apisix/test"))
             ngx.say(res.body.node.value)
-            local counter = package.loaded.counter
+            local proxy_by_conf_server = package.loaded.proxy_by_conf_server
             local n_picker = package.loaded.n_picker
-            if counter == n_picker then
+            if proxy_by_conf_server == n_picker then
                 ngx.say("OK")
             else
-                ngx.say(counter, " ", n_picker)
+                ngx.say(proxy_by_conf_server, " ", n_picker)
             end
         }
     }

From 24a15c2d8a9f913a33ccd75b6ac5171eaf4c2503 Mon Sep 17 00:00:00 2001
From: tzssangglass <tzssangglass@gmail.com>
Date: Wed, 7 Sep 2022 02:11:32 +0800
Subject: [PATCH 07/11] format

---
 apisix/core/etcd.lua |  5 ++---
 t/core/config.t      | 25 ++++++++++++++++++++++++-
 t/core/etcd-mtls.t   |  4 ++--
 t/core/etcd.t        |  2 +-
 4 files changed, 29 insertions(+), 7 deletions(-)

diff --git a/apisix/core/etcd.lua b/apisix/core/etcd.lua
index 00473d79feee..6b818e9c0799 100644
--- a/apisix/core/etcd.lua
+++ b/apisix/core/etcd.lua
@@ -182,9 +182,8 @@ local function switch_proxy()
         return etcd_cli, prefix, err
     end
     local sock = ngx_socket_tcp()
-    local res, _ = sock:connect(etcd_cli.unix_socket_proxy)
-
-    if not res then
+    local ok = sock:connect(etcd_cli.unix_socket_proxy)
+    if not ok then
         return _M.new_without_proxy()
     end
 
diff --git a/t/core/config.t b/t/core/config.t
index 0ace5b7b3fa8..29d1cc52dc07 100644
--- a/t/core/config.t
+++ b/t/core/config.t
@@ -306,7 +306,30 @@ seq: {"Block style":["Mercury","Venus","Earth","Mars","Jupiter","Saturn","Uranus
 
 
 
-=== TEST 3: allow integer worker processes
+=== TEST 3: allow environment variable
+--- config
+    location /t {
+        content_by_lua_block {
+            local config = require("apisix.core").config.local_conf()
+
+            ngx.say(config.apisix.id)
+        }
+    }
+--- main_config
+env AID=3;
+--- yaml_config
+#nginx_config:
+    #env: AID=3
+apisix:
+    id: ${{ AID }}
+--- request
+GET /t
+--- response_body
+3
+
+
+
+=== TEST 4: allow integer worker processes
 --- config
     location /t {
         content_by_lua_block {
diff --git a/t/core/etcd-mtls.t b/t/core/etcd-mtls.t
index be472de1d105..09a5c177ae49 100644
--- a/t/core/etcd-mtls.t
+++ b/t/core/etcd-mtls.t
@@ -70,7 +70,7 @@ deployment:
     end
     ngx.log(ngx.WARN, res.body.node.value)
 
-    local res, err = etcd.delete("/a", true)
+    local res, err = etcd.delete("/a")
     if not res then
         ngx.log(ngx.ERR, err)
         return
@@ -125,7 +125,7 @@ deployment:
     end
     ngx.log(ngx.WARN, res.body.node.value)
 
-    local res, err = etcd.delete("/a", true)
+    local res, err = etcd.delete("/a")
     if not res then
         ngx.log(ngx.ERR, err)
         return
diff --git a/t/core/etcd.t b/t/core/etcd.t
index a5771781248d..1b8f25db4ab2 100644
--- a/t/core/etcd.t
+++ b/t/core/etcd.t
@@ -388,7 +388,7 @@ ab
     end
     ngx.log(ngx.WARN, res.body.node.value)
 
-    local res, err = etcd.delete("/a", true)
+    local res, err = etcd.delete("/a")
     if not res then
         ngx.log(ngx.ERR, err)
         return

From 8b9fe6a6d31a2416880b143d7ec3dd679c0e0dc5 Mon Sep 17 00:00:00 2001
From: tzssangglass <tzssangglass@gmail.com>
Date: Wed, 7 Sep 2022 14:24:56 +0800
Subject: [PATCH 08/11] fix failed test cases

---
 t/deployment/conf_server.t | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/t/deployment/conf_server.t b/t/deployment/conf_server.t
index d683b3c11446..162deeb59894 100644
--- a/t/deployment/conf_server.t
+++ b/t/deployment/conf_server.t
@@ -170,12 +170,8 @@ localhost is resolved to: 127.0.0.2
 === TEST 4: update balancer if the DNS result changed
 --- extra_init_by_lua
     local etcd = require("apisix.core.etcd")
-    package.loaded.proxy_by_conf_server = 0
-    local old_etcd_new = etcd.new
-    etcd.new = function ()
-        local proxy_by_conf_server = package.loaded.proxy_by_conf_server
-        package.loaded.proxy_by_conf_server = proxy_by_conf_server + 1
-        return old_etcd_new()
+    etcd.switch_proxy = function ()
+        return etcd.new()
     end
 
     local resolver = require("apisix.core.resolver")
@@ -209,12 +205,12 @@ localhost is resolved to: 127.0.0.2
             assert(etcd.set("/apisix/test", "foo"))
             local res = assert(etcd.get("/apisix/test"))
             ngx.say(res.body.node.value)
-            local proxy_by_conf_server = package.loaded.proxy_by_conf_server
+            local counter = package.loaded.counter
             local n_picker = package.loaded.n_picker
-            if proxy_by_conf_server == n_picker then
+            if counter == n_picker then
                 ngx.say("OK")
             else
-                ngx.say(proxy_by_conf_server, " ", n_picker)
+                ngx.say(counter, " ", n_picker)
             end
         }
     }

From f90d9f51a5a50a3fbefde7284630b052e3dc7592 Mon Sep 17 00:00:00 2001
From: tzssangglass <tzssangglass@gmail.com>
Date: Wed, 7 Sep 2022 14:25:05 +0800
Subject: [PATCH 09/11] fix

---
 apisix/core/etcd.lua | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/apisix/core/etcd.lua b/apisix/core/etcd.lua
index 6b818e9c0799..f6b1ddbd5cc4 100644
--- a/apisix/core/etcd.lua
+++ b/apisix/core/etcd.lua
@@ -78,7 +78,7 @@ local function _new(etcd_conf)
 end
 
 
-function _M.new()
+local function new()
     local local_conf, err = fetch_local_conf()
     if not local_conf then
         return nil, nil, err
@@ -147,6 +147,7 @@ function _M.new()
 
     return _new(etcd_conf)
 end
+_M.new = new
 
 
 ---
@@ -157,7 +158,7 @@ end
 -- @treturn table|nil the etcd client, or nil if failed.
 -- @treturn string|nil the configured prefix of etcd keys, or nil if failed.
 -- @treturn nil|string the error message.
-function _M.new_without_proxy()
+local function new_without_proxy()
     local local_conf, err = fetch_local_conf()
     if not local_conf then
         return nil, nil, err
@@ -166,14 +167,15 @@ function _M.new_without_proxy()
     local etcd_conf = clone_tab(local_conf.etcd)
     return _new(etcd_conf)
 end
+_M.new_without_proxy = new_without_proxy
 
 
 local function switch_proxy()
     if ngx_get_phase() == "init" or ngx_get_phase() == "init_worker" then
-        return _M.new_without_proxy()
+        return new_without_proxy()
     end
 
-    local etcd_cli, prefix, err = _M.new()
+    local etcd_cli, prefix, err = new()
     if not etcd_cli or err then
         return etcd_cli, prefix, err
     end
@@ -184,7 +186,7 @@ local function switch_proxy()
     local sock = ngx_socket_tcp()
     local ok = sock:connect(etcd_cli.unix_socket_proxy)
     if not ok then
-        return _M.new_without_proxy()
+        return new_without_proxy()
     end
 
     return etcd_cli, prefix, err

From 15230e1b2873e929e748851a9fb7619d304de327 Mon Sep 17 00:00:00 2001
From: tzssangglass <tzssangglass@gmail.com>
Date: Wed, 7 Sep 2022 17:51:08 +0800
Subject: [PATCH 10/11] resolve code review

---
 apisix/cli/snippet.lua | 30 ++++++++++++++++--------------
 1 file changed, 16 insertions(+), 14 deletions(-)

diff --git a/apisix/cli/snippet.lua b/apisix/cli/snippet.lua
index d1be7dfc49b1..3b5eb3232394 100644
--- a/apisix/cli/snippet.lua
+++ b/apisix/cli/snippet.lua
@@ -39,8 +39,7 @@ function _M.generate_conf_server(env, conf)
     if servers[1]:find(prefix, 1, true) then
         enable_https = true
     end
-    -- there is not a compatible way to verify upstream TLS like the one we do in cosocket
-    -- so here we just ignore it as the verification is already done in the init phase
+
     for i, s in ipairs(servers) do
         if (s:find(prefix, 1, true) ~= nil) ~= enable_https then
             return nil, "all nodes in the etcd cluster should enable/disable TLS together"
@@ -70,15 +69,6 @@ function _M.generate_conf_server(env, conf)
         end
     end
 
-    local ssl_trusted_certificate
-    local etcd_tls_verify = etcd.tls.verify
-    if enable_https and etcd_tls_verify then
-        if not conf.apisix.ssl.ssl_trusted_certificate then
-            return nil, "should set ssl_trusted_certificate if etcd tls verify is enabled"
-        end
-        ssl_trusted_certificate = pl_path.abspath(conf.apisix.ssl.ssl_trusted_certificate)
-    end
-
     local conf_render = template.compile([[
     upstream apisix_conf_backend {
         server 0.0.0.0:80;
@@ -158,9 +148,21 @@ function _M.generate_conf_server(env, conf)
     local tls = etcd.tls
     local client_cert
     local client_cert_key
-    if tls and tls.cert then
-        client_cert = pl_path.abspath(tls.cert)
-        client_cert_key = pl_path.abspath(tls.key)
+    local ssl_trusted_certificate
+    local etcd_tls_verify
+    if tls then
+        if tls.cert then
+            client_cert = pl_path.abspath(tls.cert)
+            client_cert_key = pl_path.abspath(tls.key)
+        end
+
+        etcd_tls_verify = tls.verify
+        if enable_https and etcd_tls_verify then
+            if not conf.apisix.ssl.ssl_trusted_certificate then
+                return nil, "should set ssl_trusted_certificate if etcd tls verify is enabled"
+            end
+            ssl_trusted_certificate = pl_path.abspath(conf.apisix.ssl.ssl_trusted_certificate)
+        end
     end
 
     return conf_render({

From a87fa7c39e2bd1bfb02924e78dd77ab9357665d1 Mon Sep 17 00:00:00 2001
From: tzssangglass <tzssangglass@gmail.com>
Date: Wed, 7 Sep 2022 22:53:54 +0800
Subject: [PATCH 11/11] resolve code review

---
 t/core/etcd-mtls.t          | 7 -------
 t/deployment/conf_server.t  | 7 -------
 t/deployment/conf_server2.t | 7 -------
 t/deployment/mtls.t         | 7 -------
 4 files changed, 28 deletions(-)

diff --git a/t/core/etcd-mtls.t b/t/core/etcd-mtls.t
index 09a5c177ae49..05b3121f9ffc 100644
--- a/t/core/etcd-mtls.t
+++ b/t/core/etcd-mtls.t
@@ -24,13 +24,6 @@ if ($out !~ m/function:/) {
     plan('no_plan');
 }
 
-Test::Nginx::Socket::set_http_config_filter(sub {
-    my $config = shift;
-    my $snippet = `./t/bin/gen_snippet.lua conf_server`;
-    $config .= $snippet;
-    return $config;
-});
-
 add_block_preprocessor(sub {
     my ($block) = @_;
 
diff --git a/t/deployment/conf_server.t b/t/deployment/conf_server.t
index 162deeb59894..cd5353e373f8 100644
--- a/t/deployment/conf_server.t
+++ b/t/deployment/conf_server.t
@@ -29,13 +29,6 @@ add_block_preprocessor(sub {
 
 });
 
-Test::Nginx::Socket::set_http_config_filter(sub {
-    my $config = shift;
-    my $snippet = `./t/bin/gen_snippet.lua conf_server`;
-    $config .= $snippet;
-    return $config;
-});
-
 run_tests();
 
 __DATA__
diff --git a/t/deployment/conf_server2.t b/t/deployment/conf_server2.t
index c9eb0ac9cfd9..b8261c80c31b 100644
--- a/t/deployment/conf_server2.t
+++ b/t/deployment/conf_server2.t
@@ -29,13 +29,6 @@ add_block_preprocessor(sub {
 
 });
 
-Test::Nginx::Socket::set_http_config_filter(sub {
-    my $config = shift;
-    my $snippet = `./t/bin/gen_snippet.lua conf_server`;
-    $config .= $snippet;
-    return $config;
-});
-
 run_tests();
 
 __DATA__
diff --git a/t/deployment/mtls.t b/t/deployment/mtls.t
index 46972a8b3c90..8826dd2dd030 100644
--- a/t/deployment/mtls.t
+++ b/t/deployment/mtls.t
@@ -38,13 +38,6 @@ add_block_preprocessor(sub {
 
 });
 
-Test::Nginx::Socket::set_http_config_filter(sub {
-    my $config = shift;
-    my $snippet = `./t/bin/gen_snippet.lua conf_server`;
-    $config .= $snippet;
-    return $config;
-});
-
 run_tests();
 
 __DATA__