feat: support mTLS connection to ETCD (#1437)

.github/workflows/backend-cli-test.yml

@@ -24,6 +24,12 @@ jobs:
     steps:
       - uses: actions/checkout@v2
 
+      - name: download etcd
+        working-directory: ./api
+        run: |
+          wget https://github.com/etcd-io/etcd/releases/download/v3.4.14/etcd-v3.4.14-linux-amd64.tar.gz
+          tar zxvf etcd-v3.4.14-linux-amd64.tar.gz
+
       - name: run test
         working-directory: ./api
         run: sudo ./test/shell/cli_test.sh

api/conf/conf.yaml

@@ -28,6 +28,11 @@ conf:
                           # etcd basic auth info
     # username: "root"    # ignore etcd username if not enable etcd auth
     # password: "123456"  # ignore etcd password if not enable etcd auth
+    mtls:
+      key_file: ""          # Path of your self-signed client side key
+      cert_file: ""         # Path of your self-signed client side cert
+      ca_file: ""           # Path of your self-signed ca cert, the CA is used to sign callers' certificates
+
   log:
     error_log:
       level: warn       # supports levels, lower to higher: debug, info, warn, error, panic, fatal

api/internal/conf/conf.go

@@ -57,10 +57,17 @@ var (
 	AllowList        []string
 )
 
+type MTLS struct {
+	CaFile   string `yaml:"ca_file"`
+	CertFile string `yaml:"cert_file"`
+	KeyFile  string `yaml:"key_file"`
+}
+
 type Etcd struct {
 	Endpoints []string
 	Username  string
 	Password  string
+	MTLS      *MTLS
 }
 
 type Listen struct {
@@ -222,5 +229,6 @@ func initEtcdConfig(conf Etcd) {
 		Endpoints: endpoints,
 		Username:  conf.Username,
 		Password:  conf.Password,
+		MTLS: conf.MTLS,
 	}
 }

api/internal/core/storage/etcd.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"go.etcd.io/etcd/clientv3"
+	"go.etcd.io/etcd/pkg/transport"
 
 	"github.com/apisix/manager-api/internal/conf"
 	"github.com/apisix/manager-api/internal/log"
@@ -52,12 +53,28 @@ type EtcdV3Storage struct {
 }
 
 func InitETCDClient(etcdConf *conf.Etcd) error {
-	cli, err := clientv3.New(clientv3.Config{
+	config := clientv3.Config{
 		Endpoints:   etcdConf.Endpoints,
 		DialTimeout: 5 * time.Second,
 		Username:    etcdConf.Username,
 		Password:    etcdConf.Password,
-	})
+	}
+	// mTLS
+	if etcdConf.MTLS != nil && etcdConf.MTLS.CaFile != "" &&
+		etcdConf.MTLS.CertFile != "" && etcdConf.MTLS.KeyFile != "" {
+		tlsInfo := transport.TLSInfo{
+			CertFile:      etcdConf.MTLS.CertFile,
+			KeyFile:       etcdConf.MTLS.KeyFile,
+			TrustedCAFile: etcdConf.MTLS.CaFile,
+		}
+		tlsConfig, err := tlsInfo.ClientConfig()
+		if err != nil {
+			return err
+		}
+		config.TLS = tlsConfig
+	}
+
+	cli, err := clientv3.New(config)
 	if err != nil {
 		log.Errorf("init etcd failed: %s", err)
 		return fmt.Errorf("init etcd failed: %s", err)

api/test/certs/mtls_ca.pem

@@ -0,0 +1,25 @@
+-----BEGIN CERTIFICATE-----
+MIIEKjCCAxKgAwIBAgIUFUwVOj73RH1oKB5hkp1MiU86K6owDQYJKoZIhvcNAQEL
+BQAwgawxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMSowKAYDVQQKEyFIb25lc3QgQWNobWVkJ3MgVXNlZCBD
+ZXJ0aWZpY2F0ZXMxKTAnBgNVBAsTIEhhc3RpbHktR2VuZXJhdGVkIFZhbHVlcyBE
+aXZpc29uMRkwFwYDVQQDExBBdXRvZ2VuZXJhdGVkIENBMB4XDTIxMDIwNTA4MTkw
+MFoXDTI2MDIwNDA4MTkwMFowgawxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpDYWxp
+Zm9ybmlhMRYwFAYDVQQHEw1TYW4gRnJhbmNpc2NvMSowKAYDVQQKEyFIb25lc3Qg
+QWNobWVkJ3MgVXNlZCBDZXJ0aWZpY2F0ZXMxKTAnBgNVBAsTIEhhc3RpbHktR2Vu
+ZXJhdGVkIFZhbHVlcyBEaXZpc29uMRkwFwYDVQQDExBBdXRvZ2VuZXJhdGVkIENB
+MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAxSDAqeu4jFF7fpKT1gqp
+vhC6fGWipNLDcBMMpqCSiKwi1DF0VvDiOUMNLRhsClheLJjtGXGFBJLisHD9HB3g
+q+NsyjETueD0i93qgTl3u/9Dc9oWtoy+1vyLBp5eDSIHsh8zbYFubtf3aBiBrxxk
+J83vEjG5u6dfpfroEOHPXFN6mdQxWDpoEQoVf5cUr9ZdzO1Kf+aaRKF6p/IPTonm
+WqZ587f21H/7Yrq/5s4kcYVbVmprHnvjHruc4utbdWlwAZzDYDeNK4lT+hZ1ciDX
+EWnPSYFn5lSojPDjuhI7dmHnQk3vs+SVX+cTerwc253tbgB9EmIwqsvMne8y8dof
+mQIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNV
+HQ4EFgQUWjiJWGaoZJtQp7T4WtCNLkCrBPIwDQYJKoZIhvcNAQELBQADggEBADgj
+8hbEamDNhvxQ/QK4BEzW+W0xUzL1GgGMR5Ocr1OSx0htTfwWCjvyz8Qor5j301bN
+ek/u3z3hbV7GXgFp819M0sZibk8i3IDVtcXTQTq5aImLw73gOzF4xcpL0LZUOgsO
+Zl4/fSMNg0oIUWQXohRh4q9QnoWsWLYfyd8/NJyv75HKzvst7pUlxp1NVbEFjz3l
+HXXK1vvQvq1S5dmvS3wCxP1mBemgftormLlAFnpk1GOl5QaBfPgyg9N2uD2KHRec
+BYinzfn8uCXxs2vuRwfT4MhTgDN8/u3Z62L+85Pwcn93Dksuy6dDfQfBbCCCSuRM
+KeNO9h6V0FYMbX1eYWc=
+-----END CERTIFICATE-----

api/test/certs/mtls_client-key.pem

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAzX9YEg9zmc/44rYy5Xb4sEEeNLb+CT5VkRFej0K/8/N169Rl
+9yZyXla8cGVQMNJneDg0bc2IvABptw+zTSfqfArbCPPrG++a9oJXHLLML4Sj0Zu0
+VuUirbgKT6qEQjHhSVAoXbuR6jvv6FQj1/08BkZ672yCet3XeYQZM/Z8c80skRZi
+HjQE6HblyGgKNOTMgRFlK+4tSm2zKlP7r27NNg3DvBCq18MbJkZ17Of9Uvf5irh0
+wztWdzCW/Y4gDzOkw6tIDZq1yUljlZhtDA5Re5pmDchxOY+EKnv3ILvBezIO7oga
+rQ65/Xagr4JuO2zdASki6ajNXMPbTwaF8rR8MQIDAQABAoIBAQCY0cfL/oOocfoT
+lw04igYdBQASkbdPZnS5oiIhBbG8GGSsUVLWvle1Amm2aBF/jSj3RUzwDzZNIT18
+rodXrIR7ZJNJECParZAfHATuSaUA/XHaMiGlsVbdu4ynfBZJJ9Dy9VJfilrTx2j8
+7H2PZTobLJTFsntCJfHU40De3MHmVuxRLU/b99uIgHihjk3iUibVh/lkapWtPgfk
+s4z36H00UBMJY+SbjxRhJDP9dFZ7Sg9vcXiOU38Gq1NoPTp/lYlGWvMboakvDERt
+bFrCUFseTq1LJ+mzua0dokiFo4Dsyzz5XmOTL//jYDjNMxDhTmeq1NXtSNIu43x/
+Ch1zOGdBAoGBAOK/BSp/UonxhVDuf1GN5M+RW/uOGB/eJEC997xSrR+gStv71ztq
+Pz24W+R4a7ubNOZfXyWk03CzjzyWS2qxOdLwDaOhmRzgQndVyg8x2OITdbVYjDnD
+QP2nc7NMJU4ezuxuWUo+HDYrfRlKBgTg/IYVpZ7V8gnjXsq6R21z6U9JAoGBAOgC
+hRtlgICu0wahIha62CqoSfbOferMWCCj8niA/LZeWBW6/OCJZInM3FfdzMkybNEX
+201tsIeeliYVa6IsvgYaFaiMJgvwtvAQdv9ukJ0+VUI3+TFzIHszgSufB/1aQPj4
+ReZoV3iZOApGeudSN4V6f+dB7agqwjQMtZNDtP2pAoGANlKXVUAdsSiozOPmos5A
+1C26AMFhLDlXLB+W+4o/KcWISb3DKdvhfNLvSQREozSi7tJIhEdB1M1f8p77QHtn
+JA8Y5Wvwt8dOhTKLbyp9EGSjHagyKCCMMHjusjT69wVQg7pIMA5DSgMPPIDMgly4
+gxMqk6wkCZRsgFsyg5lyeukCgYEA1JFCfRhhRQVoKOHHBsZHucWYhr0oFtEESVuM
+kyWy5C/KSpaYi+y1pZ+BniuELi66DlTqQ6WlIIyHCvuDMwIFVDff8h392eDA63Ba
+ZqtZaggrO1FnSgwuDVLiHSJGwrRHZRSrjm+4/LB87MUoY/orDmtu9mWsJfCPH/so
+/XUCRYkCgYAj1Uf5k4iuRUuR910qYIpnBYqdO3UR+njn7F5mjDkoT0UqWofaLjo1
+fzjDuc58rTBJTixuy0hcdYZraK3NIQTswAOV2mmpBrJpK93dAqdHgdBdufojgRYM
+coShlDKGd0MINh5GS0OBPnIIZiNkVr/F+s2ecwxqNUbb8MHj+aAJOA==
+-----END RSA PRIVATE KEY-----

api/test/certs/mtls_client.pem

@@ -0,0 +1,25 @@
+-----BEGIN CERTIFICATE-----
+MIIEQTCCAymgAwIBAgIUWdSswpGwJA//LV0Ui9PPKfvFuxQwDQYJKoZIhvcNAQEL
+BQAwgawxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMSowKAYDVQQKEyFIb25lc3QgQWNobWVkJ3MgVXNlZCBD
+ZXJ0aWZpY2F0ZXMxKTAnBgNVBAsTIEhhc3RpbHktR2VuZXJhdGVkIFZhbHVlcyBE
+aXZpc29uMRkwFwYDVQQDExBBdXRvZ2VuZXJhdGVkIENBMCAXDTIxMDIwNTA4MTkw
+MFoYDzIxMjEwMTEyMDgxOTAwWjBVMRUwEwYDVQQHEwx0aGUgaW50ZXJuZXQxFjAU
+BgNVBAoTDWF1dG9nZW5lcmF0ZWQxFTATBgNVBAsTDGV0Y2QgY2x1c3RlcjENMAsG
+A1UEAxMEZXRjZDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAM1/WBIP
+c5nP+OK2MuV2+LBBHjS2/gk+VZERXo9Cv/PzdevUZfcmcl5WvHBlUDDSZ3g4NG3N
+iLwAabcPs00n6nwK2wjz6xvvmvaCVxyyzC+Eo9GbtFblIq24Ck+qhEIx4UlQKF27
+keo77+hUI9f9PAZGeu9sgnrd13mEGTP2fHPNLJEWYh40BOh25choCjTkzIERZSvu
+LUptsypT+69uzTYNw7wQqtfDGyZGdezn/VL3+Yq4dMM7Vncwlv2OIA8zpMOrSA2a
+tclJY5WYbQwOUXuaZg3IcTmPhCp79yC7wXsyDu6IGq0Ouf12oK+Cbjts3QEpIumo
+zVzD208GhfK0fDECAwEAAaOBrjCBqzAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYw
+FAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFAeJ
+xZTNvenGwl5pS/wDwUUgTsRkMB8GA1UdIwQYMBaAFFo4iVhmqGSbUKe0+FrQjS5A
+qwTyMCwGA1UdEQQlMCOCCWxvY2FsaG9zdIcEfwAAAYcECZFZeIcECZFZrYcECZFZ
+4TANBgkqhkiG9w0BAQsFAAOCAQEAuTo5k2Ycg8zg4hU4QlNr5j/GJ9qegABjJ8W6
+9kGqbgjc3PyeKmdGRXpVJeH2AZPcHFWCMWlP+jJrB6HWaSJMOtNhuOh6Y2Hrb2I4
+ad815h/yC+tKHiE/uzaDK3bH3V6IQQTY38ay45O2bCWjt8pMT2LnCddF+rTXCAGX
+fzAtHhNpBh615b/CGAZivMdnmxUcswfHghXjs5aVuV2qffyLoyBr+IFlzT+xbKF9
+9AF57B3hE28jqti8aa6HOaUkspohfEJzd9i9Y8GJuH1L6QZ0WIudISnX5FEpPxRr
+5amq6pHoFrSeiJKpCX0zAz9Rv0mV6JkFvQL4fwVpfl5oOi6cpw==
+-----END CERTIFICATE-----

api/test/certs/mtls_server-key.pem

@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEAw7tr4rado6XKq3ymvQHajaI93M9HyaWtcVWNOiXSBd0i69WQ
+QwhugEkUw2yt+qXJzD06IqscFhR1PwzMITpf3ucjfxpO8CTTzgBaq9kW28ePbfR8
+aKp7t8H/CC8PwkFi/L4AjKT/5le4w0m5i3ZZgewTD8+f9pVzXlPXiGQF/o1SWjTG
+8zfsYBQ02rK/HYszd6YdscM2NRlC1YWWUAp52v4ihEEeZS9p1o8lrSyXuMOvzRpf
+lffu/izrVukwWAQ+YdIr98OOfWvqmabfANHGP4kofpti/JaCAtwFfgcgCY8QRdZm
+BIT+r2TRebwMTXITZAqAq1/LMtuY89D9cP5X/QIDAQABAoIBAGQdDBylDVJz7Yrz
+MhHAzfndv0ie2Pgh/unWOWtBhwAq0L7RuH0g5exF9RHUF9T5UZNeycqLvMzqX+IE
++LASPJE1pmlPmoqoO5HFipsVaeS2WP2DrNKYSLl/x6N29teEPE5MHNnTV3SI798r
+aXUU7slOZ52RtB8a6CyaM8b2aj59QoxrLqDW5q9XU7OXSGAxuhTd9yofuRE3OCI8
+e6+u2FS7FE78+H8DLjAVYjY3yrBVJN6HrmGzfZC0N7dIYNkqy8n2qK3CK1S5RXj6
+3FNTLfKDQo+Sh8SHLZt7LZWJkc1SSRcfDTuiy4D2nSXijOh2tpF6FP8WdeJ/zveP
+JQTxokECgYEA88KTomq01RXjt+YI6gBq0pT9lfy5/jVvI20n+Unr77TFDfXGqj5F
+HaFQQgHdjPR/My4qYoJVNAp3iTR9wODpkX+QDxSCYANovoMt+z73WUL6nXNt7vqy
+TLEWLinx4SO+vMwTnCXCxWfRV5Bs1EXzfQYPXo3gtuZrynyb4rPoGTECgYEAzY93
+skK2pPZGH5gOphjD1MW6nzaTYs335yRz5hQFsFCNP1aqPBi2fo6Gh6GMc7DFgy77
+f4tatCNnPQHU9HOtivo0WJcy6EU8cMvFq3al1dJx3ZnX/hOKfNubasVHCU9HtlNt
+//UyLGu0skLRQ2p7Bz2WZcccWx/cpUDqRc2R+I0CgYAhpk+pER/rdn0cCtZaLzqP
+3V9wUBYA4LF563ykLi8yxPqa5b3KDJSP9Y/VvNovtiTFFO9m7+UBLRy5RRTDBolX
+u4tQeZ1R0cao3gT/9P5CRTvBdojLf7ITYjLUppesY7nV6DogyRmtFJrSgq5zU0C8
+lpSSkfVeakqhBjiiwAEfUQKBgGZ2O8isPlQtubhn1+1s7LgzMxnHX2Hhns8lOWwW
+0NsY278VmNdJzjV5H4+ds9+63kjMc2oY8UZXW09qiVasDnX2z37VJvfmAwGKYOZd
+xr21HzLBS4uG/AHOiUKIQSdf0DQOlAcAlljT+wbcDWkYO2jZhw0GWZkGYboxiFTw
+6fDFAoGAG2CFVN/4jsXMecCRH+zW9SiyC8RlB4Apfh1B9dRkdM5X82FcByS4zo7K
+0e9C+7fDyTEBuEks3xqaD5P6wdvGRXOQDmBRC7wzFYHwHnvPVpiXNA+ZsH9r7GQI
+id15Aga1zbZoRktRr81+TtV5n2iFXIhvJhKIa62MTu6MSWP2pb4=
+-----END RSA PRIVATE KEY-----

api/test/certs/mtls_server.pem

@@ -0,0 +1,25 @@
+-----BEGIN CERTIFICATE-----
+MIIEQTCCAymgAwIBAgIUbq/7ubfAd7VqX/+knutmXICXCKswDQYJKoZIhvcNAQEL
+BQAwgawxCzAJBgNVBAYTAlVTMRMwEQYDVQQIEwpDYWxpZm9ybmlhMRYwFAYDVQQH
+Ew1TYW4gRnJhbmNpc2NvMSowKAYDVQQKEyFIb25lc3QgQWNobWVkJ3MgVXNlZCBD
+ZXJ0aWZpY2F0ZXMxKTAnBgNVBAsTIEhhc3RpbHktR2VuZXJhdGVkIFZhbHVlcyBE
+aXZpc29uMRkwFwYDVQQDExBBdXRvZ2VuZXJhdGVkIENBMCAXDTIxMDIwNTA4MTkw
+MFoYDzIxMjEwMTEyMDgxOTAwWjBVMRUwEwYDVQQHEwx0aGUgaW50ZXJuZXQxFjAU
+BgNVBAoTDWF1dG9nZW5lcmF0ZWQxFTATBgNVBAsTDGV0Y2QgY2x1c3RlcjENMAsG
+A1UEAxMEZXRjZDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMO7a+K2
+naOlyqt8pr0B2o2iPdzPR8mlrXFVjTol0gXdIuvVkEMIboBJFMNsrfqlycw9OiKr
+HBYUdT8MzCE6X97nI38aTvAk084AWqvZFtvHj230fGiqe7fB/wgvD8JBYvy+AIyk
+/+ZXuMNJuYt2WYHsEw/Pn/aVc15T14hkBf6NUlo0xvM37GAUNNqyvx2LM3emHbHD
+NjUZQtWFllAKedr+IoRBHmUvadaPJa0sl7jDr80aX5X37v4s61bpMFgEPmHSK/fD
+jn1r6pmm3wDRxj+JKH6bYvyWggLcBX4HIAmPEEXWZgSE/q9k0Xm8DE1yE2QKgKtf
+yzLbmPPQ/XD+V/0CAwEAAaOBrjCBqzAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYw
+FAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFHYc
+Fgd9rBDEQ9t82v2JKo8JCA7VMB8GA1UdIwQYMBaAFFo4iVhmqGSbUKe0+FrQjS5A
+qwTyMCwGA1UdEQQlMCOCCWxvY2FsaG9zdIcEfwAAAYcECZFZeIcECZFZrYcECZFZ
+4TANBgkqhkiG9w0BAQsFAAOCAQEATeAhmtQpydjRmIo+3r6fvAHXi6BMZKjMrYQV
+hkqakZ2mZfQXZB+AHLthc5ii4zBrB7buyYx8W4lqC7DW3vC8WrEP4fTOe7M+WbhB
+cIyhCFufgs9xSiED5wWOxSfTNZBbXcOvvrOwfFF1KZvuJQWtHNWU5V3fz+uHTCZE
+67YQgMdw+dfUl7EzdZKGqXD+BC7j0zGrJR9BlYnrMrDKxL1uZ5OZvySLnSCVjO5u
+u2PCXE+VWUs+xtnDz8rIq0ETFe8Yt2CqHYJ14QvMl9oYE7Tkj0/xrtyRtRp8r0ZW
+ox/FVX9OajzUZaUErwFNuz2Vej4tojlDtulbVinO9awySrhOjQ==
+-----END CERTIFICATE-----

api/test/shell/cli_test.sh

@@ -338,3 +338,50 @@ if [[ `echo ${resp} | grep -c "${GITHASH}"` -ne '1' ]]; then
 fi
 
 check_logfile
+
+./manager-api stop
+clean_up
+
+# mtls test
+./etcd-v3.4.14-linux-amd64/etcd --name infra0 --data-dir infra0 \
+  --client-cert-auth --trusted-ca-file=$(pwd)/test/certs/mtls_ca.pem --cert-file=$(pwd)/test/certs/mtls_server.pem --key-file=$(pwd)/test/certs/mtls_server-key.pem \
+  --advertise-client-urls https://127.0.0.1:3379 --listen-client-urls https://127.0.0.1:3379 --listen-peer-urls http://127.0.0.1:3380 &
+
+currentDir=$(pwd)
+
+if [[ $KERNEL = "Darwin" ]]; then
+  sed -i "" "s@key_file: \"\"@key_file: \"$currentDir/test/certs/mtls_client-key.pem\"@g" conf/conf.yaml
+  sed -i "" "s@cert_file: \"\"@cert_file: \"$currentDir/test/certs/mtls_client.pem\"@g" conf/conf.yaml
+  sed -i "" "s@ca_file: \"\"@ca_file: \"$currentDir/test/certs/mtls_ca.pem\"@g" conf/conf.yaml
+  sed -i "" 's/127.0.0.1:2379/127.0.0.1:3379/' conf/conf.yaml
+else
+  sed -i "s@key_file: \"\"@key_file: \"$currentDir/test/certs/mtls_client-key.pem\"@g" conf/conf.yaml
+  sed -i "s@cert_file: \"\"@cert_file: \"$currentDir/test/certs/mtls_client.pem\"@g" conf/conf.yaml
+  sed -i "s@ca_file: \"\"@ca_file: \"$currentDir/test/certs/mtls_ca.pem\"@g" conf/conf.yaml
+  sed -i 's/127.0.0.1:2379/127.0.0.1:3379/' conf/conf.yaml
+fi
+
+./manager-api &
+sleep 3
+
+# validate process is right by requesting login api
+resp=$(curl http://127.0.0.1:9000/apisix/admin/user/login -H "Content-Type: application/json" -d '{"username":"admin", "password": "admin"}')
+token=$(echo "${resp}" | sed 's/{/\n/g' | sed 's/,/\n/g' | grep "token" | sed 's/:/\n/g' | sed '1d' | sed 's/}//g'  | sed 's/"//g')
+if [ -z "${token}" ]; then
+    echo "login failed(mTLS connetct to ETCD)"
+    exit 1
+fi
+
+# more validation to make sure it's ok to access etcd
+resp=$(curl -ig -XPUT http://127.0.0.1:9000/apisix/admin/consumers -i -H "Content-Type: application/json" -H "Authorization: $token" -d '{"username":"etcd_basic_auth_test"}')
+respCode=$(echo "${resp}" | sed 's/{/\n/g'| sed 's/,/\n/g' | grep "code" | sed 's/:/\n/g' | sed '1d')
+respMessage=$(echo "${resp}" | sed 's/{/\n/g'| sed 's/,/\n/g' | grep "message" | sed 's/:/\n/g' | sed '1d')
+if [ "$respCode" != "0" ] || [ $respMessage != "\"\"" ]; then
+    echo "verify writing data failed(mTLS connetct to ETCD)"
+    exit 1
+fi
+
+pkill -f etcd
+
+./manager-api stop
+clean_up