This is an automated cherry-pick of pingcap#5390

Signed-off-by: ti-chi-bot <ti-community-prow-bot@tidb.io>

dm/dm/master/server.go

@@ -190,8 +190,18 @@ func (s *Server) Start(ctx context.Context) (err error) {
 		"/status": getStatusHandle(),
 		"/debug/": getDebugHandler(),
 	}
+<<<<<<< HEAD
 	if s.cfg.ExperimentalFeatures.OpenAPI {
 		if initOpenAPIErr := s.InitOpenAPIHandles(); initOpenAPIErr != nil {
+=======
+	if s.cfg.OpenAPI {
+		// tls3 is used to openapi reverse proxy
+		tls3, err1 := toolutils.NewTLS(s.cfg.SSLCA, s.cfg.SSLCert, s.cfg.SSLKey, s.cfg.AdvertiseAddr, s.cfg.CertAllowedCN)
+		if err1 != nil {
+			return terror.ErrMasterTLSConfigNotValid.Delegate(err1)
+		}
+		if initOpenAPIErr := s.InitOpenAPIHandles(tls3.TLSConfig()); initOpenAPIErr != nil {
+>>>>>>> 359af1861 (DM/Openapi: use reverse proxy instead of redirect (#5390))
 			return terror.ErrOpenAPICommonError.Delegate(initOpenAPIErr)
 		}
 		userHandles["/api/v1/"] = s.echo

dm/tests/openapi/client/openapi_source_check

@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 import sys
 import requests
+import ssl
 
 SOURCE1_NAME = "mysql-01"
 SOURCE2_NAME = "mysql-02"
@@ -11,6 +12,10 @@ WORKER2_NAME = "worker2"
 API_ENDPOINT = "http://127.0.0.1:8261/api/v1/sources"
 API_ENDPOINT_NOT_LEADER = "http://127.0.0.1:8361/api/v1/sources"
 
+API_ENDPOINT_HTTPS = "https://127.0.0.1:8261/api/v1/sources"
+API_ENDPOINT_NOT_LEADER_HTTPS = "https://127.0.0.1:8361/api/v1/sources"
+
+
 
 def create_source_failed():
     resp = requests.post(url=API_ENDPOINT)
@@ -47,6 +52,41 @@ def create_source2_success():
     print("create_source1_success resp=", resp.json())
     assert resp.status_code == 201
 
+<<<<<<< HEAD
+=======
+def create_source_success_https(ssl_ca, ssl_cert, ssl_key):
+    req = {
+        "source": {
+            "case_sensitive": False,
+            "enable": True,
+            "enable_gtid": False,
+            "host": "127.0.0.1",
+            "password": "123456",
+            "port": 3306,
+            "source_name": SOURCE1_NAME,
+            "user": "root",
+        }
+    }
+    resp = requests.post(url=API_ENDPOINT_HTTPS, json=req, verify=ssl_ca, cert=(ssl_cert, ssl_key))
+    print("create_source_success_https resp=", resp.json())
+    assert resp.status_code == 201
+
+def update_source1_without_password_success():
+    req = {
+        "source": {
+            "case_sensitive": False,
+            "enable": True,
+            "enable_gtid": False,
+            "host": "127.0.0.1",
+            "port": 3306,
+            "source_name": SOURCE1_NAME,
+            "user": "root",
+        }
+    }
+    resp = requests.put(url=API_ENDPOINT + "/" + SOURCE1_NAME, json=req)
+    print("update_source1_without_password_success resp=", resp.json())
+    assert resp.status_code == 200
+>>>>>>> 359af1861 (DM/Openapi: use reverse proxy instead of redirect (#5390))
 
 def list_source_success(source_count):
     resp = requests.get(url=API_ENDPOINT)
@@ -55,6 +95,12 @@ def list_source_success(source_count):
     print("list_source_by_openapi_success resp=", data)
     assert data["total"] == int(source_count)
 
+def list_source_success_https(source_count, ssl_ca, ssl_cert, ssl_key):
+    resp = requests.get(url=API_ENDPOINT_HTTPS, verify=ssl_ca, cert=(ssl_cert, ssl_key))
+    assert resp.status_code == 200
+    data = resp.json()
+    print("list_source_success_https resp=", data)
+    assert data["total"] == int(source_count)
 
 def list_source_with_status_success(source_count, status_count):
     resp = requests.get(url=API_ENDPOINT + "?with_status=true")
@@ -66,13 +112,19 @@ def list_source_with_status_success(source_count, status_count):
         assert len(data["data"][i]["status_list"]) == int(status_count)
 
 
-def list_source_with_redirect(source_count):
+def list_source_with_reverse(source_count):
     resp = requests.get(url=API_ENDPOINT_NOT_LEADER)
     assert resp.status_code == 200
     data = resp.json()
-    print("list_source_by_openapi_redirect resp=", data)
+    print("list_source_with_reverse resp=", data)
     assert data["total"] == int(source_count)
 
+def list_source_with_reverse_https(source_count, ssl_ca, ssl_cert, ssl_key):
+    resp = requests.get(url=API_ENDPOINT_NOT_LEADER_HTTPS, verify=ssl_ca, cert=(ssl_cert, ssl_key))
+    assert resp.status_code == 200
+    data = resp.json()
+    print("list_source_with_reverse_https resp=", data)
+    assert data["total"] == int(source_count)
 
 def delete_source_success(source_name):
     resp = requests.delete(url=API_ENDPOINT + "/" + source_name)
@@ -215,8 +267,15 @@ if __name__ == "__main__":
         "create_source_failed": create_source_failed,
         "create_source1_success": create_source1_success,
         "create_source2_success": create_source2_success,
+<<<<<<< HEAD
+=======
+        "create_source_success_https": create_source_success_https,
+        "update_source1_without_password_success": update_source1_without_password_success,
+>>>>>>> 359af1861 (DM/Openapi: use reverse proxy instead of redirect (#5390))
         "list_source_success": list_source_success,
-        "list_source_with_redirect": list_source_with_redirect,
+        "list_source_success_https": list_source_success_https,
+        "list_source_with_reverse_https": list_source_with_reverse_https,
+        "list_source_with_reverse": list_source_with_reverse,
         "list_source_with_status_success": list_source_with_status_success,
         "delete_source_failed": delete_source_failed,
         "delete_source_success": delete_source_success,

dm/tests/openapi/run.sh

@@ -67,7 +67,7 @@ function test_source() {
 	openapi_source_check "delete_source_failed" "mysql-01"
 
 	# send request to not leader node
-	openapi_source_check "list_source_with_redirect" 0
+	openapi_source_check "list_source_with_reverse" 0
 
 	echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: SOURCE SUCCESS"
 }
@@ -294,6 +294,62 @@ function test_noshard_task() {
 	echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: NO SHARD TASK SUCCESS"
 }
 
+function test_reverse_https() {
+	echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>START TEST OPENAPI: REVERSE HTTPS"
+	cleanup_data openapi
+	cleanup_process
+
+	cp $cur/tls_conf/dm-master1.toml $WORK_DIR/
+	cp $cur/tls_conf/dm-master2.toml $WORK_DIR/
+	cp $cur/tls_conf/dm-worker1.toml $WORK_DIR/
+	cp $cur/tls_conf/dm-worker2.toml $WORK_DIR/
+	sed -i "s%dir-placeholer%$cur\/tls_conf%g" $WORK_DIR/dm-master1.toml
+	sed -i "s%dir-placeholer%$cur\/tls_conf%g" $WORK_DIR/dm-master2.toml
+	sed -i "s%dir-placeholer%$cur\/tls_conf%g" $WORK_DIR/dm-worker1.toml
+	sed -i "s%dir-placeholer%$cur\/tls_conf%g" $WORK_DIR/dm-worker2.toml
+
+	# run dm-master1
+	run_dm_master $WORK_DIR/master1 $MASTER_PORT1 $WORK_DIR/dm-master1.toml
+	check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT1 "$cur/tls_conf/ca.pem" "$cur/tls_conf/dm.pem" "$cur/tls_conf/dm.key"
+	# join master2
+	run_dm_master $WORK_DIR/master2 $MASTER_PORT2 $WORK_DIR/dm-master2.toml
+	check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT2 "$cur/tls_conf/ca.pem" "$cur/tls_conf/dm.pem" "$cur/tls_conf/dm.key"
+	# run dm-worker1
+	run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $WORK_DIR/dm-worker1.toml
+	check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT "$cur/tls_conf/ca.pem" "$cur/tls_conf/dm.pem" "$cur/tls_conf/dm.key"
+	# run dm-worker2
+	run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $WORK_DIR/dm-worker2.toml
+	check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT "$cur/tls_conf/ca.pem" "$cur/tls_conf/dm.pem" "$cur/tls_conf/dm.key"
+
+	prepare_database
+	# create source successfully
+	openapi_source_check "create_source_success_https" "$cur/tls_conf/ca.pem" "$cur/tls_conf/dm.pem" "$cur/tls_conf/dm.key"
+
+	# get source list success
+	openapi_source_check "list_source_success_https" 1 "$cur/tls_conf/ca.pem" "$cur/tls_conf/dm.pem" "$cur/tls_conf/dm.key"
+
+	# send request to not leader node
+	openapi_source_check "list_source_with_reverse_https" 1 "$cur/tls_conf/ca.pem" "$cur/tls_conf/dm.pem" "$cur/tls_conf/dm.key"
+
+	cleanup_data openapi
+	cleanup_process
+
+	# run dm-master1
+	run_dm_master $WORK_DIR/master1 $MASTER_PORT1 $cur/conf/dm-master1.toml
+	check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT1
+	# join master2
+	run_dm_master $WORK_DIR/master2 $MASTER_PORT2 $cur/conf/dm-master2.toml
+	check_rpc_alive $cur/../bin/check_master_online 127.0.0.1:$MASTER_PORT2
+	# run dm-worker1
+	run_dm_worker $WORK_DIR/worker1 $WORKER1_PORT $cur/conf/dm-worker1.toml
+	check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER1_PORT
+	# run dm-worker2
+	run_dm_worker $WORK_DIR/worker2 $WORKER2_PORT $cur/conf/dm-worker2.toml
+	check_rpc_alive $cur/../bin/check_worker_online 127.0.0.1:$WORKER2_PORT
+
+	echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST OPENAPI: REVERSE HTTPS"
+}
+
 function test_cluster() {
 	# list master and worker node
 	openapi_cluster_check "list_master_success" 2
@@ -333,6 +389,19 @@ function run() {
 
 	test_shard_task
 	test_noshard_task
+<<<<<<< HEAD
+=======
+	test_task_templates
+	test_noshard_task_dump_status
+	test_complex_operations_of_source_and_task
+	test_task_with_ignore_check_items
+	test_delete_task_with_stopped_downstream
+	test_start_task_with_condition
+	test_stop_task_with_condition
+	test_reverse_https
+
+	# NOTE: this test case MUST running at last, because it will offline some members of cluster
+>>>>>>> 359af1861 (DM/Openapi: use reverse proxy instead of redirect (#5390))
 	test_cluster
 }
 

dm/tests/openapi/tls_conf/ca.pem

@@ -0,0 +1,8 @@
+-----BEGIN CERTIFICATE-----
+MIIBGDCBwAIJAOjYXLFw5V1HMAoGCCqGSM49BAMCMBQxEjAQBgNVBAMMCWxvY2Fs
+aG9zdDAgFw0yMDAzMTcxMjAwMzNaGA8yMjkzMTIzMTEyMDAzM1owFDESMBAGA1UE
+AwwJbG9jYWxob3N0MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEglCIJD8uVBfD
+kuM+UQP+VA7Srbz17WPLA0Sqc+sQ2p6fT6HYKCW60EXiZ/yEC0925iyVbXEEbX4J
+xCc2Heow5TAKBggqhkjOPQQDAgNHADBEAiAILL3Zt/3NFeDW9c9UAcJ9lc92E0ZL
+GNDuH6i19Fex3wIgT0ZMAKAFSirGGtcLu0emceuk+zVKjJzmYbsLdpj/JuQ=
+-----END CERTIFICATE-----

dm/tests/openapi/tls_conf/dm-master1.toml

@@ -0,0 +1,16 @@
+# Master Configuration.
+advertise-addr = "127.0.0.1:8261"
+initial-cluster = "master1=https://127.0.0.1:8291"
+master-addr = ":8261"
+name = "master1"
+peer-urls = "127.0.0.1:8291"
+openapi = true
+
+ssl-ca = "dir-placeholer/ca.pem"
+ssl-cert = "dir-placeholer/dm.pem"
+ssl-key = "dir-placeholer/dm.key"
+cert-allowed-cn = ["dm"]
+auto-compaction-retention = "3s"
+
+
+

dm/tests/openapi/tls_conf/dm-master2.toml

@@ -0,0 +1,14 @@
+# Master Configuration.
+name = "master2"
+master-addr = ":8361"
+advertise-addr = "127.0.0.1:8361"
+peer-urls = "http://127.0.0.1:8292"
+join = "127.0.0.1:8261"
+openapi = true
+
+ssl-ca = "dir-placeholer/ca.pem"
+ssl-cert = "dir-placeholer/dm.pem"
+ssl-key = "dir-placeholer/dm.key"
+cert-allowed-cn = ["dm"]
+auto-compaction-retention = "3s"
+

dm/tests/openapi/tls_conf/dm-worker1.toml

@@ -0,0 +1,7 @@
+name = "worker1"
+join = "127.0.0.1:8261"
+
+ssl-ca = "dir-placeholer/ca.pem"
+ssl-cert = "dir-placeholer/dm.pem"
+ssl-key = "dir-placeholer/dm.key"
+cert-allowed-cn = ["dm"]

dm/tests/openapi/tls_conf/dm-worker2.toml

@@ -0,0 +1,7 @@
+name = "worker2"
+join = "127.0.0.1:8261"
+
+ssl-ca = "dir-placeholer/ca.pem"
+ssl-cert = "dir-placeholer/dm.pem"
+ssl-key = "dir-placeholer/dm.key"
+cert-allowed-cn = ["dm"]

dm/tests/openapi/tls_conf/dm.key

@@ -0,0 +1,8 @@
+-----BEGIN EC PARAMETERS-----
+BggqhkjOPQMBBw==
+-----END EC PARAMETERS-----
+-----BEGIN EC PRIVATE KEY-----
+MHcCAQEEICF/GDtVxhTPTP501nOu4jgwGSDY01xN+61xd9MfChw+oAoGCCqGSM49
+AwEHoUQDQgAEgQOv5bQO7xK16vZWhwJqlz2vl19+AXW2Ql7KQyGiBJVSvLbyDLOr
+kIeFlHN04iqQ39SKSOSfeGSfRt6doU6IcA==
+-----END EC PRIVATE KEY-----

dm/tests/openapi/tls_conf/dm.pem

@@ -0,0 +1,10 @@
+-----BEGIN CERTIFICATE-----
+MIIBZDCCAQqgAwIBAgIJAIT/lgXUc1JqMAoGCCqGSM49BAMCMBQxEjAQBgNVBAMM
+CWxvY2FsaG9zdDAgFw0yMDAzMTcxMjAwMzNaGA8yMjkzMTIzMTEyMDAzM1owDTEL
+MAkGA1UEAwwCZG0wWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAASBA6/ltA7vErXq
+9laHAmqXPa+XX34BdbZCXspDIaIElVK8tvIMs6uQh4WUc3TiKpDf1IpI5J94ZJ9G
+3p2hTohwo0owSDAaBgNVHREEEzARgglsb2NhbGhvc3SHBH8AAAEwCwYDVR0PBAQD
+AgWgMB0GA1UdJQQWMBQGCCsGAQUFBwMCBggrBgEFBQcDATAKBggqhkjOPQQDAgNI
+ADBFAiEAx6ljJ+tNa55ypWLGNqmXlB4UdMmKmE4RSKJ8mmEelfECIG2ZmCE59rv5
+wImM6KnK+vM2QnEiISH3PeYyyRzQzycu
+-----END CERTIFICATE-----