From 1fa9521d5a1aa02688dacdc431c5f045a947686c Mon Sep 17 00:00:00 2001
From: Wenqi Mou <wenqimou@gmail.com>
Date: Tue, 1 Oct 2024 12:10:05 -0400
Subject: [PATCH 01/19] try to fix aws tests, load balance br int tests

Signed-off-by: Wenqi Mou <wenqimou@gmail.com>
---
 br/tests/br_encryption/run.sh  | 5 ++---
 br/tests/run_group_br_tests.sh | 8 ++++----
 2 files changed, 6 insertions(+), 7 deletions(-)

diff --git a/br/tests/br_encryption/run.sh b/br/tests/br_encryption/run.sh
index 84075aff98838..3b49e5cd0fcdb 100755
--- a/br/tests/br_encryption/run.sh
+++ b/br/tests/br_encryption/run.sh
@@ -418,9 +418,8 @@ test_backup_encrypted_restore_unencrypted
 test_plaintext
 test_plaintext_data_key
 test_local_master_key
-# some issue running in CI, will fix later
-#test_aws_kms
-#test_aws_kms_with_iam
+test_aws_kms
+test_aws_kms_with_iam
 test_mixed_full_encrypted_log_plain
 test_mixed_full_plain_log_encrypted
 
diff --git a/br/tests/run_group_br_tests.sh b/br/tests/run_group_br_tests.sh
index 13b5e34178a30..e9ae610114f6e 100755
--- a/br/tests/run_group_br_tests.sh
+++ b/br/tests/run_group_br_tests.sh
@@ -20,10 +20,10 @@ mkdir -p $COV_DIR
 # Putting multiple light tests together and heavy tests in a separate group.
 declare -A groups
 groups=(
-	["G00"]="br_300_small_tables br_backup_empty br_backup_version br_cache_table br_case_sensitive br_charset_gbk br_check_new_collocation_enable br_history br_gcs br_rawkv"
-	["G01"]="br_autoid br_crypter2 br_db br_db_online br_db_online_newkv br_db_skip br_debug_meta br_ebs br_foreign_key br_full br_table_partition br_full_ddl"
-	["G02"]="br_full_cluster_restore br_full_index br_incremental_ddl br_pitr_failpoint br_pitr_gc_safepoint"
-	["G03"]='br_incompatible_tidb_config br_incremental br_incremental_index br_incremental_only_ddl br_incremental_same_table br_insert_after_restore br_key_locked br_log_test br_move_backup br_mv_index br_other br_partition_add_index br_tidb_placement_policy br_tiflash br_tiflash_conflict'
+	["G00"]="br_300_small_tables br_backup_empty br_backup_version br_cache_table br_case_sensitive br_charset_gbk br_check_new_collocation_enable br_history br_gcs br_rawkv br_tidb_placement_policy br_tiflash br_tiflash_conflict"
+	["G01"]="br_autoid br_crypter2 br_db br_db_online br_db_online_newkv br_db_skip br_debug_meta br_ebs br_foreign_key br_full br_table_partition br_full_ddl br_mv_index"
+	["G02"]="br_full_cluster_restore br_full_index br_incremental_ddl br_pitr_failpoint br_pitr_gc_safepoin br_other br_partition_add_indext"
+	["G03"]='br_incompatible_tidb_config br_incremental br_incremental_index br_incremental_only_ddl br_incremental_same_table br_insert_after_restore br_key_locked br_log_test br_move_backup'
 	["G04"]='br_range br_replica_read br_restore_TDE_enable br_restore_log_task_enable br_s3 br_shuffle_leader br_shuffle_region br_single_table'
 	["G05"]='br_skip_checksum br_split_region_fail br_systables br_table_filter br_txn br_stats br_clustered_index br_crypter'
 	["G06"]='br_tikv_outage br_tikv_outage3 br_restore_checkpoint br_encryption'

From b0c87cee3ee64bc276426fcb0973c12308f30074 Mon Sep 17 00:00:00 2001
From: Wenqi Mou <wenqimou@gmail.com>
Date: Tue, 1 Oct 2024 14:13:13 -0400
Subject: [PATCH 02/19] fix typo

Signed-off-by: Wenqi Mou <wenqimou@gmail.com>
---
 br/tests/run_group_br_tests.sh |  2 +-
 tests/_utils/run_br            | 12 +++++++++++-
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/br/tests/run_group_br_tests.sh b/br/tests/run_group_br_tests.sh
index e9ae610114f6e..1c46f7b51b84a 100755
--- a/br/tests/run_group_br_tests.sh
+++ b/br/tests/run_group_br_tests.sh
@@ -22,7 +22,7 @@ declare -A groups
 groups=(
 	["G00"]="br_300_small_tables br_backup_empty br_backup_version br_cache_table br_case_sensitive br_charset_gbk br_check_new_collocation_enable br_history br_gcs br_rawkv br_tidb_placement_policy br_tiflash br_tiflash_conflict"
 	["G01"]="br_autoid br_crypter2 br_db br_db_online br_db_online_newkv br_db_skip br_debug_meta br_ebs br_foreign_key br_full br_table_partition br_full_ddl br_mv_index"
-	["G02"]="br_full_cluster_restore br_full_index br_incremental_ddl br_pitr_failpoint br_pitr_gc_safepoin br_other br_partition_add_indext"
+	["G02"]="br_full_cluster_restore br_full_index br_incremental_ddl br_pitr_failpoint br_pitr_gc_safepoint br_other br_partition_add_index"
 	["G03"]='br_incompatible_tidb_config br_incremental br_incremental_index br_incremental_only_ddl br_incremental_same_table br_insert_after_restore br_key_locked br_log_test br_move_backup'
 	["G04"]='br_range br_replica_read br_restore_TDE_enable br_restore_log_task_enable br_s3 br_shuffle_leader br_shuffle_region br_single_table'
 	["G05"]='br_skip_checksum br_split_region_fail br_systables br_table_filter br_txn br_stats br_clustered_index br_crypter'
diff --git a/tests/_utils/run_br b/tests/_utils/run_br
index 743a25c456121..0380dd4fe7a0a 100755
--- a/tests/_utils/run_br
+++ b/tests/_utils/run_br
@@ -16,8 +16,18 @@
 
 set -eux
 
+# Add a new environment variable to control encryption
+ENABLE_ENCRYPTION=${ENABLE_ENCRYPTION:-false}
+
+# Prepare encryption arguments
+ENCRYPTION_ARGS=""
+if [ "$ENABLE_ENCRYPTION" = "true" ]; then
+    ENCRYPTION_ARGS="--crypter.method aes128-ctr --crypter.key 0123456789abcdef0123456789abcdef --log.crypter.method AES256-CTR --log.crypter.key 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
+fi
+
 br.test -test.coverprofile="$COV_DIR/cov.$TEST_NAME.$$.out.log" DEVEL "$@" \
     -L "info" \
     --ca "$TEST_DIR/certs/ca.pem" \
     --cert "$TEST_DIR/certs/br.pem" \
-    --key "$TEST_DIR/certs/br.key"
+    --key "$TEST_DIR/certs/br.key" \
+    $ENCRYPTION_ARGS

From 1a0203ef8feb5f5361f5f502d86ac7c67aa6a33e Mon Sep 17 00:00:00 2001
From: Wenqi Mou <wenqimou@gmail.com>
Date: Tue, 1 Oct 2024 17:31:46 -0400
Subject: [PATCH 03/19] run encryption everywhere, disable aws tests

Signed-off-by: Wenqi Mou <wenqimou@gmail.com>
---
 br/tests/br_crypter/run.sh     | 4 ++++
 br/tests/br_crypter2/run.sh    | 4 ++++
 br/tests/br_encryption/run.sh  | 9 +++++++--
 br/tests/br_rawkv/run.sh       | 4 ++++
 br/tests/br_txn/run.sh         | 4 ++++
 br/tests/run_group_br_tests.sh | 4 ++++
 6 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/br/tests/br_crypter/run.sh b/br/tests/br_crypter/run.sh
index b4e6c3188b9a3..3c230ae8cbe42 100755
--- a/br/tests/br_crypter/run.sh
+++ b/br/tests/br_crypter/run.sh
@@ -14,6 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# disable global ENABLE_ENCRYPTION for this script
+ENABLE_ENCRYPTION=false
+export ENABLE_ENCRYPTION
+
 set -eu
 DB="$TEST_NAME"
 TABLE="usertable"
diff --git a/br/tests/br_crypter2/run.sh b/br/tests/br_crypter2/run.sh
index a399ce7bf7660..3ec94d0a3ae57 100755
--- a/br/tests/br_crypter2/run.sh
+++ b/br/tests/br_crypter2/run.sh
@@ -14,6 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# disable global ENABLE_ENCRYPTION for this script
+ENABLE_ENCRYPTION=false
+export ENABLE_ENCRYPTION
+
 set -eu
 DB="$TEST_NAME"
 TABLE="usertable"
diff --git a/br/tests/br_encryption/run.sh b/br/tests/br_encryption/run.sh
index 3b49e5cd0fcdb..97014faebeec4 100755
--- a/br/tests/br_encryption/run.sh
+++ b/br/tests/br_encryption/run.sh
@@ -14,6 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# disable global ENABLE_ENCRYPTION for this script
+ENABLE_ENCRYPTION=false
+export ENABLE_ENCRYPTION
+
 set -eu
 . run_services
 CUR=$(cd "$(dirname "$0")" && pwd)
@@ -418,8 +422,9 @@ test_backup_encrypted_restore_unencrypted
 test_plaintext
 test_plaintext_data_key
 test_local_master_key
-test_aws_kms
-test_aws_kms_with_iam
+# localstack not working with older glibc version in our centos7 base image...
+#test_aws_kms
+#test_aws_kms_with_iam
 test_mixed_full_encrypted_log_plain
 test_mixed_full_plain_log_encrypted
 
diff --git a/br/tests/br_rawkv/run.sh b/br/tests/br_rawkv/run.sh
index c13d057d1e0dd..dd2a846f1188a 100755
--- a/br/tests/br_rawkv/run.sh
+++ b/br/tests/br_rawkv/run.sh
@@ -14,6 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# disable global ENABLE_ENCRYPTION for this script
+ENABLE_ENCRYPTION=false
+export ENABLE_ENCRYPTION
+
 set -eux
 
 # restart service without tiflash
diff --git a/br/tests/br_txn/run.sh b/br/tests/br_txn/run.sh
index 567be9d76e263..8293d04fd8d33 100755
--- a/br/tests/br_txn/run.sh
+++ b/br/tests/br_txn/run.sh
@@ -14,6 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# disable global ENABLE_ENCRYPTION for this script
+ENABLE_ENCRYPTION=false
+export ENABLE_ENCRYPTION
+
 set -eux
 
 # restart service without tiflash
diff --git a/br/tests/run_group_br_tests.sh b/br/tests/run_group_br_tests.sh
index 1c46f7b51b84a..da7beb2beee53 100755
--- a/br/tests/run_group_br_tests.sh
+++ b/br/tests/run_group_br_tests.sh
@@ -44,6 +44,10 @@ for script in "$CUR"/*/run.sh; do
 	fi
 done
 
+# enable local encryption for all tests
+ENABLE_ENCRYPTION=true
+export ENABLE_ENCRYPTION
+
 if [[ "$group" == "others" ]]; then
 	if [[ -z $others ]]; then
 		echo "All br integration test cases have been added to groups"

From 1b192f2dd63af5239e4c9602cd189912b79f3a89 Mon Sep 17 00:00:00 2001
From: Wenqi Mou <wenqimou@gmail.com>
Date: Wed, 2 Oct 2024 17:21:43 -0400
Subject: [PATCH 04/19] skip compat tests

Signed-off-by: Wenqi Mou <wenqimou@gmail.com>
---
 br/tests/br_check_new_collocation_enable/run.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/br/tests/br_check_new_collocation_enable/run.sh b/br/tests/br_check_new_collocation_enable/run.sh
index 3f8429cebded0..d9ed7bc7a32ac 100755
--- a/br/tests/br_check_new_collocation_enable/run.sh
+++ b/br/tests/br_check_new_collocation_enable/run.sh
@@ -14,6 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# disable encryption as not working in this backward compatible test
+ENABLE_ENCRYPTION=false
+export ENABLE_ENCRYPTION
+
 set -eu
 DB="$TEST_NAME"
 

From 8b4232702682a70bcdb72bedbf24d4cb9a9b2cc4 Mon Sep 17 00:00:00 2001
From: Wenqi Mou <wenqimou@gmail.com>
Date: Thu, 3 Oct 2024 14:34:20 -0400
Subject: [PATCH 05/19] fix test

Signed-off-by: Wenqi Mou <wenqimou@gmail.com>
---
 br/tests/br_gcs/run.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/br/tests/br_gcs/run.sh b/br/tests/br_gcs/run.sh
index c71cecf5c0ccd..192236a46a6b3 100755
--- a/br/tests/br_gcs/run.sh
+++ b/br/tests/br_gcs/run.sh
@@ -14,6 +14,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# disable encryption as not working with brv4.0.8
+ENABLE_ENCRYPTION=false
+export ENABLE_ENCRYPTION
+
 set -eux
 DB="$TEST_NAME"
 TABLE="usertable"

From b10ccfef3e5c52e5cd38ce53ab663763d3e6da97 Mon Sep 17 00:00:00 2001
From: Wenqi Mou <wenqimou@gmail.com>
Date: Fri, 4 Oct 2024 12:36:18 -0400
Subject: [PATCH 06/19] add debug log for br-pitr

Signed-off-by: Wenqi Mou <wenqimou@gmail.com>
---
 br/tests/br_pitr/run.sh | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/br/tests/br_pitr/run.sh b/br/tests/br_pitr/run.sh
index 096004c634f5e..524fa37daf998 100644
--- a/br/tests/br_pitr/run.sh
+++ b/br/tests/br_pitr/run.sh
@@ -72,7 +72,7 @@ echo "incremental_delete_range_count: $incremental_delete_range_count"
 # wait checkpoint advance
 echo "wait checkpoint advance"
 sleep 10
-current_ts=$(echo $(($(date +%s%3N) << 18)))
+current_ts=$(python3 -c "import time; print(int(time.time() * 1000) << 18)")
 echo "current ts: $current_ts"
 i=0
 while true; do
@@ -122,7 +122,7 @@ check_result() {
 }
 
 # start a new cluster
-echo "restart a services"
+echo "restart services"
 restart_services
 
 # non-compliant operation
@@ -130,7 +130,7 @@ echo "non compliant operation"
 restore_fail=0
 run_br --pd $PD_ADDR restore point -s "local://$TEST_DIR/$PREFIX/log" --start-ts $current_ts || restore_fail=1
 if [ $restore_fail -ne 1 ]; then
-    echo 'pitr success' 
+    echo 'pitr success on non compliant operation'
     exit 1
 fi
 
@@ -142,7 +142,7 @@ run_br --pd $PD_ADDR restore point -s "local://$TEST_DIR/$PREFIX/log" --full-bac
 check_result
 
 # start a new cluster for incremental + log
-echo "restart a services"
+echo "restart services"
 restart_services
 
 echo "run snapshot restore#2"
@@ -154,7 +154,7 @@ run_br --pd $PD_ADDR restore point -s "local://$TEST_DIR/$PREFIX/log" --full-bac
 check_result
 
 # start a new cluster for incremental + log
-echo "restart a services"
+echo "restart services"
 restart_services
 
 echo "run snapshot restore#3"
@@ -164,7 +164,7 @@ echo "run incremental restore but failed"
 restore_fail=0
 run_br --pd $PD_ADDR restore full -s "local://$TEST_DIR/$PREFIX/inc_fail" || restore_fail=1
 if [ $restore_fail -ne 1 ]; then
-    echo 'pitr success' 
+    echo 'pitr success on incremental restore'
     exit 1
 fi
 
@@ -186,7 +186,7 @@ restore_fail=0
 run_br --pd $PD_ADDR restore point -s "local://$TEST_DIR/$PREFIX/log" --full-backup-storage "local://$TEST_DIR/$PREFIX/full" || restore_fail=1
 export GO_FAILPOINTS=""
 if [ $restore_fail -ne 1 ]; then
-    echo 'pitr success' 
+    echo 'pitr success on file lost'
     exit 1
 fi
 
@@ -198,6 +198,6 @@ restore_fail=0
 run_br --pd $PD_ADDR restore point -s "local://$TEST_DIR/$PREFIX/log" --full-backup-storage "local://$TEST_DIR/$PREFIX/full" || restore_fail=1
 export GO_FAILPOINTS=""
 if [ $restore_fail -ne 1 ]; then
-    echo 'pitr success' 
+    echo 'pitr success on file corruption'
     exit 1
 fi

From aeca72cb93767a01f5ab39aa6789ca31ebac9d67 Mon Sep 17 00:00:00 2001
From: Wenqi Mou <wenqimou@gmail.com>
Date: Fri, 4 Oct 2024 21:32:32 -0400
Subject: [PATCH 07/19] test local master key

Signed-off-by: Wenqi Mou <wenqimou@gmail.com>
---
 br/tests/br_check_new_collocation_enable/run.sh |  6 +++---
 br/tests/br_crypter/run.sh                      |  6 +++---
 br/tests/br_crypter2/run.sh                     |  6 +++---
 br/tests/br_encryption/run.sh                   |  6 +++---
 br/tests/br_gcs/run.sh                          |  4 ++--
 br/tests/br_rawkv/run.sh                        |  6 +++---
 br/tests/br_txn/run.sh                          |  6 +++---
 br/tests/run.sh                                 | 11 +++++++++++
 tests/_utils/run_br                             |  9 ---------
 9 files changed, 31 insertions(+), 29 deletions(-)

diff --git a/br/tests/br_check_new_collocation_enable/run.sh b/br/tests/br_check_new_collocation_enable/run.sh
index d9ed7bc7a32ac..b453436abe996 100755
--- a/br/tests/br_check_new_collocation_enable/run.sh
+++ b/br/tests/br_check_new_collocation_enable/run.sh
@@ -14,9 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# disable encryption as not working in this backward compatible test
-ENABLE_ENCRYPTION=false
-export ENABLE_ENCRYPTION
+# disable global ENCRYPTION_ARGS for this script
+ENCRYPTION_ARGS=""
+export ENCRYPTION_ARGS
 
 set -eu
 DB="$TEST_NAME"
diff --git a/br/tests/br_crypter/run.sh b/br/tests/br_crypter/run.sh
index 3c230ae8cbe42..445c8ed051a47 100755
--- a/br/tests/br_crypter/run.sh
+++ b/br/tests/br_crypter/run.sh
@@ -14,9 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# disable global ENABLE_ENCRYPTION for this script
-ENABLE_ENCRYPTION=false
-export ENABLE_ENCRYPTION
+# disable global ENCRYPTION_ARGS for this script
+ENCRYPTION_ARGS=""
+export ENCRYPTION_ARGS
 
 set -eu
 DB="$TEST_NAME"
diff --git a/br/tests/br_crypter2/run.sh b/br/tests/br_crypter2/run.sh
index 3ec94d0a3ae57..cf37559093690 100755
--- a/br/tests/br_crypter2/run.sh
+++ b/br/tests/br_crypter2/run.sh
@@ -14,9 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# disable global ENABLE_ENCRYPTION for this script
-ENABLE_ENCRYPTION=false
-export ENABLE_ENCRYPTION
+# disable global ENCRYPTION_ARGS for this script
+ENCRYPTION_ARGS=""
+export ENCRYPTION_ARGS
 
 set -eu
 DB="$TEST_NAME"
diff --git a/br/tests/br_encryption/run.sh b/br/tests/br_encryption/run.sh
index 97014faebeec4..28fa2abfd1e3f 100755
--- a/br/tests/br_encryption/run.sh
+++ b/br/tests/br_encryption/run.sh
@@ -14,9 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# disable global ENABLE_ENCRYPTION for this script
-ENABLE_ENCRYPTION=false
-export ENABLE_ENCRYPTION
+# disable global ENCRYPTION_ARGS for this script
+ENCRYPTION_ARGS=""
+export ENCRYPTION_ARGS
 
 set -eu
 . run_services
diff --git a/br/tests/br_gcs/run.sh b/br/tests/br_gcs/run.sh
index 192236a46a6b3..8864020aa9b20 100755
--- a/br/tests/br_gcs/run.sh
+++ b/br/tests/br_gcs/run.sh
@@ -15,8 +15,8 @@
 # limitations under the License.
 
 # disable encryption as not working with brv4.0.8
-ENABLE_ENCRYPTION=false
-export ENABLE_ENCRYPTION
+ENCRYPTION_ARGS=""
+export ENCRYPTION_ARGS
 
 set -eux
 DB="$TEST_NAME"
diff --git a/br/tests/br_rawkv/run.sh b/br/tests/br_rawkv/run.sh
index dd2a846f1188a..6349f2654b816 100755
--- a/br/tests/br_rawkv/run.sh
+++ b/br/tests/br_rawkv/run.sh
@@ -14,9 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# disable global ENABLE_ENCRYPTION for this script
-ENABLE_ENCRYPTION=false
-export ENABLE_ENCRYPTION
+# disable global ENCRYPTION_ARGS for this script
+ENCRYPTION_ARGS=""
+export ENCRYPTION_ARGS
 
 set -eux
 
diff --git a/br/tests/br_txn/run.sh b/br/tests/br_txn/run.sh
index 8293d04fd8d33..594eabb52c6e9 100755
--- a/br/tests/br_txn/run.sh
+++ b/br/tests/br_txn/run.sh
@@ -14,9 +14,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# disable global ENABLE_ENCRYPTION for this script
-ENABLE_ENCRYPTION=false
-export ENABLE_ENCRYPTION
+# disable global ENCRYPTION_ARGS for this script
+ENCRYPTION_ARGS=""
+export ENCRYPTION_ARGS
 
 set -eux
 
diff --git a/br/tests/run.sh b/br/tests/run.sh
index 4490b6de40dcc..6b5318c3a5844 100755
--- a/br/tests/run.sh
+++ b/br/tests/run.sh
@@ -48,6 +48,17 @@ start_services $@
 run_curl "https://$PD_ADDR/pd/api/v1/version" | grep -o 'v[0-9.]\+' > "$TEST_DIR/cluster_version.txt"
 IFS='.' read CLUSTER_VERSION_MAJOR CLUSTER_VERSION_MINOR CLUSTER_VERSION_REVISION < "$TEST_DIR/cluster_version.txt"
 
+# enable local encryption for all tests if needed
+# generate local disk master key file
+MASTER_KEY_DIR="$TEST_DIR/master_key"
+mkdir -p "$MASTER_KEY_DIR"
+openssl rand -hex 32 > "$MASTER_KEY_DIR/master.key"
+MASTER_KEY_PATH="local:///$MASTER_KEY_DIR/master.key"
+
+ENCRYPTION_ARGS="--crypter.method aes128-ctr --crypter.key 0123456789abcdef0123456789abcdef --master-key-crypter-method AES256-CTR --master-key $MASTER_KEY_PATH"
+#ENCRYPTION_ARGS="--crypter.method aes128-ctr --crypter.key 0123456789abcdef0123456789abcdef --log.crypter.method AES256-CTR --log.crypter.key 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
+export ENCRYPTION_ARGS
+
 if [ "${1-}" = '--debug' ]; then
     echo 'You may now debug from another terminal. Press [ENTER] to continue.'
     read line
diff --git a/tests/_utils/run_br b/tests/_utils/run_br
index 0380dd4fe7a0a..5ed6d94a10d17 100755
--- a/tests/_utils/run_br
+++ b/tests/_utils/run_br
@@ -16,15 +16,6 @@
 
 set -eux
 
-# Add a new environment variable to control encryption
-ENABLE_ENCRYPTION=${ENABLE_ENCRYPTION:-false}
-
-# Prepare encryption arguments
-ENCRYPTION_ARGS=""
-if [ "$ENABLE_ENCRYPTION" = "true" ]; then
-    ENCRYPTION_ARGS="--crypter.method aes128-ctr --crypter.key 0123456789abcdef0123456789abcdef --log.crypter.method AES256-CTR --log.crypter.key 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
-fi
-
 br.test -test.coverprofile="$COV_DIR/cov.$TEST_NAME.$$.out.log" DEVEL "$@" \
     -L "info" \
     --ca "$TEST_DIR/certs/ca.pem" \

From 2c4622fc425496ea31a18c22070a06977c44e8ba Mon Sep 17 00:00:00 2001
From: Wenqi Mou <wenqimou@gmail.com>
Date: Mon, 7 Oct 2024 18:45:48 -0400
Subject: [PATCH 08/19] better version to check on files

Signed-off-by: Wenqi Mou <wenqimou@gmail.com>
---
 Makefile                                      |   3 +-
 br/tests/BUILD.bazel                          |  15 ++
 .../br_check_new_collocation_enable/run.sh    |   4 +-
 br/tests/br_crypter/run.sh                    |   4 +-
 br/tests/br_crypter2/run.sh                   |   4 +-
 br/tests/br_encryption/run.sh                 |   4 +-
 br/tests/br_gcs/run.sh                        |   2 +
 br/tests/br_rawkv/run.sh                      |   4 +-
 br/tests/br_txn/run.sh                        |   4 +-
 br/tests/run.sh                               |  13 +-
 br/tests/run_br                               |  52 +++++
 br/tests/utils.go                             | 203 ++++++++++++++++++
 tests/_utils/run_br                           |   3 +-
 13 files changed, 303 insertions(+), 12 deletions(-)
 create mode 100644 br/tests/BUILD.bazel
 create mode 100755 br/tests/run_br
 create mode 100644 br/tests/utils.go

diff --git a/Makefile b/Makefile
index 5e20aa9d20c5a..fca15ee7874f7 100644
--- a/Makefile
+++ b/Makefile
@@ -436,7 +436,8 @@ build_for_br_integration_test:
 	$(GOBUILD) $(RACE_FLAG) -o bin/gc br/tests/br_z_gc_safepoint/*.go && \
 	$(GOBUILD) $(RACE_FLAG) -o bin/fake-oauth tools/fake-oauth/main.go && \
 	$(GOBUILD) $(RACE_FLAG) -o bin/rawkv br/tests/br_rawkv/*.go && \
-	$(GOBUILD) $(RACE_FLAG) -o bin/txnkv br/tests/br_txn/*.go \
+	$(GOBUILD) $(RACE_FLAG) -o bin/txnkv br/tests/br_txn/*.go && \
+	$(GOBUILD) $(RACE_FLAG) -o bin/utils br/tests/utils.go \
 	) || (make failpoint-disable && exit 1)
 	@make failpoint-disable
 
diff --git a/br/tests/BUILD.bazel b/br/tests/BUILD.bazel
new file mode 100644
index 0000000000000..1ef2874f7acde
--- /dev/null
+++ b/br/tests/BUILD.bazel
@@ -0,0 +1,15 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library")
+
+go_library(
+    name = "tests_lib",
+    srcs = ["utils.go"],
+    importpath = "github.com/pingcap/tidb/br/tests",
+    visibility = ["//visibility:public"],
+    deps = ["@com_github_klauspost_compress//zstd"],
+)
+
+go_binary(
+    name = "tests",
+    embed = [":tests_lib"],
+    visibility = ["//visibility:public"],
+)
diff --git a/br/tests/br_check_new_collocation_enable/run.sh b/br/tests/br_check_new_collocation_enable/run.sh
index b453436abe996..5635293782c8a 100755
--- a/br/tests/br_check_new_collocation_enable/run.sh
+++ b/br/tests/br_check_new_collocation_enable/run.sh
@@ -14,9 +14,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# disable global ENCRYPTION_ARGS for this script
+# disable global ENCRYPTION_ARGS for this script as not working with brv4.0.8
 ENCRYPTION_ARGS=""
+ENABLE_ENCRYPTION_CHECK=false
 export ENCRYPTION_ARGS
+export ENABLE_ENCRYPTION_CHECK
 
 set -eu
 DB="$TEST_NAME"
diff --git a/br/tests/br_crypter/run.sh b/br/tests/br_crypter/run.sh
index 445c8ed051a47..4a0a3879cd6fe 100755
--- a/br/tests/br_crypter/run.sh
+++ b/br/tests/br_crypter/run.sh
@@ -14,9 +14,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# disable global ENCRYPTION_ARGS for this script
+# disable global ENCRYPTION_ARGS and ENABLE_ENCRYPTION_CHECK for this script
 ENCRYPTION_ARGS=""
+ENABLE_ENCRYPTION_CHECK=false
 export ENCRYPTION_ARGS
+export ENABLE_ENCRYPTION_CHECK
 
 set -eu
 DB="$TEST_NAME"
diff --git a/br/tests/br_crypter2/run.sh b/br/tests/br_crypter2/run.sh
index cf37559093690..0ac18667c828f 100755
--- a/br/tests/br_crypter2/run.sh
+++ b/br/tests/br_crypter2/run.sh
@@ -14,9 +14,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# disable global ENCRYPTION_ARGS for this script
+# disable global ENCRYPTION_ARGS and ENABLE_ENCRYPTION_CHECK for this script
 ENCRYPTION_ARGS=""
+ENABLE_ENCRYPTION_CHECK=false
 export ENCRYPTION_ARGS
+export ENABLE_ENCRYPTION_CHECK
 
 set -eu
 DB="$TEST_NAME"
diff --git a/br/tests/br_encryption/run.sh b/br/tests/br_encryption/run.sh
index 28fa2abfd1e3f..3934dd3b6103c 100755
--- a/br/tests/br_encryption/run.sh
+++ b/br/tests/br_encryption/run.sh
@@ -14,9 +14,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# disable global ENCRYPTION_ARGS for this script
+# disable global ENCRYPTION_ARGS and ENABLE_ENCRYPTION_CHECK for this script
 ENCRYPTION_ARGS=""
+ENABLE_ENCRYPTION_CHECK=false
 export ENCRYPTION_ARGS
+export ENABLE_ENCRYPTION_CHECK
 
 set -eu
 . run_services
diff --git a/br/tests/br_gcs/run.sh b/br/tests/br_gcs/run.sh
index 8864020aa9b20..54ea9a68cf1d0 100755
--- a/br/tests/br_gcs/run.sh
+++ b/br/tests/br_gcs/run.sh
@@ -16,7 +16,9 @@
 
 # disable encryption as not working with brv4.0.8
 ENCRYPTION_ARGS=""
+ENABLE_ENCRYPTION_CHECK=false
 export ENCRYPTION_ARGS
+export ENABLE_ENCRYPTION_CHECK
 
 set -eux
 DB="$TEST_NAME"
diff --git a/br/tests/br_rawkv/run.sh b/br/tests/br_rawkv/run.sh
index 6349f2654b816..25933bfac629c 100755
--- a/br/tests/br_rawkv/run.sh
+++ b/br/tests/br_rawkv/run.sh
@@ -14,9 +14,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# disable global ENCRYPTION_ARGS for this script
+# disable global ENCRYPTION_ARGS and ENABLE_ENCRYPTION_CHECK for this script
 ENCRYPTION_ARGS=""
+ENABLE_ENCRYPTION_CHECK=false
 export ENCRYPTION_ARGS
+export ENABLE_ENCRYPTION_CHECK
 
 set -eux
 
diff --git a/br/tests/br_txn/run.sh b/br/tests/br_txn/run.sh
index 594eabb52c6e9..dec59ab11648d 100755
--- a/br/tests/br_txn/run.sh
+++ b/br/tests/br_txn/run.sh
@@ -14,9 +14,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# disable global ENCRYPTION_ARGS for this script
+# disable global ENCRYPTION_ARGS and ENABLE_ENCRYPTION_CHECK for this script
 ENCRYPTION_ARGS=""
+ENABLE_ENCRYPTION_CHECK=false
 export ENCRYPTION_ARGS
+export ENABLE_ENCRYPTION_CHECK
 
 set -eux
 
diff --git a/br/tests/run.sh b/br/tests/run.sh
index 6b5318c3a5844..45d79d2b5e808 100755
--- a/br/tests/run.sh
+++ b/br/tests/run.sh
@@ -17,7 +17,7 @@
 set -eu
 CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 export UTILS_DIR="$CUR/../../tests/_utils"
-export PATH="$PATH:$CUR/../../bin:$CUR/../bin:$UTILS_DIR"
+export PATH="$CUR:$PATH:$CUR/../../bin:$CUR/../bin:$UTILS_DIR"
 export TEST_DIR=/tmp/backup_restore_test
 export COV_DIR="/tmp/group_cover"
 mkdir -p $COV_DIR || true
@@ -48,15 +48,22 @@ start_services $@
 run_curl "https://$PD_ADDR/pd/api/v1/version" | grep -o 'v[0-9.]\+' > "$TEST_DIR/cluster_version.txt"
 IFS='.' read CLUSTER_VERSION_MAJOR CLUSTER_VERSION_MINOR CLUSTER_VERSION_REVISION < "$TEST_DIR/cluster_version.txt"
 
-# enable local encryption for all tests if needed
-# generate local disk master key file
+# enable encryption validation for all tests if needed, it will check all backup files are encrypted or not
+# based on the encryption settings in the test case
+ENABLE_ENCRYPTION_CHECK=true
+export ENABLE_ENCRYPTION_CHECK
+
+# generate local disk master key file to enable local encryption
 MASTER_KEY_DIR="$TEST_DIR/master_key"
 mkdir -p "$MASTER_KEY_DIR"
 openssl rand -hex 32 > "$MASTER_KEY_DIR/master.key"
 MASTER_KEY_PATH="local:///$MASTER_KEY_DIR/master.key"
 
 ENCRYPTION_ARGS="--crypter.method aes128-ctr --crypter.key 0123456789abcdef0123456789abcdef --master-key-crypter-method AES256-CTR --master-key $MASTER_KEY_PATH"
+
+# plaintext data key
 #ENCRYPTION_ARGS="--crypter.method aes128-ctr --crypter.key 0123456789abcdef0123456789abcdef --log.crypter.method AES256-CTR --log.crypter.key 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
+
 export ENCRYPTION_ARGS
 
 if [ "${1-}" = '--debug' ]; then
diff --git a/br/tests/run_br b/br/tests/run_br
new file mode 100755
index 0000000000000..2e755c9616d00
--- /dev/null
+++ b/br/tests/run_br
@@ -0,0 +1,52 @@
+#!/bin/sh
+
+# Copyright 2024 PingCAP, Inc. Licensed under Apache-2.0.
+
+set -eux
+
+main() {
+    # Call the original run.sh script
+    "$UTILS_DIR/run_br" "$@" $ENCRYPTION_ARGS
+
+    # Check if encryption validation is enabled
+    if [ "${ENABLE_ENCRYPTION_CHECK:-false}" != "true" ]; then
+        echo "Encryption check is disabled. Skipping backup file encryption validation."
+        exit 0
+    fi
+    
+    echo "Starting backup file encryption validation..."
+
+    # Capture the output of the validation command
+    output=$(bin/utils validateBackupFiles --command="$*" --encryption="$ENCRYPTION_ARGS")
+    exit_code=$?
+
+    if [ $exit_code -ne 0 ]; then
+        echo "Validation failed. Exiting with status 1."
+        exit 1
+    fi
+
+    # Check if validation is needed
+    if echo "$output" | grep -q "doesn't need to validate"; then
+        echo "Validation not required. Skipping."
+        exit 0
+    fi
+
+    # Check for expected strings in the output
+    if [ -n "$ENCRYPTION_ARGS" ]; then
+        if ! echo "$output" | grep -q "All files in .* are encrypted, as expected with encryption"; then
+            echo "Error: Expected 'All files are encrypted' message not found in output with encryption"
+            exit 1
+        fi
+    else
+        if ! echo "$output" | grep -q "All files in .* are not encrypted, as expected without encryption"; then
+            echo "Error: Expected 'All files are not encrypted' message not found in output without encryption"
+            exit 1
+        fi
+    fi
+
+    echo "Validation completed successfully."
+}
+
+# Execute the main function
+main "$@"
+
diff --git a/br/tests/utils.go b/br/tests/utils.go
new file mode 100644
index 0000000000000..3793b66e01a00
--- /dev/null
+++ b/br/tests/utils.go
@@ -0,0 +1,203 @@
+// Copyright 2024 PingCAP, Inc. Licensed under Apache-2.0.
+
+package main
+
+import (
+	"encoding/binary"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/klauspost/compress/zstd"
+)
+
+const (
+	cmdValidateBackupFiles = "validateBackupFiles"
+	extSST                 = ".sst"
+	extLOG                 = ".log"
+)
+
+func main() {
+	if len(os.Args) < 2 {
+		fmt.Println("Usage: go run utils.go <command> [arguments]")
+		fmt.Println("Available commands:")
+		fmt.Println("  validateBackupFiles")
+		os.Exit(1)
+	}
+
+	switch os.Args[1] {
+	case cmdValidateBackupFiles:
+		if !validateBackupFiles(os.Args[2:]) {
+			fmt.Println("validation failed")
+			os.Exit(1)
+		}
+	default:
+		fmt.Printf("Unknown command: %s\n", os.Args[1])
+		os.Exit(1)
+	}
+}
+
+func validateBackupFiles(args []string) bool {
+	validateCmd := flag.NewFlagSet(cmdValidateBackupFiles, flag.ExitOnError)
+	cmd := validateCmd.String("command", "", "Backup or restore command")
+	encryptionArg := validateCmd.String("encryption", "", "Encryption argument")
+
+	validateCmd.Parse(args)
+
+	if *cmd == "" {
+		fmt.Println("Please provide the full backup or restore command using --command flag")
+		validateCmd.PrintDefaults()
+		return false
+	}
+
+	storagePath, found := parseCommand(*cmd)
+	// doesn't need to validate if it's not doing backup/restore
+	if !found {
+		fmt.Println("doesn't need to validate")
+		return true
+	}
+
+	fmt.Printf("Validating files in: %s\n", storagePath)
+	return checkCompressionAndEncryption(storagePath, *encryptionArg)
+}
+
+func parseCommand(cmd string) (string, bool) {
+	args := strings.Fields(cmd)
+	hasBackupOrRestore := false
+	storagePath := ""
+
+	for i, arg := range args {
+		if arg == "backup" || arg == "restore" {
+			hasBackupOrRestore = true
+		}
+		if arg == "-s" && i+1 < len(args) && strings.HasPrefix(args[i+1], "local://") {
+			storagePath = strings.TrimPrefix(args[i+1], "local://")
+		}
+	}
+
+	if hasBackupOrRestore && storagePath != "" {
+		return storagePath, true
+	}
+	return "", false
+}
+
+func checkCompressionAndEncryption(dir string, encryptionArg string) bool {
+	allEncrypted := true
+	allUnencrypted := true
+	totalFiles := 0
+
+	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if info.IsDir() {
+			return nil
+		}
+
+		if strings.HasSuffix(path, extSST) {
+			totalFiles++
+			isValidSST, err := isLikelySSTFile(path)
+			if err != nil {
+				fmt.Printf("Error checking SST file %s: %v\n", path, err)
+				return err
+			}
+			if isValidSST {
+				allEncrypted = false
+			} else {
+				allUnencrypted = false
+			}
+		} else if strings.HasSuffix(path, extLOG) {
+			totalFiles++
+			isCompressed, err := isZstdCompressed(path)
+			if err != nil {
+				fmt.Printf("Error checking if file is encrypted %s: %v\n", path, err)
+				return err
+			}
+			if isCompressed {
+				allEncrypted = false
+			} else {
+				allUnencrypted = false
+			}
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		fmt.Printf("Error walking through directory: %v\n", err)
+		os.Exit(1)
+	}
+
+	if encryptionArg != "" {
+		if allEncrypted {
+			fmt.Printf("All files in %s are encrypted, as expected with encryption\n", dir)
+			return true
+		} else {
+			fmt.Printf("Error: Some files in %s are not encrypted, which is unexpected with encryption\n", dir)
+			return false
+		}
+	} else {
+		if allUnencrypted {
+			fmt.Printf("All files in %s are not encrypted, as expected without encryption\n", dir)
+			return true
+		} else if allEncrypted {
+			fmt.Printf("Error: All files in %s are encrypted, which is unexpected without encryption\n", dir)
+			return false
+		} else {
+			fmt.Printf("Error: Mixed encryption in %s. Some files are encrypted, some are not.\n", dir)
+			return false
+		}
+	}
+}
+
+func isZstdCompressed(filePath string) (bool, error) {
+	file, err := os.Open(filePath)
+	if err != nil {
+		return false, err
+	}
+	defer file.Close()
+
+	decoder, err := zstd.NewReader(file)
+	if err != nil {
+		return false, nil // Not compressed or error in compression
+	}
+	defer decoder.Close()
+
+	// Try to read a small amount of data
+	_, err = decoder.Read(make([]byte, 1))
+	if err != nil {
+		return false, nil // Not compressed or error in decompression
+	}
+
+	return true, nil
+}
+
+func isLikelySSTFile(filePath string) (bool, error) {
+	file, err := os.Open(filePath)
+	if err != nil {
+		return false, err
+	}
+	defer file.Close()
+
+	// Seek to 8 bytes from the end of the file
+	_, err = file.Seek(-8, io.SeekEnd)
+	if err != nil {
+		return false, err
+	}
+
+	// Read the last 8 bytes
+	footer := make([]byte, 8)
+	_, err = file.Read(footer)
+	if err != nil {
+		return false, err
+	}
+
+	// Check for SST magic number (kLegacyBlockBasedTableMagicNumber)
+	// or (kBlockBasedTableMagicNumber)
+	magicNumber := binary.LittleEndian.Uint64(footer)
+	return magicNumber == 0xdb4775248b80fb57 || magicNumber == 0x88e241b785f4cff7, nil
+}
diff --git a/tests/_utils/run_br b/tests/_utils/run_br
index 5ed6d94a10d17..d76f01f5d263b 100755
--- a/tests/_utils/run_br
+++ b/tests/_utils/run_br
@@ -20,5 +20,4 @@ br.test -test.coverprofile="$COV_DIR/cov.$TEST_NAME.$$.out.log" DEVEL "$@" \
     -L "info" \
     --ca "$TEST_DIR/certs/ca.pem" \
     --cert "$TEST_DIR/certs/br.pem" \
-    --key "$TEST_DIR/certs/br.key" \
-    $ENCRYPTION_ARGS
+    --key "$TEST_DIR/certs/br.key" \
\ No newline at end of file

From 4f4571f4d4dad1c43860c918e79c2d8f37ad8696 Mon Sep 17 00:00:00 2001
From: Wenqi Mou <wenqimou@gmail.com>
Date: Mon, 7 Oct 2024 20:56:12 -0400
Subject: [PATCH 09/19] fix style

Signed-off-by: Wenqi Mou <wenqimou@gmail.com>
---
 br/tests/run_br   |  2 +-
 br/tests/utils.go | 22 ++++++++++++----------
 2 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/br/tests/run_br b/br/tests/run_br
index 2e755c9616d00..e2eb14f7216af 100755
--- a/br/tests/run_br
+++ b/br/tests/run_br
@@ -26,7 +26,7 @@ main() {
     fi
 
     # Check if validation is needed
-    if echo "$output" | grep -q "doesn't need to validate"; then
+    if echo "$output" | grep -q "No need to validate"; then
         echo "Validation not required. Skipping."
         exit 0
     fi
diff --git a/br/tests/utils.go b/br/tests/utils.go
index 3793b66e01a00..ef27938b39e41 100644
--- a/br/tests/utils.go
+++ b/br/tests/utils.go
@@ -45,7 +45,11 @@ func validateBackupFiles(args []string) bool {
 	cmd := validateCmd.String("command", "", "Backup or restore command")
 	encryptionArg := validateCmd.String("encryption", "", "Encryption argument")
 
-	validateCmd.Parse(args)
+	err := validateCmd.Parse(args)
+	if err != nil {
+		fmt.Println("Failed to parse arguments")
+		return false
+	}
 
 	if *cmd == "" {
 		fmt.Println("Please provide the full backup or restore command using --command flag")
@@ -56,7 +60,7 @@ func validateBackupFiles(args []string) bool {
 	storagePath, found := parseCommand(*cmd)
 	// doesn't need to validate if it's not doing backup/restore
 	if !found {
-		fmt.Println("doesn't need to validate")
+		fmt.Println("No need to validate")
 		return true
 	}
 
@@ -136,10 +140,9 @@ func checkCompressionAndEncryption(dir string, encryptionArg string) bool {
 		if allEncrypted {
 			fmt.Printf("All files in %s are encrypted, as expected with encryption\n", dir)
 			return true
-		} else {
-			fmt.Printf("Error: Some files in %s are not encrypted, which is unexpected with encryption\n", dir)
-			return false
 		}
+		fmt.Printf("Error: Some files in %s are not encrypted, which is unexpected with encryption\n", dir)
+		return false
 	} else {
 		if allUnencrypted {
 			fmt.Printf("All files in %s are not encrypted, as expected without encryption\n", dir)
@@ -147,19 +150,18 @@ func checkCompressionAndEncryption(dir string, encryptionArg string) bool {
 		} else if allEncrypted {
 			fmt.Printf("Error: All files in %s are encrypted, which is unexpected without encryption\n", dir)
 			return false
-		} else {
-			fmt.Printf("Error: Mixed encryption in %s. Some files are encrypted, some are not.\n", dir)
-			return false
 		}
+		fmt.Printf("Error: Mixed encryption in %s. Some files are encrypted, some are not.\n", dir)
+		return false
 	}
 }
 
 func isZstdCompressed(filePath string) (bool, error) {
 	file, err := os.Open(filePath)
+	defer file.Close()
 	if err != nil {
 		return false, err
 	}
-	defer file.Close()
 
 	decoder, err := zstd.NewReader(file)
 	if err != nil {
@@ -178,10 +180,10 @@ func isZstdCompressed(filePath string) (bool, error) {
 
 func isLikelySSTFile(filePath string) (bool, error) {
 	file, err := os.Open(filePath)
+	defer file.Close()
 	if err != nil {
 		return false, err
 	}
-	defer file.Close()
 
 	// Seek to 8 bytes from the end of the file
 	_, err = file.Seek(-8, io.SeekEnd)

From 835d8ea8730cfae5fbf90a1914841973fc8dd897 Mon Sep 17 00:00:00 2001
From: Wenqi Mou <wenqimou@gmail.com>
Date: Mon, 7 Oct 2024 21:09:08 -0400
Subject: [PATCH 10/19] fix style

Signed-off-by: Wenqi Mou <wenqimou@gmail.com>
---
 br/tests/utils.go | 30 +++++++++++++++++-------------
 1 file changed, 17 insertions(+), 13 deletions(-)

diff --git a/br/tests/utils.go b/br/tests/utils.go
index ef27938b39e41..d7104988f38f5 100644
--- a/br/tests/utils.go
+++ b/br/tests/utils.go
@@ -136,6 +136,7 @@ func checkCompressionAndEncryption(dir string, encryptionArg string) bool {
 		os.Exit(1)
 	}
 
+	// handle with encryption case
 	if encryptionArg != "" {
 		if allEncrypted {
 			fmt.Printf("All files in %s are encrypted, as expected with encryption\n", dir)
@@ -143,25 +144,27 @@ func checkCompressionAndEncryption(dir string, encryptionArg string) bool {
 		}
 		fmt.Printf("Error: Some files in %s are not encrypted, which is unexpected with encryption\n", dir)
 		return false
-	} else {
-		if allUnencrypted {
-			fmt.Printf("All files in %s are not encrypted, as expected without encryption\n", dir)
-			return true
-		} else if allEncrypted {
-			fmt.Printf("Error: All files in %s are encrypted, which is unexpected without encryption\n", dir)
-			return false
-		}
-		fmt.Printf("Error: Mixed encryption in %s. Some files are encrypted, some are not.\n", dir)
+	}
+
+	// handle without encryption case
+	if allUnencrypted {
+		fmt.Printf("All files in %s are not encrypted, as expected without encryption\n", dir)
+		return true
+	} else if allEncrypted {
+		fmt.Printf("Error: All files in %s are encrypted, which is unexpected without encryption\n", dir)
 		return false
 	}
+	fmt.Printf("Error: Mixed encryption in %s. Some files are encrypted, some are not.\n", dir)
+	return false
 }
 
 func isZstdCompressed(filePath string) (bool, error) {
-	file, err := os.Open(filePath)
-	defer file.Close()
+	cleanedPath := filepath.Clean(filePath)
+	file, err := os.Open(cleanedPath)
 	if err != nil {
 		return false, err
 	}
+	defer file.Close()
 
 	decoder, err := zstd.NewReader(file)
 	if err != nil {
@@ -179,11 +182,12 @@ func isZstdCompressed(filePath string) (bool, error) {
 }
 
 func isLikelySSTFile(filePath string) (bool, error) {
-	file, err := os.Open(filePath)
-	defer file.Close()
+	cleanedPath := filepath.Clean(filePath)
+	file, err := os.Open(cleanedPath)
 	if err != nil {
 		return false, err
 	}
+	defer file.Close()
 
 	// Seek to 8 bytes from the end of the file
 	_, err = file.Seek(-8, io.SeekEnd)

From 720b7db81e18d4891a7a20c2fca985172d9860b1 Mon Sep 17 00:00:00 2001
From: Wenqi Mou <wenqimou@gmail.com>
Date: Mon, 7 Oct 2024 21:28:47 -0400
Subject: [PATCH 11/19] another try

Signed-off-by: Wenqi Mou <wenqimou@gmail.com>
---
 br/tests/utils.go | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/br/tests/utils.go b/br/tests/utils.go
index d7104988f38f5..09ebf64490e5b 100644
--- a/br/tests/utils.go
+++ b/br/tests/utils.go
@@ -159,8 +159,7 @@ func checkCompressionAndEncryption(dir string, encryptionArg string) bool {
 }
 
 func isZstdCompressed(filePath string) (bool, error) {
-	cleanedPath := filepath.Clean(filePath)
-	file, err := os.Open(cleanedPath)
+	file, err := os.OpenFile(filePath, os.O_RDONLY, 0)
 	if err != nil {
 		return false, err
 	}
@@ -182,8 +181,7 @@ func isZstdCompressed(filePath string) (bool, error) {
 }
 
 func isLikelySSTFile(filePath string) (bool, error) {
-	cleanedPath := filepath.Clean(filePath)
-	file, err := os.Open(cleanedPath)
+	file, err := os.OpenFile(filePath, os.O_RDONLY, 0)
 	if err != nil {
 		return false, err
 	}

From de48d89a6e43c87e7a376ce76092df9d766d9de1 Mon Sep 17 00:00:00 2001
From: Wenqi Mou <wenqimou@gmail.com>
Date: Mon, 7 Oct 2024 22:02:22 -0400
Subject: [PATCH 12/19] suppress link

Signed-off-by: Wenqi Mou <wenqimou@gmail.com>
---
 br/tests/utils.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/br/tests/utils.go b/br/tests/utils.go
index 09ebf64490e5b..24a662efa7e5a 100644
--- a/br/tests/utils.go
+++ b/br/tests/utils.go
@@ -159,7 +159,7 @@ func checkCompressionAndEncryption(dir string, encryptionArg string) bool {
 }
 
 func isZstdCompressed(filePath string) (bool, error) {
-	file, err := os.OpenFile(filePath, os.O_RDONLY, 0)
+	file, err := os.OpenFile(filePath, os.O_RDONLY, 0) //nolint:gosec
 	if err != nil {
 		return false, err
 	}
@@ -181,7 +181,7 @@ func isZstdCompressed(filePath string) (bool, error) {
 }
 
 func isLikelySSTFile(filePath string) (bool, error) {
-	file, err := os.OpenFile(filePath, os.O_RDONLY, 0)
+	file, err := os.OpenFile(filePath, os.O_RDONLY, 0) //nolint:gosec
 	if err != nil {
 		return false, err
 	}

From c18be90c63c3c094a024dfa450499e93726ca10c Mon Sep 17 00:00:00 2001
From: Wenqi Mou <wenqimou@gmail.com>
Date: Tue, 8 Oct 2024 15:09:32 -0400
Subject: [PATCH 13/19] address comments

Signed-off-by: Wenqi Mou <wenqimou@gmail.com>
---
 br/tests/BUILD.bazel |   5 ++-
 br/tests/run.sh      |   2 +-
 br/tests/run_br      |   3 ++
 br/tests/utils.go    | 100 +++++++++++++++++++++++++++----------------
 tests/_utils/run_br  |   2 +-
 5 files changed, 72 insertions(+), 40 deletions(-)

diff --git a/br/tests/BUILD.bazel b/br/tests/BUILD.bazel
index 1ef2874f7acde..a3e75646eab56 100644
--- a/br/tests/BUILD.bazel
+++ b/br/tests/BUILD.bazel
@@ -5,7 +5,10 @@ go_library(
     srcs = ["utils.go"],
     importpath = "github.com/pingcap/tidb/br/tests",
     visibility = ["//visibility:public"],
-    deps = ["@com_github_klauspost_compress//zstd"],
+    deps = [
+        "@com_github_klauspost_compress//zstd",
+        "@com_github_spf13_cobra//:cobra",
+    ],
 )
 
 go_binary(
diff --git a/br/tests/run.sh b/br/tests/run.sh
index 45d79d2b5e808..755f3bbdc6a47 100755
--- a/br/tests/run.sh
+++ b/br/tests/run.sh
@@ -57,7 +57,7 @@ export ENABLE_ENCRYPTION_CHECK
 MASTER_KEY_DIR="$TEST_DIR/master_key"
 mkdir -p "$MASTER_KEY_DIR"
 openssl rand -hex 32 > "$MASTER_KEY_DIR/master.key"
-MASTER_KEY_PATH="local:///$MASTER_KEY_DIR/master.key"
+MASTER_KEY_PATH="local://$MASTER_KEY_DIR/master.key"
 
 ENCRYPTION_ARGS="--crypter.method aes128-ctr --crypter.key 0123456789abcdef0123456789abcdef --master-key-crypter-method AES256-CTR --master-key $MASTER_KEY_PATH"
 
diff --git a/br/tests/run_br b/br/tests/run_br
index e2eb14f7216af..86b669100928b 100755
--- a/br/tests/run_br
+++ b/br/tests/run_br
@@ -26,6 +26,7 @@ main() {
     fi
 
     # Check if validation is needed
+    # strings are from utils.go
     if echo "$output" | grep -q "No need to validate"; then
         echo "Validation not required. Skipping."
         exit 0
@@ -33,11 +34,13 @@ main() {
 
     # Check for expected strings in the output
     if [ -n "$ENCRYPTION_ARGS" ]; then
+        # strings are from utils.go
         if ! echo "$output" | grep -q "All files in .* are encrypted, as expected with encryption"; then
             echo "Error: Expected 'All files are encrypted' message not found in output with encryption"
             exit 1
         fi
     else
+        # strings are from utils.go
         if ! echo "$output" | grep -q "All files in .* are not encrypted, as expected without encryption"; then
             echo "Error: Expected 'All files are not encrypted' message not found in output without encryption"
             exit 1
diff --git a/br/tests/utils.go b/br/tests/utils.go
index 24a662efa7e5a..e8653aaaabb43 100644
--- a/br/tests/utils.go
+++ b/br/tests/utils.go
@@ -4,7 +4,6 @@ package main
 
 import (
 	"encoding/binary"
-	"flag"
 	"fmt"
 	"io"
 	"os"
@@ -12,6 +11,7 @@ import (
 	"strings"
 
 	"github.com/klauspost/compress/zstd"
+	"github.com/spf13/cobra"
 )
 
 const (
@@ -21,68 +21,94 @@ const (
 )
 
 func main() {
-	if len(os.Args) < 2 {
-		fmt.Println("Usage: go run utils.go <command> [arguments]")
-		fmt.Println("Available commands:")
-		fmt.Println("  validateBackupFiles")
-		os.Exit(1)
+	rootCmd := &cobra.Command{
+		Use:   "utils",
+		Short: "Utility commands for backup and restore",
 	}
 
-	switch os.Args[1] {
-	case cmdValidateBackupFiles:
-		if !validateBackupFiles(os.Args[2:]) {
-			fmt.Println("validation failed")
-			os.Exit(1)
-		}
-	default:
-		fmt.Printf("Unknown command: %s\n", os.Args[1])
-		os.Exit(1)
+	validateCmd := &cobra.Command{
+		Use:   cmdValidateBackupFiles,
+		Short: "Validate backup files",
+		Run:   runValidateBackupFiles,
 	}
-}
 
-func validateBackupFiles(args []string) bool {
-	validateCmd := flag.NewFlagSet(cmdValidateBackupFiles, flag.ExitOnError)
-	cmd := validateCmd.String("command", "", "Backup or restore command")
-	encryptionArg := validateCmd.String("encryption", "", "Encryption argument")
+	validateCmd.Flags().String("command", "", "Backup or restore command")
+	validateCmd.Flags().String("encryption", "", "Encryption argument")
 
-	err := validateCmd.Parse(args)
-	if err != nil {
-		fmt.Println("Failed to parse arguments")
-		return false
+	rootCmd.AddCommand(validateCmd)
+
+	if err := rootCmd.Execute(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
 	}
+}
 
-	if *cmd == "" {
+func runValidateBackupFiles(cmd *cobra.Command, args []string) {
+	command, _ := cmd.Flags().GetString("command")
+	encryptionArg, _ := cmd.Flags().GetString("encryption")
+
+	if command == "" {
 		fmt.Println("Please provide the full backup or restore command using --command flag")
-		validateCmd.PrintDefaults()
-		return false
+		err := cmd.Usage()
+		if err != nil {
+			fmt.Println("Usage error")
+			return
+		}
+		os.Exit(1)
 	}
 
-	storagePath, found := parseCommand(*cmd)
+	storagePath, found := parseCommand(command)
 	// doesn't need to validate if it's not doing backup/restore
 	if !found {
 		fmt.Println("No need to validate")
-		return true
+		return
 	}
 
 	fmt.Printf("Validating files in: %s\n", storagePath)
-	return checkCompressionAndEncryption(storagePath, *encryptionArg)
+	if !checkCompressionAndEncryption(storagePath, encryptionArg) {
+		fmt.Println("validation failed")
+		os.Exit(1)
+	}
 }
 
+// parseCommand parses the command and only returns the storage path if it's a full backup or restore point
+// as full backup will have backup files ready in the storage path after returning from the command
+// and log backup will not, so we can only use restore point to validate.
 func parseCommand(cmd string) (string, bool) {
+	// Create a temporary cobra command to parse the input
+	tempCmd := &cobra.Command{}
+	tempCmd.Flags().String("s", "", "Storage path (short)")
+	tempCmd.Flags().String("storage", "", "Storage path (long)")
+
+	// Split the command string into args
 	args := strings.Fields(cmd)
-	hasBackupOrRestore := false
-	storagePath := ""
 
+	// Parse the args
+	if err := tempCmd.Flags().Parse(args); err != nil {
+		return "", false
+	}
+
+	// Check for backup or restore point command
+	hasBackupOrRestorePoint := false
 	for i, arg := range args {
-		if arg == "backup" || arg == "restore" {
-			hasBackupOrRestore = true
+		if arg == "backup" {
+			hasBackupOrRestorePoint = true
+			break
 		}
-		if arg == "-s" && i+1 < len(args) && strings.HasPrefix(args[i+1], "local://") {
-			storagePath = strings.TrimPrefix(args[i+1], "local://")
+		if i < len(args)-1 && arg == "restore" && args[i+1] == "point" {
+			hasBackupOrRestorePoint = true
+			break
 		}
 	}
 
-	if hasBackupOrRestore && storagePath != "" {
+	// Get the storage path from either -s or -storage flag
+	storagePath, _ := tempCmd.Flags().GetString("s")
+	if storagePath == "" {
+		storagePath, _ = tempCmd.Flags().GetString("storage")
+	}
+	storagePath = strings.TrimPrefix(storagePath, "local://")
+
+	if hasBackupOrRestorePoint && storagePath != "" {
 		return storagePath, true
 	}
 	return "", false
diff --git a/tests/_utils/run_br b/tests/_utils/run_br
index d76f01f5d263b..743a25c456121 100755
--- a/tests/_utils/run_br
+++ b/tests/_utils/run_br
@@ -20,4 +20,4 @@ br.test -test.coverprofile="$COV_DIR/cov.$TEST_NAME.$$.out.log" DEVEL "$@" \
     -L "info" \
     --ca "$TEST_DIR/certs/ca.pem" \
     --cert "$TEST_DIR/certs/br.pem" \
-    --key "$TEST_DIR/certs/br.key" \
\ No newline at end of file
+    --key "$TEST_DIR/certs/br.key"

From c4a502d8da8c7dcb7fc804212483acb92a6e3bd6 Mon Sep 17 00:00:00 2001
From: Wenqi Mou <wenqimou@gmail.com>
Date: Tue, 8 Oct 2024 17:27:47 -0400
Subject: [PATCH 14/19] move around test to see if fails

Signed-off-by: Wenqi Mou <wenqimou@gmail.com>
---
 br/tests/run_group_br_tests.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/br/tests/run_group_br_tests.sh b/br/tests/run_group_br_tests.sh
index da7beb2beee53..57b43bd09018a 100755
--- a/br/tests/run_group_br_tests.sh
+++ b/br/tests/run_group_br_tests.sh
@@ -22,13 +22,13 @@ declare -A groups
 groups=(
 	["G00"]="br_300_small_tables br_backup_empty br_backup_version br_cache_table br_case_sensitive br_charset_gbk br_check_new_collocation_enable br_history br_gcs br_rawkv br_tidb_placement_policy br_tiflash br_tiflash_conflict"
 	["G01"]="br_autoid br_crypter2 br_db br_db_online br_db_online_newkv br_db_skip br_debug_meta br_ebs br_foreign_key br_full br_table_partition br_full_ddl br_mv_index"
-	["G02"]="br_full_cluster_restore br_full_index br_incremental_ddl br_pitr_failpoint br_pitr_gc_safepoint br_other br_partition_add_index"
+	["G02"]="br_full_cluster_restore br_full_index br_incremental_ddl br_pitr_failpoint br_pitr_gc_safepoint br_partition_add_index"
 	["G03"]='br_incompatible_tidb_config br_incremental br_incremental_index br_incremental_only_ddl br_incremental_same_table br_insert_after_restore br_key_locked br_log_test br_move_backup'
 	["G04"]='br_range br_replica_read br_restore_TDE_enable br_restore_log_task_enable br_s3 br_shuffle_leader br_shuffle_region br_single_table'
 	["G05"]='br_skip_checksum br_split_region_fail br_systables br_table_filter br_txn br_stats br_clustered_index br_crypter'
 	["G06"]='br_tikv_outage br_tikv_outage3 br_restore_checkpoint br_encryption'
 	["G07"]='br_pitr'
-	["G08"]='br_tikv_outage2 br_ttl br_views_and_sequences br_z_gc_safepoint br_autorandom br_file_corruption'
+	["G08"]='br_tikv_outage2 br_ttl br_views_and_sequences br_z_gc_safepoint br_autorandom br_file_corruption br_other'
 )
 
 # Get other cases not in groups, to avoid missing any case

From af6394d8a2b708e9d0c248a89a9ccf2b09fbc54a Mon Sep 17 00:00:00 2001
From: Wenqi Mou <wenqimou@gmail.com>
Date: Tue, 8 Oct 2024 21:53:01 -0400
Subject: [PATCH 15/19] don't move br_other, see if it still fails

Signed-off-by: Wenqi Mou <wenqimou@gmail.com>
---
 br/tests/run_group_br_tests.sh | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/br/tests/run_group_br_tests.sh b/br/tests/run_group_br_tests.sh
index 57b43bd09018a..9c830cb97b2ad 100755
--- a/br/tests/run_group_br_tests.sh
+++ b/br/tests/run_group_br_tests.sh
@@ -20,15 +20,15 @@ mkdir -p $COV_DIR
 # Putting multiple light tests together and heavy tests in a separate group.
 declare -A groups
 groups=(
-	["G00"]="br_300_small_tables br_backup_empty br_backup_version br_cache_table br_case_sensitive br_charset_gbk br_check_new_collocation_enable br_history br_gcs br_rawkv br_tidb_placement_policy br_tiflash br_tiflash_conflict"
-	["G01"]="br_autoid br_crypter2 br_db br_db_online br_db_online_newkv br_db_skip br_debug_meta br_ebs br_foreign_key br_full br_table_partition br_full_ddl br_mv_index"
-	["G02"]="br_full_cluster_restore br_full_index br_incremental_ddl br_pitr_failpoint br_pitr_gc_safepoint br_partition_add_index"
-	["G03"]='br_incompatible_tidb_config br_incremental br_incremental_index br_incremental_only_ddl br_incremental_same_table br_insert_after_restore br_key_locked br_log_test br_move_backup'
+	["G00"]="br_300_small_tables br_backup_empty br_backup_version br_cache_table br_case_sensitive br_charset_gbk br_check_new_collocation_enable br_history br_gcs br_rawkv br_partition_add_index"
+	["G01"]="br_autoid br_crypter2 br_db br_db_online br_db_online_newkv br_db_skip br_debug_meta br_ebs br_foreign_key br_full br_table_partition br_full_ddl"
+	["G02"]="br_full_cluster_restore br_full_index br_incremental_ddl br_pitr_failpoint br_pitr_gc_safepoint br_tiflash_conflict"
+	["G03"]='br_incompatible_tidb_config br_incremental br_incremental_index br_incremental_only_ddl br_incremental_same_table br_insert_after_restore br_key_locked br_log_test br_move_backup br_mv_index br_other'
 	["G04"]='br_range br_replica_read br_restore_TDE_enable br_restore_log_task_enable br_s3 br_shuffle_leader br_shuffle_region br_single_table'
-	["G05"]='br_skip_checksum br_split_region_fail br_systables br_table_filter br_txn br_stats br_clustered_index br_crypter'
+	["G05"]='br_skip_checksum br_split_region_fail br_systables br_table_filter br_txn br_stats br_clustered_index br_crypter br_tidb_placement_policy'
 	["G06"]='br_tikv_outage br_tikv_outage3 br_restore_checkpoint br_encryption'
 	["G07"]='br_pitr'
-	["G08"]='br_tikv_outage2 br_ttl br_views_and_sequences br_z_gc_safepoint br_autorandom br_file_corruption br_other'
+	["G08"]='br_tikv_outage2 br_ttl br_views_and_sequences br_z_gc_safepoint br_autorandom br_file_corruption br_tiflash'
 )
 
 # Get other cases not in groups, to avoid missing any case

From 7c6cf331c5956addf337e1df5f7b65f7e6e01925 Mon Sep 17 00:00:00 2001
From: Wenqi Mou <wenqimou@gmail.com>
Date: Tue, 8 Oct 2024 22:01:37 -0400
Subject: [PATCH 16/19] revert back to original, no idea what's going on

Signed-off-by: Wenqi Mou <wenqimou@gmail.com>
---
 br/tests/run_group_br_tests.sh | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/br/tests/run_group_br_tests.sh b/br/tests/run_group_br_tests.sh
index 9c830cb97b2ad..d0273924071ed 100755
--- a/br/tests/run_group_br_tests.sh
+++ b/br/tests/run_group_br_tests.sh
@@ -20,15 +20,15 @@ mkdir -p $COV_DIR
 # Putting multiple light tests together and heavy tests in a separate group.
 declare -A groups
 groups=(
-	["G00"]="br_300_small_tables br_backup_empty br_backup_version br_cache_table br_case_sensitive br_charset_gbk br_check_new_collocation_enable br_history br_gcs br_rawkv br_partition_add_index"
+	["G00"]="br_300_small_tables br_backup_empty br_backup_version br_cache_table br_case_sensitive br_charset_gbk br_check_new_collocation_enable br_history br_gcs br_rawkv"
 	["G01"]="br_autoid br_crypter2 br_db br_db_online br_db_online_newkv br_db_skip br_debug_meta br_ebs br_foreign_key br_full br_table_partition br_full_ddl"
-	["G02"]="br_full_cluster_restore br_full_index br_incremental_ddl br_pitr_failpoint br_pitr_gc_safepoint br_tiflash_conflict"
-	["G03"]='br_incompatible_tidb_config br_incremental br_incremental_index br_incremental_only_ddl br_incremental_same_table br_insert_after_restore br_key_locked br_log_test br_move_backup br_mv_index br_other'
+	["G02"]="br_full_cluster_restore br_full_index br_incremental_ddl br_pitr_failpoint br_pitr_gc_safepoint"
+	["G03"]='br_incompatible_tidb_config br_incremental br_incremental_index br_incremental_only_ddl br_incremental_same_table br_insert_after_restore br_key_locked br_log_test br_move_backup br_mv_index br_other br_partition_add_index br_tidb_placement_policy br_tiflash br_tiflash_conflict'
 	["G04"]='br_range br_replica_read br_restore_TDE_enable br_restore_log_task_enable br_s3 br_shuffle_leader br_shuffle_region br_single_table'
-	["G05"]='br_skip_checksum br_split_region_fail br_systables br_table_filter br_txn br_stats br_clustered_index br_crypter br_tidb_placement_policy'
+	["G05"]='br_skip_checksum br_split_region_fail br_systables br_table_filter br_txn br_stats br_clustered_index br_crypter'
 	["G06"]='br_tikv_outage br_tikv_outage3 br_restore_checkpoint br_encryption'
 	["G07"]='br_pitr'
-	["G08"]='br_tikv_outage2 br_ttl br_views_and_sequences br_z_gc_safepoint br_autorandom br_file_corruption br_tiflash'
+	["G08"]='br_tikv_outage2 br_ttl br_views_and_sequences br_z_gc_safepoint br_autorandom br_file_corruption'
 )
 
 # Get other cases not in groups, to avoid missing any case

From fcfa30989ca8b33acc165addf9c794e598514a58 Mon Sep 17 00:00:00 2001
From: Wenqi Mou <wenqimou@gmail.com>
Date: Wed, 9 Oct 2024 12:49:33 -0400
Subject: [PATCH 17/19] finally figure out what's going on...

Signed-off-by: Wenqi Mou <wenqimou@gmail.com>
---
 br/tests/br_other/run.sh       |  6 +++---
 br/tests/run.sh                |  3 ++-
 br/tests/run_group_br_tests.sh | 12 ++++++------
 tests/_utils/run_curl          |  2 +-
 4 files changed, 12 insertions(+), 11 deletions(-)

diff --git a/br/tests/br_other/run.sh b/br/tests/br_other/run.sh
index 71f937793bcbf..f1cdb2a11ce95 100644
--- a/br/tests/br_other/run.sh
+++ b/br/tests/br_other/run.sh
@@ -75,15 +75,15 @@ GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/utils/determined-pprof-port=return
 run_br --pd $PD_ADDR backup full -s "local://$TEST_DIR/$DB/lock" \
     --remove-schedulers \
     --ratelimit 1 \
-    --ratelimit-unit 1 \
-    --concurrency 4 &> $TEST_DIR/br-other-stdout.log & # It will be killed after test finish.
+    --ratelimit-unit 1 &> $TEST_DIR/br-other-stdout.log &# It will be killed after test finish.
 
 # record last backup pid
 _pid=$!
 
 # give the former backup some time to write down lock file (and initialize signal listener).
 sleep 1
-pkill -10 -P $_pid
+# don't use numeric value as it's mapped differently on different unix like system
+pkill -USR1 -P $_pid
 echo "starting pprof..."
 
 # give the former backup some time to write down lock file (and start pprof server).
diff --git a/br/tests/run.sh b/br/tests/run.sh
index 755f3bbdc6a47..7f8a75d1d0f08 100755
--- a/br/tests/run.sh
+++ b/br/tests/run.sh
@@ -59,9 +59,10 @@ mkdir -p "$MASTER_KEY_DIR"
 openssl rand -hex 32 > "$MASTER_KEY_DIR/master.key"
 MASTER_KEY_PATH="local://$MASTER_KEY_DIR/master.key"
 
+# testing encryption using local master key
 ENCRYPTION_ARGS="--crypter.method aes128-ctr --crypter.key 0123456789abcdef0123456789abcdef --master-key-crypter-method AES256-CTR --master-key $MASTER_KEY_PATH"
 
-# plaintext data key
+# uncomment it to test plaintext data key manually.
 #ENCRYPTION_ARGS="--crypter.method aes128-ctr --crypter.key 0123456789abcdef0123456789abcdef --log.crypter.method AES256-CTR --log.crypter.key 0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"
 
 export ENCRYPTION_ARGS
diff --git a/br/tests/run_group_br_tests.sh b/br/tests/run_group_br_tests.sh
index d0273924071ed..5064127142204 100755
--- a/br/tests/run_group_br_tests.sh
+++ b/br/tests/run_group_br_tests.sh
@@ -20,15 +20,15 @@ mkdir -p $COV_DIR
 # Putting multiple light tests together and heavy tests in a separate group.
 declare -A groups
 groups=(
-	["G00"]="br_300_small_tables br_backup_empty br_backup_version br_cache_table br_case_sensitive br_charset_gbk br_check_new_collocation_enable br_history br_gcs br_rawkv"
-	["G01"]="br_autoid br_crypter2 br_db br_db_online br_db_online_newkv br_db_skip br_debug_meta br_ebs br_foreign_key br_full br_table_partition br_full_ddl"
-	["G02"]="br_full_cluster_restore br_full_index br_incremental_ddl br_pitr_failpoint br_pitr_gc_safepoint"
-	["G03"]='br_incompatible_tidb_config br_incremental br_incremental_index br_incremental_only_ddl br_incremental_same_table br_insert_after_restore br_key_locked br_log_test br_move_backup br_mv_index br_other br_partition_add_index br_tidb_placement_policy br_tiflash br_tiflash_conflict'
+	["G00"]="br_300_small_tables br_backup_empty br_backup_version br_cache_table br_case_sensitive br_charset_gbk br_check_new_collocation_enable br_history br_gcs br_rawkv br_tidb_placement_policy"
+	["G01"]="br_autoid br_crypter2 br_db br_db_online br_db_online_newkv br_db_skip br_debug_meta br_ebs br_foreign_key br_full br_table_partition br_full_ddl br_tiflash"
+	["G02"]="br_full_cluster_restore br_full_index br_incremental_ddl br_pitr_failpoint br_pitr_gc_safepoint br_other"
+	["G03"]='br_incompatible_tidb_config br_incremental br_incremental_index br_incremental_only_ddl br_incremental_same_table br_insert_after_restore br_key_locked br_log_test br_move_backup br_mv_index'
 	["G04"]='br_range br_replica_read br_restore_TDE_enable br_restore_log_task_enable br_s3 br_shuffle_leader br_shuffle_region br_single_table'
-	["G05"]='br_skip_checksum br_split_region_fail br_systables br_table_filter br_txn br_stats br_clustered_index br_crypter'
+	["G05"]='br_skip_checksum br_split_region_fail br_systables br_table_filter br_txn br_stats br_clustered_index br_crypter br_partition_add_index'
 	["G06"]='br_tikv_outage br_tikv_outage3 br_restore_checkpoint br_encryption'
 	["G07"]='br_pitr'
-	["G08"]='br_tikv_outage2 br_ttl br_views_and_sequences br_z_gc_safepoint br_autorandom br_file_corruption'
+	["G08"]='br_tikv_outage2 br_ttl br_views_and_sequences br_z_gc_safepoint br_autorandom br_file_corruption br_tiflash_conflict'
 )
 
 # Get other cases not in groups, to avoid missing any case
diff --git a/tests/_utils/run_curl b/tests/_utils/run_curl
index 6023e0e44f336..7e4723dcb5b88 100755
--- a/tests/_utils/run_curl
+++ b/tests/_utils/run_curl
@@ -26,7 +26,7 @@ fi
 
 # FIXME: use `wget` instead of `curl` because the latter rejects ECC certs on our CI.
 # CentOS is re~~ally old.
-wget -q -O - \
+wget --verbose -O - \
     --ca-certificate="$TEST_DIR/certs/ca.pem" \
     --certificate="$TEST_DIR/certs/curl.pem" \
     --private-key="$TEST_DIR/certs/curl.key" \

From 9d2dc5030424d7789bc71068ba64235975c15b43 Mon Sep 17 00:00:00 2001
From: Wenqi Mou <wenqimou@gmail.com>
Date: Wed, 9 Oct 2024 12:52:09 -0400
Subject: [PATCH 18/19] forgot to revert

Signed-off-by: Wenqi Mou <wenqimou@gmail.com>
---
 br/tests/br_other/run.sh | 2 +-
 tests/_utils/run_curl    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/br/tests/br_other/run.sh b/br/tests/br_other/run.sh
index f1cdb2a11ce95..27c4c1638cb83 100644
--- a/br/tests/br_other/run.sh
+++ b/br/tests/br_other/run.sh
@@ -75,7 +75,7 @@ GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/utils/determined-pprof-port=return
 run_br --pd $PD_ADDR backup full -s "local://$TEST_DIR/$DB/lock" \
     --remove-schedulers \
     --ratelimit 1 \
-    --ratelimit-unit 1 &> $TEST_DIR/br-other-stdout.log &# It will be killed after test finish.
+    --ratelimit-unit 1 &> $TEST_DIR/br-other-stdout.log & # It will be killed after test finish.
 
 # record last backup pid
 _pid=$!
diff --git a/tests/_utils/run_curl b/tests/_utils/run_curl
index 7e4723dcb5b88..6023e0e44f336 100755
--- a/tests/_utils/run_curl
+++ b/tests/_utils/run_curl
@@ -26,7 +26,7 @@ fi
 
 # FIXME: use `wget` instead of `curl` because the latter rejects ECC certs on our CI.
 # CentOS is re~~ally old.
-wget --verbose -O - \
+wget -q -O - \
     --ca-certificate="$TEST_DIR/certs/ca.pem" \
     --certificate="$TEST_DIR/certs/curl.pem" \
     --private-key="$TEST_DIR/certs/curl.key" \

From 28005d778a31085d79efa7e59ca227ecad5de984 Mon Sep 17 00:00:00 2001
From: Wenqi Mou <wenqimou@gmail.com>
Date: Wed, 9 Oct 2024 14:56:56 -0400
Subject: [PATCH 19/19] confirm fixed

Signed-off-by: Wenqi Mou <wenqimou@gmail.com>
---
 br/tests/br_other/run.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/br/tests/br_other/run.sh b/br/tests/br_other/run.sh
index 27c4c1638cb83..4aee3c7e8f6ae 100644
--- a/br/tests/br_other/run.sh
+++ b/br/tests/br_other/run.sh
@@ -71,8 +71,9 @@ fi
 # backup full with ratelimit = 1 to make sure this backup task won't finish quickly
 echo "backup start to test lock file"
 PPROF_PORT=6080
+# not easy to trap USR1 signal in our custom run_br, so we use the original run_br to run the backup command
 GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/utils/determined-pprof-port=return($PPROF_PORT)" \
-run_br --pd $PD_ADDR backup full -s "local://$TEST_DIR/$DB/lock" \
+$UTILS_DIR/run_br --pd $PD_ADDR backup full -s "local://$TEST_DIR/$DB/lock" \
     --remove-schedulers \
     --ratelimit 1 \
     --ratelimit-unit 1 &> $TEST_DIR/br-other-stdout.log & # It will be killed after test finish.