Local example 101: idempotent on existing clusters (#13373)

* local example 101: idempotent Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * grep --quiet Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> * failing if waiting for super_read_only=false times out Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --------- Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com>
vitessio · Jun 27, 2023 · aa0fdff · aa0fdff
1 parent d4c5096
commit aa0fdff
Show file tree

Hide file tree

Showing 3 changed files with 50 additions and 9 deletions.
diff --git a/examples/common/lib/utils.sh b/examples/common/lib/utils.sh
@@ -38,9 +38,9 @@ function wait_for_shard_tablets() {
 	done;
 
 	cur_tablets=$(vtctldclient GetTablets --keyspace "${keyspace}" --shard "${shard}" | wc -l)
-        if [[ ${cur_tablets} -lt ${num_tablets} ]]; then
+	if [[ ${cur_tablets} -lt ${num_tablets} ]]; then
 		fail "Timed out after ${wait_secs} seconds waiting for tablets to come up in ${keyspace}/${shard}"
-        fi
+	fi
 }
 
 # Wait for a primary tablet to be elected and become healthy and serving
@@ -56,17 +56,43 @@ function wait_for_healthy_shard_primary() {
 	local wait_secs=180
 
 	for _ in $(seq 1 ${wait_secs}); do
-                if ! vtctldclient --server=localhost:15999 GetShard "${keyspace}/${shard}" | grep -qi "${unhealthy_indicator}"; then
+		if ! vtctldclient --server=localhost:15999 GetShard "${keyspace}/${shard}" | grep -qi "${unhealthy_indicator}"; then
 			break
 		fi
 		sleep 1
 	done;
 
-        if vtctldclient --server=localhost:15999 GetShard "${keyspace}/${shard}" | grep -qi "${unhealthy_indicator}"; then
+	if vtctldclient --server=localhost:15999 GetShard "${keyspace}/${shard}" | grep -qi "${unhealthy_indicator}"; then
 		fail "Timed out after ${wait_secs} seconds waiting for a primary tablet to be elected and become healthy in ${keyspace}/${shard}"
 	fi
 }
 
+
+# Wait for a primary tablet to be writeable, ie read_only=0 and super_read_only=0
+function wait_for_writeable_shard_primary() {
+	if [[ -z ${1} || -z ${2} ]]; then
+		fail "A keyspace and shard must be specified when waiting for the shard's primary to be healthy"
+	fi
+	local keyspace=${1}
+	local shard=${2}
+	local wait_secs=30
+
+	PRIMARY_TABLET="$(vtctldclient --server=localhost:15999 GetTablets --keyspace "$keyspace" --shard "$shard" | grep -w "primary" | awk '{print $1}')"
+	if [ -z "$PRIMARY_TABLET" ] ; then
+		fail "Cannot determine primary tablet for keyspace/shard $keyspace/$shard"
+	fi
+
+	for _ in $(seq 1 ${wait_secs}); do
+		if vtctldclient --server=localhost:15999 GetFullStatus "$PRIMARY_TABLET" | grep "super_read_only" | grep --quiet "false" ; then
+			break
+		fi
+		sleep 1
+	done
+	if vtctldclient --server=localhost:15999 GetFullStatus "$PRIMARY_TABLET" | grep "super_read_only" | grep --quiet "true" ; then
+		fail "Timed out after ${wait_secs} seconds waiting for a primary tablet $PRIMARY_TABLET to be writeable in ${keyspace}/${shard}"
+	fi
+}
+
 # Wait for the shard primary tablet's VReplication engine to open.
 # There is currently no API call or client command that can be specifically used
 # to check the VReplication engine's status (no vars in /debug/vars etc. either).
@@ -109,6 +135,7 @@ function wait_for_healthy_shard() {
 
 	wait_for_shard_tablets "${keyspace}" "${shard}" "${num_tablets}"
 	wait_for_healthy_shard_primary "${keyspace}" "${shard}"
+	wait_for_writeable_shard_primary "${keyspace}" "${shard}"
 	wait_for_shard_vreplication_engine "${keyspace}" "${shard}"
 }
 

diff --git a/examples/local/101_initial_cluster.sh b/examples/local/101_initial_cluster.sh
@@ -40,11 +40,16 @@ fi
 # start vtctld
 CELL=zone1 ../common/scripts/vtctld-up.sh
 
-# Create the keyspace with the sidecar database name and set the
-# correct durability policy. Please see the comment above for
-# more context on using a custom sidecar database name in your
-# Vitess clusters.
-vtctldclient --server localhost:15999 CreateKeyspace --sidecar-db-name="${SIDECAR_DB_NAME}" --durability-policy=semi_sync commerce || fail "Failed to create and configure the commerce keyspace"
+if vtctldclient GetKeyspace commerce > /dev/null 2>&1 ; then
+	# Keyspace already exists: we could be running this 101 example on an non-empty VTDATAROOT
+	vtctldclient --server localhost:15999 SetKeyspaceDurabilityPolicy --durability-policy=semi_sync commerce || fail "Failed to set keyspace durability policy on the commerce keyspace"
+else
+	# Create the keyspace with the sidecar database name and set the
+	# correct durability policy. Please see the comment above for
+	# more context on using a custom sidecar database name in your
+	# Vitess clusters.
+	vtctldclient --server localhost:15999 CreateKeyspace --sidecar-db-name="${SIDECAR_DB_NAME}" --durability-policy=semi_sync commerce || fail "Failed to create and configure the commerce keyspace"
+fi
 
 # start vttablets for keyspace commerce
 for i in 100 101 102; do

diff --git a/test/local_example.sh b/test/local_example.sh
@@ -18,6 +18,10 @@
 # It should be kept in sync with the steps in https://vitess.io/docs/get-started/local/
 # So we can detect if a regression affecting a tutorial is introduced.
 
+killall_vtdataroot() {
+  pkill -9 -e -f '(vtdataroot|VTDATAROOT)' # kill Vitess processes
+}
+
 source build.env
 
 set -xeo pipefail
@@ -28,7 +32,12 @@ unset VTROOT # ensure that the examples can run without VTROOT now.
 source ../common/env.sh # Required so that "mysql" works from alias
 
 ./101_initial_cluster.sh
+sleep 5 # Give vtgate time to really start.
 
+killall_vtdataroot
+# verify local example is able to start on an existing setup
+
+./101_initial_cluster.sh
 sleep 5 # Give vtgate time to really start.
 
 mysql < ../common/insert_commerce_data.sql