Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Local example 101: idempotent on existing clusters #13373

Merged
merged 3 commits into from
Jun 27, 2023
Merged

Local example 101: idempotent on existing clusters #13373

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
730100d
local example 101: idempotent
shlomi-noach Jun 25, 2023
3010d5f
grep --quiet
shlomi-noach Jun 25, 2023
043e711
failing if waiting for super_read_only=false times out
shlomi-noach Jun 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 28 additions & 4 deletions examples/common/lib/utils.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,9 +38,9 @@ function wait_for_shard_tablets() {
done;

cur_tablets=$(vtctldclient GetTablets --keyspace "${keyspace}" --shard "${shard}" | wc -l)
if [[ ${cur_tablets} -lt ${num_tablets} ]]; then
if [[ ${cur_tablets} -lt ${num_tablets} ]]; then
fail "Timed out after ${wait_secs} seconds waiting for tablets to come up in ${keyspace}/${shard}"
fi
fi
}

# Wait for a primary tablet to be elected and become healthy and serving
Expand All @@ -56,17 +56,40 @@ function wait_for_healthy_shard_primary() {
local wait_secs=180

for _ in $(seq 1 ${wait_secs}); do
if ! vtctldclient --server=localhost:15999 GetShard "${keyspace}/${shard}" | grep -qi "${unhealthy_indicator}"; then
if ! vtctldclient --server=localhost:15999 GetShard "${keyspace}/${shard}" | grep -qi "${unhealthy_indicator}"; then
break
fi
sleep 1
done;

if vtctldclient --server=localhost:15999 GetShard "${keyspace}/${shard}" | grep -qi "${unhealthy_indicator}"; then
if vtctldclient --server=localhost:15999 GetShard "${keyspace}/${shard}" | grep -qi "${unhealthy_indicator}"; then
fail "Timed out after ${wait_secs} seconds waiting for a primary tablet to be elected and become healthy in ${keyspace}/${shard}"
fi
}


# Wait for a primary tablet to be writeable, ie read_only=0 and super_read_only=0
function wait_for_writeable_shard_primary() {
if [[ -z ${1} || -z ${2} ]]; then
fail "A keyspace and shard must be specified when waiting for the shard's primary to be healthy"
fi
local keyspace=${1}
local shard=${2}
local wait_secs=30

PRIMARY_TABLET="$(vtctldclient --server=localhost:15999 GetTablets --keyspace "$keyspace" --shard "$shard" | grep -w "primary" | awk '{print $1}')"
if [ -z "$PRIMARY_TABLET" ] ; then
fail "Cannot determine primary tablet for keyspace/shard $keyspace/$shard"
fi

for _ in $(seq 1 ${wait_secs}); do
if vtctldclient --server=localhost:15999 GetFullStatus "$PRIMARY_TABLET" | grep "super_read_only" | grep --quiet "false" ; then
break
fi
sleep 1
done;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shouldn't we fail if we did not find super_read_only after wait_secs has elapsed?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done

}

# Wait for the shard primary tablet's VReplication engine to open.
# There is currently no API call or client command that can be specifically used
# to check the VReplication engine's status (no vars in /debug/vars etc. either).
Expand Down Expand Up @@ -109,6 +132,7 @@ function wait_for_healthy_shard() {

wait_for_shard_tablets "${keyspace}" "${shard}" "${num_tablets}"
wait_for_healthy_shard_primary "${keyspace}" "${shard}"
wait_for_writeable_shard_primary "${keyspace}" "${shard}"
wait_for_shard_vreplication_engine "${keyspace}" "${shard}"
}

Expand Down
15 changes: 10 additions & 5 deletions examples/local/101_initial_cluster.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,11 +40,16 @@ fi
# start vtctld
CELL=zone1 ../common/scripts/vtctld-up.sh

# Create the keyspace with the sidecar database name and set the
# correct durability policy. Please see the comment above for
# more context on using a custom sidecar database name in your
# Vitess clusters.
vtctldclient --server localhost:15999 CreateKeyspace --sidecar-db-name="${SIDECAR_DB_NAME}" --durability-policy=semi_sync commerce || fail "Failed to create and configure the commerce keyspace"
if vtctldclient GetKeyspace commerce > /dev/null 2>&1 ; then
# Keyspace already exists: we could be running this 101 example on an non-empty VTDATAROOT
vtctldclient --server localhost:15999 SetKeyspaceDurabilityPolicy --durability-policy=semi_sync commerce || fail "Failed to set keyspace durability policy on the commerce keyspace"
else
# Create the keyspace with the sidecar database name and set the
# correct durability policy. Please see the comment above for
# more context on using a custom sidecar database name in your
# Vitess clusters.
vtctldclient --server localhost:15999 CreateKeyspace --sidecar-db-name="${SIDECAR_DB_NAME}" --durability-policy=semi_sync commerce || fail "Failed to create and configure the commerce keyspace"
fi

# start vttablets for keyspace commerce
for i in 100 101 102; do
Expand Down
9 changes: 9 additions & 0 deletions test/local_example.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,10 @@
# It should be kept in sync with the steps in https://vitess.io/docs/get-started/local/
# So we can detect if a regression affecting a tutorial is introduced.

killall_vtdataroot() {
pkill -9 -e -f '(vtdataroot|VTDATAROOT)' # kill Vitess processes
}

source build.env

set -xeo pipefail
Expand All @@ -28,7 +32,12 @@ unset VTROOT # ensure that the examples can run without VTROOT now.
source ../common/env.sh # Required so that "mysql" works from alias

./101_initial_cluster.sh
sleep 5 # Give vtgate time to really start.

killall_vtdataroot
# verify local example is able to start on an existing setup

./101_initial_cluster.sh
sleep 5 # Give vtgate time to really start.

mysql < ../common/insert_commerce_data.sql
Expand Down