Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Flakes: Improve reliability of vreplication_copy_parallel test #13857

Merged
merged 3 commits into from
Aug 26, 2023
Merged

Flakes: Improve reliability of vreplication_copy_parallel test #13857

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4edb612
Wait for vrepl to catch up again after vdiff
mattlord Aug 25, 2023
0e048b2
Ensure that we can switch before trying
mattlord Aug 25, 2023
8329b11
kick the CI
mattlord Aug 26, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions go/test/endtoend/vreplication/vreplication_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1401,6 +1401,7 @@ func switchReadsDryRun(t *testing.T, workflowType, cells, ksWorkflow string, dry
require.FailNowf(t, "Invalid workflow type for SwitchTraffic, must be MoveTables or Reshard",
"workflow type specified: %s", workflowType)
}
ensureCanSwitch(t, workflowType, cells, ksWorkflow)
output, err := vc.VtctlClient.ExecuteCommandWithOutput(workflowType, "--", "--cells="+cells, "--tablet_types=rdonly,replica",
"--dry_run", "SwitchTraffic", ksWorkflow)
require.NoError(t, err, fmt.Sprintf("Switching Reads DryRun Error: %s: %s", err, output))
Expand All @@ -1409,6 +1410,23 @@ func switchReadsDryRun(t *testing.T, workflowType, cells, ksWorkflow string, dry
}
}

func ensureCanSwitch(t *testing.T, workflowType, cells, ksWorkflow string) {
timer := time.NewTimer(defaultTimeout)
defer timer.Stop()
for {
_, err := vc.VtctlClient.ExecuteCommandWithOutput(workflowType, "--", "--cells="+cells, "--dry_run", "SwitchTraffic", ksWorkflow)
if err == nil {
return
}
select {
case <-timer.C:
t.Fatalf("Did not become ready to switch traffic for %s before the timeout of %s", ksWorkflow, defaultTimeout)
default:
time.Sleep(defaultTick)
}
}
}

func switchReads(t *testing.T, workflowType, cells, ksWorkflow string, reverse bool) {
if workflowType != binlogdatapb.VReplicationWorkflowType_MoveTables.String() &&
workflowType != binlogdatapb.VReplicationWorkflowType_Reshard.String() {
Expand All @@ -1421,6 +1439,7 @@ func switchReads(t *testing.T, workflowType, cells, ksWorkflow string, reverse b
if reverse {
command = "ReverseTraffic"
}
ensureCanSwitch(t, workflowType, cells, ksWorkflow)
output, err = vc.VtctlClient.ExecuteCommandWithOutput(workflowType, "--", "--cells="+cells, "--tablet_types=rdonly",
command, ksWorkflow)
require.NoError(t, err, fmt.Sprintf("%s Error: %s: %s", command, err, output))
Expand All @@ -1440,6 +1459,7 @@ func switchWrites(t *testing.T, workflowType, ksWorkflow string, reverse bool) {
command = "ReverseTraffic"
}
const SwitchWritesTimeout = "91s" // max: 3 tablet picker 30s waits + 1
ensureCanSwitch(t, workflowType, "", ksWorkflow)
// Use vtctldclient for MoveTables SwitchTraffic ~ 50% of the time.
if workflowType == binlogdatapb.VReplicationWorkflowType_MoveTables.String() && time.Now().Second()%2 == 0 {
parts := strings.Split(ksWorkflow, ".")
Expand Down Expand Up @@ -1535,6 +1555,7 @@ func generateInnoDBRowHistory(t *testing.T, sourceKS string, neededTrxHistory in
// expected length.
func waitForInnoDBHistoryLength(t *testing.T, tablet *cluster.VttabletProcess, expectedLength int64) {
timer := time.NewTimer(defaultTimeout)
defer timer.Stop()
historyLen := int64(0)
for {
res, err := tablet.QueryTablet(historyLenQuery, tablet.Keyspace, false)
Expand Down