CI: Misc test improvements to limit failures with various runners (#1…

…3825)

Signed-off-by: Matt Lord <mattalord@gmail.com>

.github/workflows/cluster_endtoend_backup_pitr_xtrabackup.yml

@@ -104,7 +104,7 @@ jobs:
         # install JUnit report formatter
         go install github.com/vitessio/go-junit-report@HEAD
 
-        sudo apt-get install percona-xtrabackup-80 lz4
+        sudo apt-get install -y percona-xtrabackup-80 lz4
 
     - name: Setup launchable dependencies
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main'

.github/workflows/cluster_endtoend_xb_backup.yml

@@ -104,7 +104,7 @@ jobs:
         # install JUnit report formatter
         go install github.com/vitessio/go-junit-report@HEAD
 
-        sudo apt-get install percona-xtrabackup-80 lz4
+        sudo apt-get install -y percona-xtrabackup-80 lz4
 
     - name: Setup launchable dependencies
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main'

.github/workflows/cluster_endtoend_xb_recovery.yml

@@ -104,7 +104,7 @@ jobs:
         # install JUnit report formatter
         go install github.com/vitessio/go-junit-report@HEAD
 
-        sudo apt-get install percona-xtrabackup-80 lz4
+        sudo apt-get install -y percona-xtrabackup-80 lz4
 
     - name: Setup launchable dependencies
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main'

.github/workflows/codeql_analysis.yml

@@ -79,7 +79,7 @@ jobs:
           sudo apt-get install -y gnupg2
           sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
           sudo apt-get update
-          sudo apt-get install percona-xtrabackup-24
+          sudo apt-get install -y percona-xtrabackup-24
 
       - name: Building binaries
         timeout-minutes: 30

.github/workflows/endtoend.yml

@@ -84,4 +84,8 @@ jobs:
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
       timeout-minutes: 30
       run: |
+        # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file
+        # which musn't be more than 107 characters long.
+        export VTDATAROOT="/tmp/"
+
         eatmydata -- tools/e2e_test_runner.sh

.github/workflows/unit_race.yml

@@ -95,5 +95,9 @@ jobs:
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true'
       timeout-minutes: 45
       run: |
+        # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file
+        # which musn't be more than 107 characters long.
+        export VTDATAROOT="/tmp/"
         export NOVTADMINBUILD=1
+
         eatmydata -- make unit_test_race

.github/workflows/unit_test_mysql57.yml

@@ -144,6 +144,9 @@ jobs:
       timeout-minutes: 30
       run: |
         set -exo pipefail
+        # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file
+        # which musn't be more than 107 characters long.
+        export VTDATAROOT="/tmp/"
 
         export NOVTADMINBUILD=1
         eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml

.github/workflows/unit_test_mysql80.yml

@@ -141,6 +141,9 @@ jobs:
       timeout-minutes: 30
       run: |
         set -exo pipefail
+        # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file
+        # which musn't be more than 107 characters long.
+        export VTDATAROOT="/tmp/"
 
         export NOVTADMINBUILD=1
         eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml

.github/workflows/upgrade_downgrade_test_backups_e2e.yml

@@ -114,7 +114,7 @@ jobs:
         sudo apt-get install -y gnupg2
         sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
         sudo apt-get update
-        sudo apt-get install percona-xtrabackup-24
+        sudo apt-get install -y percona-xtrabackup-24
 
     # Checkout to the last release of Vitess
     - name: Check out other version's code (${{ needs.get_previous_release.outputs.previous_release }})

.github/workflows/upgrade_downgrade_test_backups_e2e_next_release.yml

@@ -117,7 +117,7 @@ jobs:
         sudo apt-get install -y gnupg2
         sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
         sudo apt-get update
-        sudo apt-get install percona-xtrabackup-24
+        sudo apt-get install -y percona-xtrabackup-24
 
     # Checkout to the next release of Vitess
     - name: Check out other version's code (${{ needs.get_next_release.outputs.next_release }})

.github/workflows/upgrade_downgrade_test_backups_manual.yml

@@ -134,7 +134,7 @@ jobs:
         sudo apt-get install -y gnupg2
         sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
         sudo apt-get update
-        sudo apt-get install percona-xtrabackup-24
+        sudo apt-get install -y percona-xtrabackup-24
 
     # Checkout to the last release of Vitess
     - name: Checkout to the other version's code (${{ needs.get_previous_release.outputs.previous_release }})

.github/workflows/upgrade_downgrade_test_backups_manual_next_release.yml

@@ -137,7 +137,7 @@ jobs:
         sudo apt-get install -y gnupg2
         sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
         sudo apt-get update
-        sudo apt-get install percona-xtrabackup-24
+        sudo apt-get install -y percona-xtrabackup-24
 
     # Checkout to the next release of Vitess
     - name: Checkout to the other version's code (${{ needs.get_next_release.outputs.next_release }})

.github/workflows/upgrade_downgrade_test_query_serving_queries.yml

@@ -132,7 +132,7 @@ jobs:
         sudo apt-get install -y gnupg2
         sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
         sudo apt-get update
-        sudo apt-get install percona-xtrabackup-24
+        sudo apt-get install -y percona-xtrabackup-24
 
     # Checkout to the last release of Vitess
     - name: Check out other version's code (${{ needs.get_previous_release.outputs.previous_release }})

.github/workflows/upgrade_downgrade_test_query_serving_queries_next_release.yml

@@ -135,7 +135,7 @@ jobs:
         sudo apt-get install -y gnupg2
         sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
         sudo apt-get update
-        sudo apt-get install percona-xtrabackup-24
+        sudo apt-get install -y percona-xtrabackup-24
 
     # Checkout to the next release of Vitess
     - name: Check out other version's code (${{ needs.get_next_release.outputs.next_release }})

.github/workflows/upgrade_downgrade_test_query_serving_schema.yml

@@ -132,7 +132,7 @@ jobs:
         sudo apt-get install -y gnupg2
         sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
         sudo apt-get update
-        sudo apt-get install percona-xtrabackup-24
+        sudo apt-get install -y percona-xtrabackup-24
 
     # Checkout to the last release of Vitess
     - name: Check out other version's code (${{ needs.get_previous_release.outputs.previous_release }})

.github/workflows/upgrade_downgrade_test_query_serving_schema_next_release.yml

@@ -135,7 +135,7 @@ jobs:
         sudo apt-get install -y gnupg2
         sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
         sudo apt-get update
-        sudo apt-get install percona-xtrabackup-24
+        sudo apt-get install -y percona-xtrabackup-24
 
     # Checkout to the next release of Vitess
     - name: Check out other version's code (${{ needs.get_next_release.outputs.next_release }})

.github/workflows/upgrade_downgrade_test_reparent_new_vtctl.yml

@@ -135,7 +135,7 @@ jobs:
         sudo apt-get install -y gnupg2
         sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
         sudo apt-get update
-        sudo apt-get install percona-xtrabackup-24
+        sudo apt-get install -y percona-xtrabackup-24
 
     # Checkout to the next release of Vitess
     - name: Check out other version's code (${{ needs.get_next_release.outputs.next_release }})

.github/workflows/upgrade_downgrade_test_reparent_new_vttablet.yml

@@ -135,7 +135,7 @@ jobs:
         sudo apt-get install -y gnupg2
         sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
         sudo apt-get update
-        sudo apt-get install percona-xtrabackup-24
+        sudo apt-get install -y percona-xtrabackup-24
 
     # Checkout to the next release of Vitess
     - name: Check out other version's code (${{ needs.get_next_release.outputs.next_release }})

.github/workflows/upgrade_downgrade_test_reparent_old_vtctl.yml

@@ -132,7 +132,7 @@ jobs:
         sudo apt-get install -y gnupg2
         sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
         sudo apt-get update
-        sudo apt-get install percona-xtrabackup-24
+        sudo apt-get install -y percona-xtrabackup-24
 
     # Checkout to the last release of Vitess
     - name: Check out other version's code (${{ needs.get_previous_release.outputs.previous_release }})

.github/workflows/upgrade_downgrade_test_reparent_old_vttablet.yml

@@ -132,7 +132,7 @@ jobs:
         sudo apt-get install -y gnupg2
         sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
         sudo apt-get update
-        sudo apt-get install percona-xtrabackup-24
+        sudo apt-get install -y percona-xtrabackup-24
 
     # Checkout to the last release of Vitess
     - name: Check out other version's code (${{ needs.get_previous_release.outputs.previous_release }})

go/cmd/vttestserver/vttestserver_test.go

@@ -256,7 +256,10 @@ func TestMtlsAuth(t *testing.T) {
 		fmt.Sprintf("--vtctld_grpc_ca=%s", caCert),
 		fmt.Sprintf("--grpc_auth_mtls_allowed_substrings=%s", "CN=ClientApp"))
 	assert.NoError(t, err)
-	defer cluster.TearDown()
+	defer func() {
+		cluster.PersistentMode = false // Cleanup the tmpdir as we're done
+		cluster.TearDown()
+	}()
 
 	// startCluster will apply vschema migrations using vtctl grpc and the clientCert.
 	assertColumnVindex(t, cluster, columnVindex{keyspace: "test_keyspace", table: "test_table", vindex: "my_vdx", vindexType: "hash", column: "id"})

go/test/endtoend/utils/utils.go

@@ -252,7 +252,7 @@ func WaitForAuthoritative(t *testing.T, ks, tbl string, readVSchema func() (*int
 
 // WaitForColumn waits for a table's column to be present
 func WaitForColumn(t *testing.T, vtgateProcess cluster.VtgateProcess, ks, tbl, col string) error {
-	timeout := time.After(10 * time.Second)
+	timeout := time.After(60 * time.Second)
 	for {
 		select {
 		case <-timeout:

go/vt/grpcclient/client_flaky_test.go

@@ -43,7 +43,7 @@ func TestDialErrors(t *testing.T) {
 			t.Fatal(err)
 		}
 		vtg := vtgateservicepb.NewVitessClient(gconn)
-		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Millisecond)
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
 		_, err = vtg.Execute(ctx, &vtgatepb.ExecuteRequest{})
 		cancel()
 		gconn.Close()

go/vt/vttest/toxiproxyctl.go

@@ -63,13 +63,12 @@ func NewToxiproxyctl(binary string, apiPort, mysqlPort int, mysqlctl *Mysqlctl,
 
 	// The original initFile does not have any users who can access through TCP/IP connection.
 	// Here we update the init file to create the user.
-	// We're using IPv6 localhost because that's what toxiproxy uses by default.
 	initDb, _ := os.ReadFile(mysqlctl.InitFile)
 	createUserCmd := fmt.Sprintf(`
 		# Admin user for TCP/IP connection with all privileges.
-		CREATE USER '%s'@'::1';
-		GRANT ALL ON *.* TO '%s'@'::1';
-		GRANT GRANT OPTION ON *.* TO '%s'@'::1';
+		CREATE USER '%s'@'127.0.0.1';
+		GRANT ALL ON *.* TO '%s'@'127.0.0.1';
+		GRANT GRANT OPTION ON *.* TO '%s'@'127.0.0.1';
 	`, dbaUser, dbaUser, dbaUser)
 	sql, err := getInitDBSQL(string(initDb), createUserCmd)
 	if err != nil {
@@ -143,11 +142,11 @@ func (ctl *Toxiproxyctl) run() error {
 		// Wait for toxiproxy to start
 		time.Sleep(1 * time.Second)
 
-		toxiClient := toxiproxy.NewClient("localhost:" + fmt.Sprintf("%d", ctl.apiPort))
+		toxiClient := toxiproxy.NewClient("127.0.0.1:" + fmt.Sprintf("%d", ctl.apiPort))
 		proxy, err := toxiClient.CreateProxy(
 			"mysql",
-			"localhost:"+fmt.Sprintf("%d", ctl.port),
-			"localhost:"+fmt.Sprintf("%d", ctl.mysqlctl.Port),
+			"127.0.0.1:"+fmt.Sprintf("%d", ctl.port),
+			"127.0.0.1:"+fmt.Sprintf("%d", ctl.mysqlctl.Port),
 		)
 		if err == nil {
 			ctl.proxy = proxy
@@ -206,7 +205,7 @@ func (ctl *Toxiproxyctl) Params(dbname string) mysql.ConnParams {
 	params := ctl.mysqlctl.Params(dbname)
 
 	params.UnixSocket = ""
-	params.Host = "localhost"
+	params.Host = "127.0.0.1"
 	params.Port = ctl.port
 	params.Uname = dbaUser
 	return params

test/templates/cluster_endtoend_test.tpl

@@ -122,7 +122,7 @@ jobs:
 
         {{if .InstallXtraBackup}}
 
-        sudo apt-get install percona-xtrabackup-80 lz4
+        sudo apt-get install -y percona-xtrabackup-80 lz4
 
         {{end}}
 

test/templates/cluster_endtoend_test_docker.tpl

@@ -65,4 +65,8 @@ jobs:
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
       timeout-minutes: 30
       run: |
+        # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file
+        # which musn't be more than 107 characters long.
+        export VTDATAROOT="/tmp/"
+
         go run test.go -docker=true --follow -shard {{.Shard}}

test/templates/cluster_endtoend_test_self_hosted.tpl

@@ -61,7 +61,12 @@ jobs:
       - name: Run test
         if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'
         timeout-minutes: 30
-        run: docker run --name "{{.ImageName}}_$GITHUB_SHA" {{.ImageName}}:$GITHUB_SHA /bin/bash -c 'source build.env && go run test.go -keep-data=true -docker=false -print-log -follow -shard {{.Shard}} -- -- --keep-data=true'
+        run: |
+          # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file
+          # which musn't be more than 107 characters long.
+          export VTDATAROOT="/tmp/"
+
+          docker run --name "{{.ImageName}}_$GITHUB_SHA" {{.ImageName}}:$GITHUB_SHA /bin/bash -c 'source build.env && go run test.go -keep-data=true -docker=false -print-log -follow -shard {{.Shard}} -- -- --keep-data=true'
 
       - name: Print Volume Used
         if: always() && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true'

test/templates/unit_test.tpl

@@ -157,7 +157,13 @@ jobs:
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true'
       timeout-minutes: 30
       run: |
-        eatmydata -- NOVTADMINBUILD=1 make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml
+        set -exo pipefail
+        # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file
+        # which musn't be more than 107 characters long.
+        export VTDATAROOT="/tmp/"
+
+        export NOVTADMINBUILD=1
+        eatmydata -- make unit_test | tee -a output.txt | go-junit-report -set-exit-code > report.xml
 
     - name: Print test output and Record test result in launchable
       if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true' && always()

test/templates/unit_test_self_hosted.tpl

@@ -59,7 +59,13 @@ jobs:
       - name: Run test
         if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true'
         timeout-minutes: 30
-        run: docker run --name "{{.ImageName}}_$GITHUB_SHA" {{.ImageName}}:$GITHUB_SHA /bin/bash -c 'NOVTADMINBUILD=1 make unit_test'
+        run: |
+          set -exo pipefail
+          # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file
+          # which musn't be more than 107 characters long.
+          export VTDATAROOT="/tmp/"
+
+          docker run --name "{{.ImageName}}_$GITHUB_SHA" {{.ImageName}}:$GITHUB_SHA /bin/bash -c 'NOVTADMINBUILD=1 make unit_test'
 
       - name: Print Volume Used
         if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.unit_tests == 'true'