From aa0cb0a8c3e6d9e8c24331ff9df55e30078987d2 Mon Sep 17 00:00:00 2001 From: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> Date: Wed, 28 Jun 2023 15:16:02 +0300 Subject: [PATCH] Refactor backup_pitr into two distinct CI tests: builtin vs Xtrabackup each of those two is further split by mysql80 and mysql57. The reasonsing for this split is that for 'builtin' we want to test with mysql-server, whereas for 'Xtrabackup' we want to test with percona-server. It much easier to do so on two distinct CI workflows than on a single one. Signed-off-by: Shlomi Noach <2607934+shlomi-noach@users.noreply.github.com> --- .../cluster_endtoend_backup_pitr.yml | 17 +- .../cluster_endtoend_backup_pitr_mysql57.yml | 16 -- ...luster_endtoend_backup_pitr_xtrabackup.yml | 142 +++++++++++ ...ndtoend_backup_pitr_xtrabackup_mysql57.yml | 166 +++++++++++++ .../backup/pitr/backup_mysqlctld_pitr_test.go | 235 ------------------ .../endtoend/backup/pitr/backup_pitr_test.go | 33 +++ .../backup_pitr_xtrabackup_test.go | 35 +++ .../backup/vtctlbackup/pitr_test_framework.go | 221 ++++++++++++++++ test/ci_workflow_gen.go | 5 +- test/config.json | 9 + 10 files changed, 617 insertions(+), 262 deletions(-) create mode 100644 .github/workflows/cluster_endtoend_backup_pitr_xtrabackup.yml create mode 100644 .github/workflows/cluster_endtoend_backup_pitr_xtrabackup_mysql57.yml delete mode 100644 go/test/endtoend/backup/pitr/backup_mysqlctld_pitr_test.go create mode 100644 go/test/endtoend/backup/pitr/backup_pitr_test.go create mode 100644 go/test/endtoend/backup/pitr_xtrabackup/backup_pitr_xtrabackup_test.go create mode 100644 go/test/endtoend/backup/vtctlbackup/pitr_test_framework.go diff --git a/.github/workflows/cluster_endtoend_backup_pitr.yml b/.github/workflows/cluster_endtoend_backup_pitr.yml index 7e70dabe84f..727c4f7b7ec 100644 --- a/.github/workflows/cluster_endtoend_backup_pitr.yml +++ b/.github/workflows/cluster_endtoend_backup_pitr.yml @@ -84,16 +84,15 @@ jobs: if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' run: | - # Setup Percona Server for MySQL 8.0 + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + # Setup MySQL 8.0 + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.24-1_all.deb + echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* sudo apt-get update - sudo apt-get install -y lsb-release gnupg2 curl - wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb - sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb - sudo percona-release setup ps80 - sudo apt-get update - # Install everything else we need, and configure - sudo apt-get install -y percona-server-server percona-server-client make unzip g++ etcd git wget eatmydata xz-utils libncurses5 + sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata xz-utils libncurses5 sudo service mysql stop sudo service etcd stop @@ -104,8 +103,6 @@ jobs: # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - sudo apt-get install percona-xtrabackup-80 lz4 - - name: Setup launchable dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | diff --git a/.github/workflows/cluster_endtoend_backup_pitr_mysql57.yml b/.github/workflows/cluster_endtoend_backup_pitr_mysql57.yml index 8d93e57728d..e5d5f9f2190 100644 --- a/.github/workflows/cluster_endtoend_backup_pitr_mysql57.yml +++ b/.github/workflows/cluster_endtoend_backup_pitr_mysql57.yml @@ -13,10 +13,6 @@ env: LAUNCHABLE_WORKSPACE: "vitess-app" GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" - # This is used if we need to pin the xtrabackup version used in tests. - # If this is NOT set then the latest version available will be used. - #XTRABACKUP_VERSION: "2.4.24-1" - jobs: build: name: Run endtoend tests on Cluster (backup_pitr) mysql57 @@ -118,18 +114,6 @@ jobs: # install JUnit report formatter go install github.com/vitessio/go-junit-report@HEAD - wget "https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb" - sudo apt-get install -y gnupg2 - sudo dpkg -i "percona-release_latest.$(lsb_release -sc)_all.deb" - sudo apt-get update - if [[ -n $XTRABACKUP_VERSION ]]; then - debfile="percona-xtrabackup-24_$XTRABACKUP_VERSION.$(lsb_release -sc)_amd64.deb" - wget "https://repo.percona.com/pxb-24/apt/pool/main/p/percona-xtrabackup-24/$debfile" - sudo apt install -y "./$debfile" - else - sudo apt-get install -y percona-xtrabackup-24 - fi - - name: Setup launchable dependencies if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' run: | diff --git a/.github/workflows/cluster_endtoend_backup_pitr_xtrabackup.yml b/.github/workflows/cluster_endtoend_backup_pitr_xtrabackup.yml new file mode 100644 index 00000000000..e5a269d080d --- /dev/null +++ b/.github/workflows/cluster_endtoend_backup_pitr_xtrabackup.yml @@ -0,0 +1,142 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (backup_pitr_xtrabackup) +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (backup_pitr_xtrabackup)') + cancel-in-progress: true + +permissions: read-all + +env: + LAUNCHABLE_ORGANIZATION: "vitess" + LAUNCHABLE_WORKSPACE: "vitess-app" + GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" + +jobs: + build: + name: Run endtoend tests on Cluster (backup_pitr_xtrabackup) + runs-on: ubuntu-22.04 + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@v3 + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: frouioui/paths-filter@main + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/cluster_endtoend_backup_pitr_xtrabackup.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@v4 + with: + go-version: 1.20.5 + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@v4 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + # Limit local port range to not use ports that overlap with server side + # ports that we listen on. + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf + + - name: Get dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + + # Setup Percona Server for MySQL 8.0 + sudo apt-get update + sudo apt-get install -y lsb-release gnupg2 curl + wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb + sudo percona-release setup ps80 + sudo apt-get update + + # Install everything else we need, and configure + sudo apt-get install -y percona-server-server percona-server-client make unzip g++ etcd git wget eatmydata xz-utils libncurses5 + + sudo service mysql stop + sudo service etcd stop + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + go mod download + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + sudo apt-get install percona-xtrabackup-80 lz4 + + - name: Setup launchable dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + run: | + # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up + pip3 install --user launchable~=1.0 > /dev/null + + # verify that launchable setup is all correct. + launchable verify || true + + # Tell Launchable about the build you are producing and testing + launchable record build --name "$GITHUB_RUN_ID" --source . + + - name: Run cluster endtoend test + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 45 + run: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -x + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard backup_pitr_xtrabackup | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Print test output and Record test result in launchable + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + + # print test output + cat output.txt diff --git a/.github/workflows/cluster_endtoend_backup_pitr_xtrabackup_mysql57.yml b/.github/workflows/cluster_endtoend_backup_pitr_xtrabackup_mysql57.yml new file mode 100644 index 00000000000..ca8dd6a92b5 --- /dev/null +++ b/.github/workflows/cluster_endtoend_backup_pitr_xtrabackup_mysql57.yml @@ -0,0 +1,166 @@ +# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows" + +name: Cluster (backup_pitr_xtrabackup) mysql57 +on: [push, pull_request] +concurrency: + group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (backup_pitr_xtrabackup) mysql57') + cancel-in-progress: true + +permissions: read-all + +env: + LAUNCHABLE_ORGANIZATION: "vitess" + LAUNCHABLE_WORKSPACE: "vitess-app" + GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}" + + # This is used if we need to pin the xtrabackup version used in tests. + # If this is NOT set then the latest version available will be used. + #XTRABACKUP_VERSION: "2.4.24-1" + +jobs: + build: + name: Run endtoend tests on Cluster (backup_pitr_xtrabackup) mysql57 + runs-on: ubuntu-22.04 + + steps: + - name: Skip CI + run: | + if [[ "${{contains( github.event.pull_request.labels.*.name, 'Skip CI')}}" == "true" ]]; then + echo "skipping CI due to the 'Skip CI' label" + exit 1 + fi + + - name: Check if workflow needs to be skipped + id: skip-workflow + run: | + skip='false' + if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then + skip='true' + fi + echo Skip ${skip} + echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT + + - name: Check out code + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: actions/checkout@v3 + + - name: Check for changes in relevant files + if: steps.skip-workflow.outputs.skip-workflow == 'false' + uses: frouioui/paths-filter@main + id: changes + with: + token: '' + filters: | + end_to_end: + - 'go/**/*.go' + - 'test.go' + - 'Makefile' + - 'build.env' + - 'go.sum' + - 'go.mod' + - 'proto/*.proto' + - 'tools/**' + - 'config/**' + - 'bootstrap.sh' + - '.github/workflows/cluster_endtoend_backup_pitr_xtrabackup_mysql57.yml' + + - name: Set up Go + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-go@v4 + with: + go-version: 1.20.5 + + - name: Set up python + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + uses: actions/setup-python@v4 + + - name: Tune the OS + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo sysctl -w net.ipv4.ip_local_port_range="22768 65535" + # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio + echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf + sudo sysctl -p /etc/sysctl.conf + + - name: Get dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + run: | + sudo apt-get update + + # Uninstall any previously installed MySQL first + sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ + sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld + + sudo systemctl stop apparmor + sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common + sudo apt-get -y autoremove + sudo apt-get -y autoclean + sudo deluser mysql + sudo rm -rf /var/lib/mysql + sudo rm -rf /etc/mysql + + # Get key to latest MySQL repo + sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 + + wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.24-1_all.deb + # Bionic packages are still compatible for Jammy since there's no MySQL 5.7 + # packages for Jammy. + echo mysql-apt-config mysql-apt-config/repo-codename select bionic | sudo debconf-set-selections + echo mysql-apt-config mysql-apt-config/select-server select mysql-5.7 | sudo debconf-set-selections + sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* + sudo apt-get update + sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-client=5.7* mysql-community-server=5.7* mysql-server=5.7* libncurses5 + + sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata + sudo service mysql stop + sudo service etcd stop + + # install JUnit report formatter + go install github.com/vitessio/go-junit-report@HEAD + + wget "https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb" + sudo apt-get install -y gnupg2 + sudo dpkg -i "percona-release_latest.$(lsb_release -sc)_all.deb" + sudo apt-get update + if [[ -n $XTRABACKUP_VERSION ]]; then + debfile="percona-xtrabackup-24_$XTRABACKUP_VERSION.$(lsb_release -sc)_amd64.deb" + wget "https://repo.percona.com/pxb-24/apt/pool/main/p/percona-xtrabackup-24/$debfile" + sudo apt install -y "./$debfile" + else + sudo apt-get install -y percona-xtrabackup-24 + fi + + - name: Setup launchable dependencies + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && github.base_ref == 'main' + run: | + # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up + pip3 install --user launchable~=1.0 > /dev/null + + # verify that launchable setup is all correct. + launchable verify || true + + # Tell Launchable about the build you are producing and testing + launchable record build --name "$GITHUB_RUN_ID" --source . + + - name: Run cluster endtoend test + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' + timeout-minutes: 45 + run: | + # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file + # which musn't be more than 107 characters long. + export VTDATAROOT="/tmp/" + source build.env + + set -x + + # run the tests however you normally do, then produce a JUnit XML file + eatmydata -- go run test.go -docker=false -follow -shard backup_pitr_xtrabackup | tee -a output.txt | go-junit-report -set-exit-code > report.xml + + - name: Print test output and Record test result in launchable + if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' && always() + run: | + # send recorded tests to launchable + launchable record tests --build "$GITHUB_RUN_ID" go-test . || true + + # print test output + cat output.txt diff --git a/go/test/endtoend/backup/pitr/backup_mysqlctld_pitr_test.go b/go/test/endtoend/backup/pitr/backup_mysqlctld_pitr_test.go deleted file mode 100644 index 8cb8261606c..00000000000 --- a/go/test/endtoend/backup/pitr/backup_mysqlctld_pitr_test.go +++ /dev/null @@ -1,235 +0,0 @@ -/* -Copyright 2022 The Vitess Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package mysqlctld - -import ( - "context" - "fmt" - "math/rand" - "testing" - "time" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "vitess.io/vitess/go/mysql" - backup "vitess.io/vitess/go/test/endtoend/backup/vtctlbackup" - "vitess.io/vitess/go/test/endtoend/cluster" -) - -func waitForReplica(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - pMsgs := backup.ReadRowsFromPrimary(t) - for { - rMsgs := backup.ReadRowsFromReplica(t) - if len(pMsgs) == len(rMsgs) { - // success - return - } - select { - case <-ctx.Done(): - assert.FailNow(t, "timeout waiting for replica to catch up") - return - case <-time.After(time.Second): - // - } - } -} - -// TestIncrementalBackupMysqlctld - tests incremental backups using myslctld -func TestIncrementalBackup(t *testing.T) { - defer cluster.PanicHandler(t) - - tcases := []struct { - name string - setupType int - comprss *backup.CompressionDetails - }{ - { - "BuiltinBackup", backup.BuiltinBackup, nil, - }, - { - "XtraBackup", backup.XtraBackup, &backup.CompressionDetails{ - CompressorEngineName: "pgzip", - }, - }, - { - "Mysqlctld", backup.Mysqlctld, nil, - }, - } - for _, tcase := range tcases { - t.Run(tcase.name, func(t *testing.T) { - // setup cluster for the testing - code, err := backup.LaunchCluster(tcase.setupType, "xbstream", 0, tcase.comprss) - require.NoError(t, err, "setup failed with status code %d", code) - defer backup.TearDownCluster() - - backup.InitTestTable(t) - - rowsPerPosition := map[string]int{} - backupPositions := []string{} - - recordRowsPerPosition := func(t *testing.T) { - pos := backup.GetReplicaPosition(t) - msgs := backup.ReadRowsFromReplica(t) - if _, ok := rowsPerPosition[pos]; !ok { - backupPositions = append(backupPositions, pos) - rowsPerPosition[pos] = len(msgs) - } - } - - var fullBackupPos mysql.Position - t.Run("full backup", func(t *testing.T) { - backup.InsertRowOnPrimary(t, "before-full-backup") - waitForReplica(t) - - manifest, _ := backup.TestReplicaFullBackup(t) - fullBackupPos = manifest.Position - require.False(t, fullBackupPos.IsZero()) - // - msgs := backup.ReadRowsFromReplica(t) - pos := mysql.EncodePosition(fullBackupPos) - backupPositions = append(backupPositions, pos) - rowsPerPosition[pos] = len(msgs) - }) - - lastBackupPos := fullBackupPos - backup.InsertRowOnPrimary(t, "before-incremental-backups") - - tt := []struct { - name string - writeBeforeBackup bool - fromFullPosition bool - autoPosition bool - expectError string - }{ - { - name: "first incremental backup", - }, - { - name: "make writes, succeed", - writeBeforeBackup: true, - }, - { - name: "fail, no binary logs to backup", - expectError: "no binary logs to backup", - }, - { - name: "make writes again, succeed", - writeBeforeBackup: true, - }, - { - name: "auto position, succeed", - writeBeforeBackup: true, - autoPosition: true, - }, - { - name: "fail auto position, no binary logs to backup", - autoPosition: true, - expectError: "no binary logs to backup", - }, - { - name: "auto position, make writes again, succeed", - writeBeforeBackup: true, - autoPosition: true, - }, - { - name: "from full backup position", - fromFullPosition: true, - }, - } - var fromFullPositionBackups []string - for _, tc := range tt { - t.Run(tc.name, func(t *testing.T) { - if tc.writeBeforeBackup { - backup.InsertRowOnPrimary(t, "") - } - // we wait for 1 second because backups are written to a directory named after the current timestamp, - // in 1 second resolution. We want to avoid two backups that have the same pathname. Realistically this - // is only ever a problem in this end-to-end test, not in production. - // Also, we gie the replica a chance to catch up. - time.Sleep(1100 * time.Millisecond) - waitForReplica(t) - recordRowsPerPosition(t) - // configure --incremental-from-pos to either: - // - auto - // - explicit last backup pos - // - back in history to the original full backup - var incrementalFromPos mysql.Position - if !tc.autoPosition { - incrementalFromPos = lastBackupPos - if tc.fromFullPosition { - incrementalFromPos = fullBackupPos - } - } - manifest, backupName := backup.TestReplicaIncrementalBackup(t, incrementalFromPos, tc.expectError) - if tc.expectError != "" { - return - } - defer func() { - lastBackupPos = manifest.Position - }() - if tc.fromFullPosition { - fromFullPositionBackups = append(fromFullPositionBackups, backupName) - } - require.False(t, manifest.FromPosition.IsZero()) - require.NotEqual(t, manifest.Position, manifest.FromPosition) - require.True(t, manifest.Position.GTIDSet.Union(manifest.PurgedPosition.GTIDSet).Contains(manifest.FromPosition.GTIDSet)) - - gtidPurgedPos, err := mysql.ParsePosition(mysql.Mysql56FlavorID, backup.GetReplicaGtidPurged(t)) - require.NoError(t, err) - fromPositionIncludingPurged := manifest.FromPosition.GTIDSet.Union(gtidPurgedPos.GTIDSet) - - expectFromPosition := lastBackupPos.GTIDSet.Union(gtidPurgedPos.GTIDSet) - if !incrementalFromPos.IsZero() { - expectFromPosition = incrementalFromPos.GTIDSet.Union(gtidPurgedPos.GTIDSet) - } - require.Equalf(t, expectFromPosition, fromPositionIncludingPurged, "expected: %v, found: %v, gtid_purged: %v, manifest.Position: %v", expectFromPosition, fromPositionIncludingPurged, gtidPurgedPos, manifest.Position) - }) - } - - testRestores := func(t *testing.T) { - for _, r := range rand.Perm(len(backupPositions)) { - pos := backupPositions[r] - testName := fmt.Sprintf("%s, %d records", pos, rowsPerPosition[pos]) - t.Run(testName, func(t *testing.T) { - restoreToPos, err := mysql.DecodePosition(pos) - require.NoError(t, err) - backup.TestReplicaRestoreToPos(t, restoreToPos, "") - msgs := backup.ReadRowsFromReplica(t) - count, ok := rowsPerPosition[pos] - require.True(t, ok) - assert.Equalf(t, count, len(msgs), "messages: %v", msgs) - }) - } - } - t.Run("PITR", func(t *testing.T) { - testRestores(t) - }) - t.Run("remove full position backups", func(t *testing.T) { - // Delete the fromFullPosition backup(s), which leaves us with less restore options. Try again. - for _, backupName := range fromFullPositionBackups { - backup.RemoveBackup(t, backupName) - } - }) - t.Run("PITR-2", func(t *testing.T) { - testRestores(t) - }) - }) - } -} diff --git a/go/test/endtoend/backup/pitr/backup_pitr_test.go b/go/test/endtoend/backup/pitr/backup_pitr_test.go new file mode 100644 index 00000000000..d7f76b012e5 --- /dev/null +++ b/go/test/endtoend/backup/pitr/backup_pitr_test.go @@ -0,0 +1,33 @@ +/* +Copyright 2022 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mysqlctld + +import ( + "testing" + + backup "vitess.io/vitess/go/test/endtoend/backup/vtctlbackup" +) + +// TestIncrementalBackupAndRestoreToPos +func TestIncrementalBackupAndRestoreToPos(t *testing.T) { + tcase := &backup.PITRTestCase{ + Name: "BuiltinBackup", + SetupType: backup.BuiltinBackup, + ComprssDetails: nil, + } + backup.ExecTestIncrementalBackupAndRestoreToPos(t, tcase) +} diff --git a/go/test/endtoend/backup/pitr_xtrabackup/backup_pitr_xtrabackup_test.go b/go/test/endtoend/backup/pitr_xtrabackup/backup_pitr_xtrabackup_test.go new file mode 100644 index 00000000000..21f43f01a83 --- /dev/null +++ b/go/test/endtoend/backup/pitr_xtrabackup/backup_pitr_xtrabackup_test.go @@ -0,0 +1,35 @@ +/* +Copyright 2022 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mysqlctld + +import ( + "testing" + + backup "vitess.io/vitess/go/test/endtoend/backup/vtctlbackup" +) + +// TestIncrementalBackupAndRestoreToPos +func TestIncrementalBackupAndRestoreToPos(t *testing.T) { + tcase := &backup.PITRTestCase{ + Name: "XtraBackup", + SetupType: backup.XtraBackup, + ComprssDetails: &backup.CompressionDetails{ + CompressorEngineName: "pgzip", + }, + } + backup.ExecTestIncrementalBackupAndRestoreToPos(t, tcase) +} diff --git a/go/test/endtoend/backup/vtctlbackup/pitr_test_framework.go b/go/test/endtoend/backup/vtctlbackup/pitr_test_framework.go new file mode 100644 index 00000000000..a5bc6d15b0f --- /dev/null +++ b/go/test/endtoend/backup/vtctlbackup/pitr_test_framework.go @@ -0,0 +1,221 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vtctlbackup + +import ( + "context" + "fmt" + "math/rand" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/test/endtoend/cluster" +) + +type PITRTestCase struct { + Name string + SetupType int + ComprssDetails *CompressionDetails +} + +func waitForReplica(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + pMsgs := ReadRowsFromPrimary(t) + for { + rMsgs := ReadRowsFromReplica(t) + if len(pMsgs) == len(rMsgs) { + // success + return + } + select { + case <-ctx.Done(): + assert.FailNow(t, "timeout waiting for replica to catch up") + return + case <-time.After(time.Second): + // + } + } +} + +// ExecTestIncrementalBackupAndRestoreToPos +func ExecTestIncrementalBackupAndRestoreToPos(t *testing.T, tcase *PITRTestCase) { + defer cluster.PanicHandler(t) + + t.Run(tcase.Name, func(t *testing.T) { + // setup cluster for the testing + code, err := LaunchCluster(tcase.SetupType, "xbstream", 0, tcase.ComprssDetails) + require.NoError(t, err, "setup failed with status code %d", code) + defer TearDownCluster() + + InitTestTable(t) + + rowsPerPosition := map[string]int{} + backupPositions := []string{} + + recordRowsPerPosition := func(t *testing.T) { + pos := GetReplicaPosition(t) + msgs := ReadRowsFromReplica(t) + if _, ok := rowsPerPosition[pos]; !ok { + backupPositions = append(backupPositions, pos) + rowsPerPosition[pos] = len(msgs) + } + } + + var fullBackupPos mysql.Position + t.Run("full backup", func(t *testing.T) { + InsertRowOnPrimary(t, "before-full-backup") + waitForReplica(t) + + manifest, _ := TestReplicaFullBackup(t) + fullBackupPos = manifest.Position + require.False(t, fullBackupPos.IsZero()) + // + msgs := ReadRowsFromReplica(t) + pos := mysql.EncodePosition(fullBackupPos) + backupPositions = append(backupPositions, pos) + rowsPerPosition[pos] = len(msgs) + }) + + lastBackupPos := fullBackupPos + InsertRowOnPrimary(t, "before-incremental-backups") + + tt := []struct { + name string + writeBeforeBackup bool + fromFullPosition bool + autoPosition bool + expectError string + }{ + { + name: "first incremental backup", + }, + { + name: "make writes, succeed", + writeBeforeBackup: true, + }, + { + name: "fail, no binary logs to backup", + expectError: "no binary logs to backup", + }, + { + name: "make writes again, succeed", + writeBeforeBackup: true, + }, + { + name: "auto position, succeed", + writeBeforeBackup: true, + autoPosition: true, + }, + { + name: "fail auto position, no binary logs to backup", + autoPosition: true, + expectError: "no binary logs to backup", + }, + { + name: "auto position, make writes again, succeed", + writeBeforeBackup: true, + autoPosition: true, + }, + { + name: "from full backup position", + fromFullPosition: true, + }, + } + var fromFullPositionBackups []string + for _, tc := range tt { + t.Run(tc.name, func(t *testing.T) { + if tc.writeBeforeBackup { + InsertRowOnPrimary(t, "") + } + // we wait for 1 second because backups are written to a directory named after the current timestamp, + // in 1 second resolution. We want to avoid two backups that have the same pathname. Realistically this + // is only ever a problem in this end-to-end test, not in production. + // Also, we gie the replica a chance to catch up. + time.Sleep(1100 * time.Millisecond) + waitForReplica(t) + recordRowsPerPosition(t) + // configure --incremental-from-pos to either: + // - auto + // - explicit last backup pos + // - back in history to the original full backup + var incrementalFromPos mysql.Position + if !tc.autoPosition { + incrementalFromPos = lastBackupPos + if tc.fromFullPosition { + incrementalFromPos = fullBackupPos + } + } + manifest, backupName := TestReplicaIncrementalBackup(t, incrementalFromPos, tc.expectError) + if tc.expectError != "" { + return + } + defer func() { + lastBackupPos = manifest.Position + }() + if tc.fromFullPosition { + fromFullPositionBackups = append(fromFullPositionBackups, backupName) + } + require.False(t, manifest.FromPosition.IsZero()) + require.NotEqual(t, manifest.Position, manifest.FromPosition) + require.True(t, manifest.Position.GTIDSet.Union(manifest.PurgedPosition.GTIDSet).Contains(manifest.FromPosition.GTIDSet)) + + gtidPurgedPos, err := mysql.ParsePosition(mysql.Mysql56FlavorID, GetReplicaGtidPurged(t)) + require.NoError(t, err) + fromPositionIncludingPurged := manifest.FromPosition.GTIDSet.Union(gtidPurgedPos.GTIDSet) + + expectFromPosition := lastBackupPos.GTIDSet.Union(gtidPurgedPos.GTIDSet) + if !incrementalFromPos.IsZero() { + expectFromPosition = incrementalFromPos.GTIDSet.Union(gtidPurgedPos.GTIDSet) + } + require.Equalf(t, expectFromPosition, fromPositionIncludingPurged, "expected: %v, found: %v, gtid_purged: %v, manifest.Position: %v", expectFromPosition, fromPositionIncludingPurged, gtidPurgedPos, manifest.Position) + }) + } + + testRestores := func(t *testing.T) { + for _, r := range rand.Perm(len(backupPositions)) { + pos := backupPositions[r] + testName := fmt.Sprintf("%s, %d records", pos, rowsPerPosition[pos]) + t.Run(testName, func(t *testing.T) { + restoreToPos, err := mysql.DecodePosition(pos) + require.NoError(t, err) + TestReplicaRestoreToPos(t, restoreToPos, "") + msgs := ReadRowsFromReplica(t) + count, ok := rowsPerPosition[pos] + require.True(t, ok) + assert.Equalf(t, count, len(msgs), "messages: %v", msgs) + }) + } + } + t.Run("PITR", func(t *testing.T) { + testRestores(t) + }) + t.Run("remove full position backups", func(t *testing.T) { + // Delete the fromFullPosition backup(s), which leaves us with less restore options. Try again. + for _, backupName := range fromFullPositionBackups { + RemoveBackup(t, backupName) + } + }) + t.Run("PITR-2", func(t *testing.T) { + testRestores(t) + }) + }) +} diff --git a/test/ci_workflow_gen.go b/test/ci_workflow_gen.go index 711beb23717..6484d41b777 100644 --- a/test/ci_workflow_gen.go +++ b/test/ci_workflow_gen.go @@ -76,6 +76,7 @@ var ( "18", "xb_backup", "backup_pitr", + "backup_pitr_xtrabackup", "21", "22", "mysql_server_vault", @@ -131,7 +132,7 @@ var ( clustersRequiringXtraBackup = []string{ "xb_backup", "xb_recovery", - "backup_pitr", + "backup_pitr_xtrabackup", } clustersRequiringMakeTools = []string{ "18", @@ -170,6 +171,8 @@ func clusterMySQLVersions(clusterName string) mysqlVersions { return allMySQLVersions case clusterName == "backup_pitr": return allMySQLVersions + case clusterName == "backup_pitr_xtrabackup": + return allMySQLVersions case clusterName == "tabletmanager_tablegc": return allMySQLVersions case clusterName == "vtorc": diff --git a/test/config.json b/test/config.json index 0070720f1a0..4c1249c8e21 100644 --- a/test/config.json +++ b/test/config.json @@ -100,6 +100,15 @@ "RetryMax": 1, "Tags": [] }, + "backup_pitr_xtrabackup": { + "File": "unused.go", + "Args": ["vitess.io/vitess/go/test/endtoend/backup/pitr_xtrabackup", "-timeout", "30m"], + "Command": [], + "Manual": false, + "Shard": "backup_pitr_xtrabackup", + "RetryMax": 1, + "Tags": [] + }, "backup": { "File": "unused.go", "Args": ["vitess.io/vitess/go/test/endtoend/backup/vtctlbackup", "-timeout", "30m"],