[DO NOT MERGE] verify ci testing #2154
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Upgrade Downgrade Testing - Backups - Manual | |
| on: | |
| push: | |
| pull_request: | |
| concurrency: | |
| group: format('{0}-{1}', ${{ github.ref }}, 'Upgrade Downgrade Testing - Backups - Manual') | |
| cancel-in-progress: true | |
| jobs: | |
| get_previous_release: | |
| if: always() | |
| name: Get Previous Release - Backups - Manual | |
| runs-on: ubuntu-20.04 | |
| outputs: | |
| previous_release: ${{ steps.output-previous-release-ref.outputs.previous_release_ref }} | |
| steps: | |
| - name: Check out to HEAD | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Set output with latest release branch | |
| id: output-previous-release-ref | |
| run: | | |
| previous_release_ref=$(./tools/get_previous_release.sh ${{github.base_ref}} ${{github.ref}}) | |
| echo $previous_release_ref | |
| echo "previous_release_ref=${previous_release_ref}" >> $GITHUB_OUTPUT | |
| # This job usually execute in ± 20 minutes | |
| upgrade_downgrade_test_manual: | |
| timeout-minutes: 40 | |
| if: always() && (needs.get_previous_release.result == 'success') | |
| name: Run Upgrade Downgrade Test - Backups - Manual | |
| runs-on: ubuntu-20.04 | |
| needs: | |
| - get_previous_release | |
| steps: | |
| - name: Check if workflow needs to be skipped | |
| id: skip-workflow | |
| run: | | |
| skip='false' | |
| if [[ "${{github.event.pull_request}}" == "" ]] && [[ "${{github.ref}}" != "refs/heads/main" ]] && [[ ! "${{github.ref}}" =~ ^refs/heads/release-[0-9]+\.[0-9]$ ]] && [[ ! "${{github.ref}}" =~ "refs/tags/.*" ]]; then | |
| skip='true' | |
| fi | |
| echo Skip ${skip} | |
| echo "skip-workflow=${skip}" >> $GITHUB_OUTPUT | |
| # Checkout to this build's commit | |
| - name: Checkout to commit's code | |
| if: steps.skip-workflow.outputs.skip-workflow == 'false' | |
| uses: actions/checkout@v3 | |
| - name: Check for changes in relevant files | |
| if: steps.skip-workflow.outputs.skip-workflow == 'false' | |
| uses: frouioui/paths-filter@main | |
| id: changes | |
| with: | |
| token: '' | |
| filters: | | |
| end_to_end: | |
| - 'go/**' | |
| - 'go/**/*.go' | |
| - 'test.go' | |
| - 'Makefile' | |
| - 'build.env' | |
| - 'go.sum' | |
| - 'go.mod' | |
| - 'proto/*.proto' | |
| - 'tools/**' | |
| - 'config/**' | |
| - 'bootstrap.sh' | |
| - '.github/workflows/upgrade_downgrade_test_backups_manual.yml' | |
| - name: Set up Go | |
| if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' | |
| uses: actions/setup-go@v3 | |
| with: | |
| go-version: 1.18.9 | |
| - name: Set up python | |
| if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' | |
| uses: actions/setup-python@v4 | |
| - name: Tune the OS | |
| if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' | |
| run: | | |
| echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range | |
| - name: Get base dependencies | |
| if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' | |
| run: | | |
| sudo DEBIAN_FRONTEND="noninteractive" apt-get update | |
| # Uninstall any previously installed MySQL first | |
| sudo systemctl stop apparmor | |
| sudo DEBIAN_FRONTEND="noninteractive" apt-get remove -y --purge mysql-server mysql-client mysql-common | |
| sudo apt-get -y autoremove | |
| sudo apt-get -y autoclean | |
| sudo deluser mysql | |
| sudo rm -rf /var/lib/mysql | |
| sudo rm -rf /etc/mysql | |
| # Install MySQL 8.0 | |
| #### | |
| ## Temporarily pin the MySQL version at 8.0.29 as Vitess 14.0.1 does not have the fix to support | |
| ## backups of 8.0.30+. See: https://github.com/vitessio/vitess/pull/10847 | |
| ## TODO: remove this pin once the above fixes are included in a v14 release (will be in v14.0.2) OR | |
| ## Vitess 16.0.0-SNAPSHOT becomes the dev version on vitessio/main | |
| #sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 467B942D3A79BD29 | |
| #wget -c https://dev.mysql.com/get/mysql-apt-config_0.8.20-1_all.deb | |
| #echo mysql-apt-config mysql-apt-config/select-server select mysql-8.0 | sudo debconf-set-selections | |
| #sudo DEBIAN_FRONTEND="noninteractive" dpkg -i mysql-apt-config* | |
| #sudo apt-get update | |
| #sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y mysql-server mysql-client | |
| #### | |
| wget -c https://cdn.mysql.com/archives/mysql-8.0/mysql-common_8.0.28-1ubuntu20.04_amd64.deb \ | |
| https://cdn.mysql.com/archives/mysql-8.0/mysql-community-client-core_8.0.28-1ubuntu20.04_amd64.deb \ | |
| https://cdn.mysql.com/archives/mysql-8.0/mysql-community-client-plugins_8.0.28-1ubuntu20.04_amd64.deb \ | |
| https://cdn.mysql.com/archives/mysql-8.0/mysql-client_8.0.28-1ubuntu20.04_amd64.deb \ | |
| https://cdn.mysql.com/archives/mysql-8.0/mysql-community-server-core_8.0.28-1ubuntu20.04_amd64.deb \ | |
| https://cdn.mysql.com/archives/mysql-8.0/mysql-community-server_8.0.28-1ubuntu20.04_amd64.deb \ | |
| https://cdn.mysql.com/archives/mysql-8.0/mysql-community-client_8.0.28-1ubuntu20.04_amd64.deb | |
| sudo DEBIAN_FRONTEND="noninteractive" apt-get install -y ./mysql-*.deb | |
| # Install everything else we need, and configure | |
| sudo apt-get install -y make unzip g++ etcd curl git wget eatmydata grep | |
| sudo service mysql stop | |
| sudo service etcd stop | |
| sudo bash -c "echo '/usr/sbin/mysqld { }' > /etc/apparmor.d/usr.sbin.mysqld" # https://bugs.launchpad.net/ubuntu/+source/mariadb-10.1/+bug/1806263 | |
| sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/ | |
| sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld || echo "could not remove mysqld profile" | |
| # install JUnit report formatter | |
| go install github.com/vitessio/go-junit-report@HEAD | |
| wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb | |
| sudo apt-get install -y gnupg2 | |
| sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb | |
| sudo apt-get update | |
| sudo apt-get install percona-xtrabackup-24 | |
| # Checkout to the last release of Vitess | |
| - name: Checkout to the other version's code (${{ needs.get_previous_release.outputs.previous_release }}) | |
| if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ needs.get_previous_release.outputs.previous_release }} | |
| - name: Get dependencies for the last release | |
| if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' | |
| run: | | |
| go mod download | |
| - name: Building last release's binaries | |
| if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' | |
| timeout-minutes: 5 | |
| run: | | |
| source build.env | |
| make build | |
| mkdir -p /tmp/vitess-build-other/ | |
| cp -R bin /tmp/vitess-build-other/ | |
| rm -Rf bin/* | |
| # Checkout to this build's commit | |
| - name: Checkout to commit's code | |
| if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' | |
| uses: actions/checkout@v3 | |
| - name: Get dependencies for this commit | |
| if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' | |
| run: | | |
| go mod download | |
| - name: Run make minimaltools | |
| if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' | |
| run: | | |
| make minimaltools | |
| - name: Building the binaries for this commit | |
| if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' | |
| timeout-minutes: 5 | |
| run: | | |
| source build.env | |
| make build | |
| mkdir -p /tmp/vitess-build-current/ | |
| cp -R bin /tmp/vitess-build-current/ | |
| # We create a sharded Vitess cluster following the local example. | |
| # We also insert a few rows in our three tables. | |
| - name: Create the example Vitess cluster with all components using version N | |
| if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' | |
| timeout-minutes: 5 | |
| run: | | |
| source build.env ; cd examples/local | |
| ./backups/start_cluster.sh | |
| # Taking a backup | |
| - name: Take a backup of all the shards | |
| if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' | |
| timeout-minutes: 2 | |
| run: | | |
| source build.env ; cd examples/local | |
| ./backups/take_backups.sh | |
| # We insert more data in every table after the backup. | |
| # When we restore the backup made in the previous step, we do not want to see the rows we are about to insert now. | |
| # The initial number of rows for each table is: | |
| # - customer: 5 | |
| # - product: 2 | |
| # - corder: 5 | |
| # We shall see the same number of rows after restoring the backup. | |
| - name: Insert more data after the backup | |
| if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' | |
| run: | | |
| source build.env ; cd examples/local ; source ./env.sh | |
| echo "insert into customer(email) values('new_user_1@domain.com');" | mysql | |
| echo "insert into product(sku, description, price) values('SKU-1009', 'description', 89);" | mysql | |
| echo "insert into corder(customer_id, sku, price) values(1, 'SKU-1009', 100);" | mysql | |
| # Stop all the tablets and remove their data | |
| - name: Stop tablets | |
| if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' | |
| timeout-minutes: 5 | |
| run: | | |
| source build.env ; cd examples/local | |
| ./backups/stop_tablets.sh | |
| # We downgrade: we use the version N-1 of vttablet | |
| - name: Downgrade - Swap binaries, use VTTablet N-1 | |
| if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' | |
| run: | | |
| source build.env | |
| rm -f $PWD/bin/vttablet | |
| cp /tmp/vitess-build-other/bin/vttablet $PWD/bin/vttablet | |
| vttablet --version | |
| # Starting the tablets again, they will automatically start restoring the last backup. | |
| - name: Start new tablets and restore | |
| if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' | |
| timeout-minutes: 5 | |
| run: | | |
| source build.env ; cd examples/local | |
| ./backups/restart_tablets.sh | |
| # give enough time to the tablets to restore the backup | |
| sleep 60 | |
| # Count the number of rows in each table to make sure the restoration is successful. | |
| - name: Assert the number of rows in every table | |
| if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' | |
| run: | | |
| source build.env ; cd examples/local ; source ./env.sh | |
| echo "select count(sku) from product;" | mysql 2>&1| grep 2 | |
| echo "select count(email) from customer;" | mysql 2>&1| grep 5 | |
| echo "select count(sku) from corder;" | mysql 2>&1| grep 5 | |
| # We insert one more row in every table. | |
| - name: Insert more rows in the tables | |
| if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' | |
| run: | | |
| source build.env ; cd examples/local ; source ./env.sh | |
| echo "insert into customer(email) values('new_user_2@domain.com');" | mysql | |
| echo "insert into product(sku, description, price) values('SKU-1011', 'description', 111);" | mysql | |
| echo "insert into corder(customer_id, sku, price) values(1, 'SKU-1011', 111);" | mysql | |
| # Taking a second backup of the cluster. | |
| - name: Take a second backup of all the shards | |
| if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' | |
| timeout-minutes: 2 | |
| run: | | |
| source build.env ; cd examples/local | |
| ./backups/take_backups.sh | |
| # Stopping the tablets so we can perform the upgrade. | |
| - name: Stop tablets | |
| if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' | |
| timeout-minutes: 5 | |
| run: | | |
| source build.env ; cd examples/local | |
| ./backups/stop_tablets.sh | |
| # We upgrade: we swap binaries and use the version N of the tablet. | |
| - name: Upgrade - Swap binaries, use VTTablet N | |
| if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' | |
| run: | | |
| source build.env | |
| rm -f $PWD/bin/vttablet | |
| cp /tmp/vitess-build-current/bin/vttablet $PWD/bin/vttablet | |
| vttablet --version | |
| # Starting the tablets again and restoring the previous backup. | |
| - name: Start new tablets and restore | |
| if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' | |
| timeout-minutes: 5 | |
| run: | | |
| source build.env ; cd examples/local | |
| ./backups/restart_tablets.sh | |
| # give enough time to the tablets to restore the backup | |
| sleep 60 | |
| # We count the number of rows in every table to check that the restore step was successful. | |
| - name: Assert the number of rows in every table | |
| if: steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' | |
| run: | | |
| source build.env ; cd examples/local ; source ./env.sh | |
| echo "select count(sku) from product;" | mysql 2>&1| grep 3 | |
| echo "select count(email) from customer;" | mysql 2>&1| grep 6 | |
| echo "select count(sku) from corder;" | mysql 2>&1| grep 6 | |
| - name: Stop the Vitess cluster | |
| if: always() && steps.skip-workflow.outputs.skip-workflow == 'false' && steps.changes.outputs.end_to_end == 'true' | |
| run: | | |
| source build.env ; cd examples/local | |
| ./401_teardown.sh || true |