implement #131: create bucket if it doesn't exist yet, mc only yet #300
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Dev Build | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| branches: ["main"] | |
| jobs: | |
| dev-build: | |
| runs-on: ubuntu-latest | |
| env: | |
| APP_VERSION: dev | |
| MINIO_ROOT_USER: AKIAIOSFODNN7EXAMPLE | |
| MINIO_ROOT_PASSWORD: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY | |
| services: | |
| mongodb: | |
| image: mongo:7.0.1 | |
| ports: | |
| - 27017:27017 | |
| minio: | |
| image: bitnami/minio | |
| ports: | |
| - 9000:9000 | |
| - 9001:9001 | |
| env: | |
| MINIO_ROOT_USER: ${{ env.MINIO_ROOT_USER }} | |
| MINIO_ROOT_PASSWORD: ${{ env.MINIO_ROOT_PASSWORD }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup MongoDB | |
| run: yarn && node client.js | |
| working-directory: ./test/setup | |
| - name: Setup minio | |
| run: | | |
| sleep 10 | |
| sudo curl -s -o /usr/bin/mc https://dl.minio.io/client/mc/release/linux-amd64/mc | |
| sudo chmod u+x /usr/bin/mc | |
| sudo mc config host add local http://127.0.0.1:9000 ${{ env.MINIO_ROOT_USER }} ${{ env.MINIO_ROOT_PASSWORD }} --api S3v4 | |
| sudo mc mb local/backup | |
| - name: Setup SFTP | |
| run: | | |
| mkdir -p test/gh-actions/sftp/ssh | |
| mkdir -p test/backups/sftp | |
| ssh-keygen -t rsa -b 4096 -f test/gh-actions/sftp/ssh/id_rsa -q -N "YOUR_PASSPHRASE_HERE" < /dev/null | |
| sudo chown 1001:1001 test/backups/sftp | |
| docker run -d --name sftp \ | |
| -v `pwd`/test/gh-actions/sftp/ssh/id_rsa.pub:/home/mgob/.ssh/keys/id_rsa.pub:ro \ | |
| -v `pwd`/test/backups/sftp:/home/mgob/upload \ | |
| --ulimit nofile=65536:65536 \ | |
| -p 2222:22 \ | |
| atmoz/sftp:alpine \ | |
| mgob:secret:1001::upload | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Set current date as env variable | |
| run: echo "NOW=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_ENV | |
| - name: Login to DockerHub | |
| if: github.event_name == 'workflow_dispatch' | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| - name: Build Mgob image | |
| uses: docker/build-push-action@v3 | |
| with: | |
| build-args: | | |
| BUILD_DATE=${{ env.NOW }} | |
| VCS_REF=${{ github.sha }} | |
| VERSION=${{ env.APP_VERSION }}.${{ github.run_number }} | |
| EN_AWS_CLI=true | |
| EN_AZURE=true | |
| EN_GCLOUD=true | |
| EN_MINIO=true | |
| EN_RCLONE=true | |
| EN_GPG=true | |
| tags: ${{ github.repository }}:${{ env.APP_VERSION }}.${{ github.run_number }} | |
| push: false | |
| load: true | |
| cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/mgob:buildcache-dev | |
| cache-to: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/mgob:buildcache-dev,mode=max | |
| - name: Setup key | |
| env: | |
| GCLOUD_KEY: ${{ secrets.GCLOUD_KEY }} | |
| run: | | |
| echo "$GCLOUD_KEY" | base64 --decode > ${{ github.workspace }}/test/gh-actions/gcloud.json | |
| - name: Run Mgob image | |
| env: | |
| AZURE_CONNECTIONSTRING: ${{ secrets.AZURE_CONNECTIONSTRING }} | |
| run: | | |
| docker run -d \ | |
| --name mgob \ | |
| --network "host" \ | |
| -e MONGO_TEST__AZURE_CONNECTIONSTRING="$AZURE_CONNECTIONSTRING" \ | |
| -v ${{ github.workspace }}/test/gh-actions:/config \ | |
| -v ${{ github.workspace }}/test/backups:/storage \ | |
| ${{ github.repository }}:${{ env.APP_VERSION }}.${{ github.run_number }} | |
| - name: Verify mgob backup | |
| run: | | |
| sleep 90 | |
| echo '💥💥💥💥💥💥💥💥 Get mgob Version 💥💥💥💥💥💥💥💥💥💥' | |
| curl http://127.0.0.1:8090/version | tee version.txt | |
| echo '💥💥💥💥💥💥💥💥 Get mgob Metrics 💥💥💥💥💥💥💥💥💥💥' | |
| curl http://127.0.0.1:8090/metrics > metrics.txt | |
| echo '💥💥💥💥💥💥💥💥 Get mgob logs 💥💥💥💥💥💥💥💥💥💥' | |
| logs=$(docker logs mgob 2>&1) | |
| echo "$logs" | tee logs.txt | |
| echo '💥💥💥💥💥💥💥💥 verify mgob backup 💥💥💥💥💥💥💥💥💥💥' | |
| # verify for s3 | |
| if ! grep -q "S3 upload finished" logs.txt; then | |
| echo "S3 integration test failed" | |
| exit 1 | |
| fi | |
| if ! grep -q "Local backup finished.*plan=mongo-test" logs.txt; then | |
| echo "Local backup integration test failed" | |
| exit 1 | |
| fi | |
| # verify for no-local plan | |
| if grep -q "Local backup finished.*plan=no-local" logs.txt; then | |
| echo "Skip Local backup integration test failed" | |
| exit 1 | |
| fi | |
| # shouldn't have any files in the no-local directory | |
| if [[ $(ls -A "${{ github.workspace }}/test/backups/no-local") ]]; then | |
| echo "There are files in the directory." | |
| exit 1 | |
| fi | |
| # verify for gcloud | |
| if ! grep -q "GCloud upload finished Copying" logs.txt; then | |
| echo "GCloud integration test failed" | |
| exit 1 | |
| fi | |
| # # verify for azure | |
| # if ! grep -q "Azure upload finished" logs.txt; then | |
| # echo "Azure integration test failed, check if the connection string is expired" | |
| # exit 1 | |
| # fi | |
| # verify for sftp | |
| if ! grep -q "SFTP upload finished" logs.txt; then | |
| echo "SFTP integration test failed" | |
| exit 1 | |
| fi | |
| - name: Upload file as artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: logs | |
| path: logs.txt | |
| - name: Upload version as artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: version | |
| path: version.txt | |
| - name: Upload metrics as artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: metrics | |
| path: metrics.txt | |
| - name: Verify mgob restore | |
| run: | | |
| echo 'Restore integration test' | |
| fullpath=$(find "${{ github.workspace }}/test/backups/mongo-test" -type f -name "*.gz" | head -n 1) | |
| if [[ "$fullpath" == "" ]]; then | |
| echo "No backup file found" | |
| exit 1 | |
| fi | |
| filename=$(basename -- "$fullpath") | |
| echo "http://127.0.0.1:8090/restore/mongo-test/$filename" | |
| # Make the curl request and capture the status code | |
| status_code=$(curl -o /dev/null -s -w "%{http_code}" "http://127.0.0.1:8090/restore/mongo-test/$filename" -X POST) | |
| docker logs mgob 2>&1 | |
| # Check if the status code is 200 | |
| if [ "$status_code" -eq 200 ]; then | |
| echo "Restore backup successfully" | |
| else | |
| echo "Failed to restore backup" | |
| exit 1 | |
| fi | |
| - name: Verify metrics | |
| run: | | |
| echo 'Verify metrics' | |
| # should find "mgob_scheduler_backup_total{plan="mongo-test",status="200"}" | |
| if ! grep -q "mgob_scheduler_backup_total{plan=\"mongo-test\",status=\"200\"}" metrics.txt; then | |
| echo "Metrics integration test failed for mongo-test" | |
| exit 1 | |
| fi | |
| # should find "mgob_scheduler_backup_total{plan="no-local",status="200"}" | |
| if ! grep -q "mgob_scheduler_backup_total{plan=\"no-local\",status=\"200\"}" metrics.txt; then | |
| echo "Metrics integration test failed for no-local" | |
| exit 1 | |
| fi | |
| - name: Push mgob image | |
| if: github.event_name == 'workflow_dispatch' | |
| uses: docker/build-push-action@v3 | |
| with: | |
| build-args: | | |
| BUILD_DATE=${{ env.NOW }} | |
| VCS_REF=${{ github.sha }} | |
| VERSION=${{ env.APP_VERSION }}.${{ github.run_number }} | |
| EN_AWS_CLI=true | |
| EN_AZURE=true | |
| EN_GCLOUD=true | |
| EN_MINIO=true | |
| EN_RCLONE=true | |
| EN_GPG=true | |
| tags: ${{ github.repository }}:${{ env.APP_VERSION }}.${{ github.run_number }} | |
| push: true | |
| cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/mgob:buildcache-dev | |
| cache-to: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/mgob:buildcache-dev,mode=max |