build.yml

name: Dev Build

on:
  workflow_dispatch:
  pull_request:
    branches: ["main"]
    paths-ignore:
      - ".github/workflows/*.yml"
      - "**/*.md"
      - ".vscode/*"

jobs:
  dev-build:
    runs-on: ubuntu-latest
    env:
      APP_VERSION: ${{ vars.APP_VERSION || 'dev' }}
      DOCKER_REPOSITORY: ${{ vars.DOCKER_REPOSITORY || github.repository_owner }}
      MINIO_ROOT_USER: AKIAIOSFODNN7EXAMPLE
      MINIO_ROOT_PASSWORD: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
    services:
      mongodb:
        image: mongo:7.0.1
        ports:
          - 27017:27017
      minio:
        image: bitnami/minio
        ports:
          - 9000:9000
          - 9001:9001
        env:
          MINIO_ROOT_USER: ${{ env.MINIO_ROOT_USER }}
          MINIO_ROOT_PASSWORD: ${{ env.MINIO_ROOT_PASSWORD }}

    steps:
      - uses: actions/checkout@v4

      - name: Setup MongoDB
        run: yarn && node client.js
        working-directory: ./test/setup

      - name: Setup minio
        run: |
          sleep 10
          sudo curl -s -o /usr/bin/mc https://dl.minio.io/client/mc/release/linux-amd64/mc
          sudo chmod u+x /usr/bin/mc
          sudo mc config host add local http://127.0.0.1:9000 ${{ env.MINIO_ROOT_USER }} ${{ env.MINIO_ROOT_PASSWORD }} --api S3v4
          sudo mc mb local/backup

      - name: Setup SFTP
        run: |
          mkdir -p test/gh-actions/sftp/ssh
          mkdir -p test/backups/sftp
          ssh-keygen -t rsa -b 4096 -f test/gh-actions/sftp/ssh/id_rsa -q -N "YOUR_PASSPHRASE_HERE" < /dev/null
          sudo chown 1001:1001 test/backups/sftp
          docker run -d --name sftp \
              -v `pwd`/test/gh-actions/sftp/ssh/id_rsa.pub:/home/mgob/.ssh/keys/id_rsa.pub:ro \
              -v `pwd`/test/backups/sftp:/home/mgob/upload \
              --ulimit nofile=65536:65536 \
              -p 2222:22 \
              atmoz/sftp:alpine \
              mgob:secret:1001::upload

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Set current date as env variable
        run: echo "NOW=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_ENV

      - name: Login to DockerHub
        if: github.event_name == 'workflow_dispatch'
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}

      - name: Build Mgob image
        if: github.event_name == 'workflow_dispatch'
        uses: docker/build-push-action@v6
        with:
          platforms: linux/amd64
          build-args: |
            BUILD_DATE=${{ env.NOW }}
            VCS_REF=${{ github.sha }}
            VERSION=${{ env.APP_VERSION }}.${{ github.run_number }}
            EN_AWS_CLI=true
            EN_AZURE=true
            EN_GCLOUD=true
            EN_MINIO=true
            EN_RCLONE=true
            EN_GPG=true
          tags: ${{ env.DOCKER_REPOSITORY }}/mgob:${{ env.APP_VERSION }}.${{ github.run_number }}
          push: false
          load: true
          cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/mgob:buildcache-dev
          cache-to: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/mgob:buildcache-dev,mode=max

      - name: Build Mgob image for PR
        if: github.event_name == 'pull_request'
        uses: docker/build-push-action@v6
        with:
          platforms: linux/amd64
          build-args: |
            BUILD_DATE=${{ env.NOW }}
            VCS_REF=${{ github.sha }}
            VERSION=${{ env.APP_VERSION }}.${{ github.run_number }}
            EN_AWS_CLI=true
            EN_AZURE=true
            EN_GCLOUD=true
            EN_MINIO=true
            EN_RCLONE=true
            EN_GPG=true
          tags: ${{ env.DOCKER_REPOSITORY }}/mgob:${{ env.APP_VERSION }}.${{ github.run_number }}
          push: false
          load: true
          cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/mgob:buildcache-dev

      - name: Setup key
        if: github.event_name == 'workflow_dispatch'
        env:
          GCLOUD_KEY: ${{ secrets.GCLOUD_KEY }}
        run: |
          echo "$GCLOUD_KEY" | base64 --decode > ${{ github.workspace }}/test/gh-actions/gcloud.json

      - name: Install yq and remove gcloud section if event is pull_request
        if: github.event_name == 'pull_request'
        run: |
          sudo add-apt-repository ppa:rmescandon/yq -y
          sudo apt-get install -y yq
          yq eval 'del(.gcloud)' -i test/gh-actions/mongo-test.yml

      - name: Run Mgob image
        env:
          AZURE_CONNECTIONSTRING: ${{ secrets.AZURE_CONNECTIONSTRING }}
        run: |
          docker run -d \
          --name mgob \
          --network "host" \
          -e MONGO_TEST__AZURE_CONNECTIONSTRING="$AZURE_CONNECTIONSTRING" \
          -v ${{ github.workspace }}/test/gh-actions:/config \
          -v ${{ github.workspace }}/test/backups:/storage \
          ${{ env.DOCKER_REPOSITORY }}:${{ env.APP_VERSION }}.${{ github.run_number }}

      - name: Verify Mgob container is running
        run: |
          if [ "$(docker inspect -f '{{.State.Status}}' mgob)" != "running" ]; then
            echo "Mgob container is not running!"
            docker logs mgob
            exit 1
          fi
          echo "Mgob container is running."

      - name: Check Mgob container logs for startup errors
        run: |
          if docker logs mgob 2>&1 | grep -iq "error"; then
            echo "Error found in Mgob startup logs!"
            docker logs mgob
            exit 1
          fi
          echo "No errors in Mgob startup logs."

      - name: Wait for Mgob service to be ready
        run: |
          echo 'Waiting for Mgob to be ready...'
          for i in {1..10}; do
            if curl -s http://127.0.0.1:8090/version -o /dev/null; then
              echo "Mgob is ready!"
              break
            elif [ "$i" -eq "10" ]; then
              echo "Mgob not ready after multiple attempts."
              docker logs mgob
              exit 1
            fi
            echo "Waiting for Mgob... retrying in 5 seconds"
            sleep 5
          done
          echo 'Mgob service is available.'

      - name: Verify mgob backup for PR
        run: |
          sleep 90
          echo '💥💥💥💥💥💥💥💥 Get mgob Version 💥💥💥💥💥💥💥💥💥💥'
          curl http://127.0.0.1:8090/version | tee version.txt
          echo '💥💥💥💥💥💥💥💥 Get mgob Metrics 💥💥💥💥💥💥💥💥💥💥'
          curl http://127.0.0.1:8090/metrics > metrics.txt
          echo '💥💥💥💥💥💥💥💥 Get mgob logs 💥💥💥💥💥💥💥💥💥💥'
          logs=$(docker logs mgob 2>&1)
          echo "$logs" | tee logs.txt
          echo '💥💥💥💥💥💥💥💥 verify mgob backup 💥💥💥💥💥💥💥💥💥💥'
          # verify for s3
          if ! grep -q "S3 upload finished" logs.txt; then
            echo "S3 integration test failed"
            exit 1
          fi
          if ! grep -q "Local backup finished.*plan=mongo-test" logs.txt; then
            echo "Local backup integration test failed"
            exit 1
          fi
          # verify for no-local plan
          if grep -q "Local backup finished.*plan=no-local" logs.txt; then
            echo "Skip Local backup integration test failed"
            exit 1
          fi
          # shouldn't have any files in the no-local directory
          if [[ $(ls -A "${{ github.workspace }}/test/backups/no-local") ]]; then
              echo "There are files in the directory."
              exit 1
          fi
          # verify for sftp
          if ! grep -q "SFTP upload finished" logs.txt; then
            echo "SFTP integration test failed"
            exit 1
          fi

      - name: Verify mgob cloud backup
        if: github.event_name == 'workflow_dispatch'
        run: |
          logs=$(docker logs mgob 2>&1)
          echo "$logs" | tee logs.txt
          echo '💥💥💥💥💥💥💥💥 verify mgob backup 💥💥💥💥💥💥💥💥💥💥'
          # verify for gcloud
          if ! grep -q "GCloud upload finished Copying" logs.txt; then
            echo "GCloud integration test failed"
            exit 1
          fi
          # # verify for azure
          # if ! grep -q "Azure upload finished" logs.txt; then
          #   echo "Azure integration test failed, check if the connection string is expired"
          #   exit 1
          # fi

      - name: Upload file as artifact
        uses: actions/upload-artifact@v4
        with:
          name: logs
          path: logs.txt
      - name: Upload version as artifact
        uses: actions/upload-artifact@v4
        with:
          name: version
          path: version.txt
      - name: Upload metrics as artifact
        uses: actions/upload-artifact@v4
        with:
          name: metrics
          path: metrics.txt

      - name: Verify mgob restore
        run: |
          echo 'Restore integration test'
          fullpath=$(find "${{ github.workspace }}/test/backups/mongo-test" -type f -name "*.gz" | head -n 1)
          if [[ "$fullpath" == "" ]]; then
            echo "No backup file found"
            exit 1
          fi
          filename=$(basename -- "$fullpath")

          echo "http://127.0.0.1:8090/restore/mongo-test/$filename"

          # Make the curl request and capture the status code
          status_code=$(curl -o /dev/null -s -w "%{http_code}" "http://127.0.0.1:8090/restore/mongo-test/$filename" -X POST)

          docker logs mgob 2>&1

          # Check if the status code is 200
          if [ "$status_code" -eq 200 ]; then
              echo "Restore backup successfully"
          else
              echo "Failed to restore backup"
              exit 1
          fi

      - name: Verify metrics
        run: |
          echo 'Verify metrics'
          # should find "mgob_scheduler_backup_total{plan="mongo-test",status="200"}"
          if ! grep -q "mgob_scheduler_backup_total{plan=\"mongo-test\",status=\"200\"}" metrics.txt; then
            echo "Metrics integration test failed for mongo-test"
            exit 1
          fi
          # should find "mgob_scheduler_backup_total{plan="no-local",status="200"}"
          if ! grep -q "mgob_scheduler_backup_total{plan=\"no-local\",status=\"200\"}" metrics.txt; then
            echo "Metrics integration test failed for no-local"
            exit 1
          fi

      - name: Push mgob image
        if: github.event_name == 'workflow_dispatch'
        uses: docker/build-push-action@v6
        with:
          platforms: linux/amd64,linux/arm64
          build-args: |
            BUILDKIT_MULTI_PLATFORM=1
            BUILD_DATE=${{ env.NOW }}
            VCS_REF=${{ github.sha }}
            VERSION=${{ env.APP_VERSION }}.${{ github.run_number }}
            EN_AWS_CLI=true
            EN_AZURE=true
            EN_GCLOUD=true
            EN_MINIO=true
            EN_RCLONE=true
            EN_GPG=true
          tags: ${{ env.DOCKER_REPOSITORY }}/mgob:${{ env.APP_VERSION }}.${{ github.run_number }}
          push: true
          cache-from: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/mgob:buildcache-dev
          cache-to: type=registry,ref=${{ secrets.DOCKER_USERNAME }}/mgob:buildcache-dev,mode=max