minor CI updates #3260
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test, Build and Deploy Images | |
| env: | |
| DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }} | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| push: | |
| schedule: | |
| # Run every 6 days to keep our caches alive | |
| - cron: '0 0 */6 * *' | |
| jobs: | |
| python-tests: | |
| runs-on: ubuntu-latest | |
| env: | |
| python-version: 3.9 # Our base image has Python 3.9 | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: Set up Python ${{ env.python-version }} | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: ${{ env.python-version }} | |
| - name: Install dependencies | |
| run: | | |
| sudo apt install shellcheck | |
| python -m pip install --upgrade pip | |
| pip install pipenv | |
| pipenv install poetry && pipenv run poetry install | |
| echo "installing mavlink-router" | |
| pipenv run ./core/tools/mavlink_router/bootstrap.sh | |
| echo "installing mavp2p" | |
| pipenv run ./core/tools/mavp2p/bootstrap.sh | |
| echo "installing ardupilot_tools" | |
| pipenv run ./core/tools/ardupilot_tools/bootstrap.sh | |
| pipenv run python ./core/libs/bridges/setup.py install | |
| pipenv run python ./core/libs/commonwealth/setup.py install | |
| rm -rf core/libs/bridges/build | |
| rm -rf core/libs/commonwealth/build | |
| - name: Run tests | |
| run: | | |
| .hooks/pre-push | |
| - name: Upload coverage | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: coverage | |
| path: htmlcov | |
| frontend-tests: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| - uses: oven-sh/setup-bun@v1 | |
| with: | |
| bun-version: latest | |
| - name: Bun install | |
| run: bun install --cwd ./core/frontend | |
| - name: Bun lint | |
| run: bun --cwd ./core/frontend lint | |
| - name: Bun build | |
| run: bun run --cwd ./core/frontend build | |
| deploy-docker-images: | |
| runs-on: ubuntu-latest | |
| needs: [python-tests, frontend-tests] | |
| strategy: | |
| matrix: | |
| docker: [bootstrap, core] | |
| project: [blueos] | |
| platforms: ["linux/arm/v7,linux/arm64/v8,linux/amd64"] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 #Number of commits to fetch. 0 indicates all history for all branches and tags. | |
| submodules: recursive | |
| - name: Prepare | |
| id: prepare | |
| run: | | |
| # Deploy image with the name of the branch, if the build is a git tag, replace tag with the tag name. | |
| # If git tag matches semver, append latest tag to the push. | |
| DOCKER_IMAGE=${DOCKER_USERNAME:-bluerobotics}/${{ matrix.project }}-${{ matrix.docker }} | |
| VERSION=${GITHUB_REF##*/} | |
| if [[ $GITHUB_REF == refs/tags/* ]]; then | |
| VERSION=${GITHUB_REF#refs/tags/} | |
| fi | |
| TAGS="--tag ${DOCKER_IMAGE}:${VERSION}" | |
| if [[ $VERSION =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then | |
| TAGS="$TAGS --tag ${DOCKER_IMAGE}:latest" | |
| fi | |
| echo "docker_image=${DOCKER_IMAGE}" >> $GITHUB_OUTPUT | |
| echo "version=${VERSION}" >> $GITHUB_OUTPUT | |
| echo "buildx_args=\ | |
| --build-arg GIT_DESCRIBE_TAGS=$(git describe --tags --long --always) \ | |
| --build-arg VUE_APP_GIT_DESCRIBE=$(git describe --long --always --dirty --all) \ | |
| --cache-from 'type=local,src=/tmp/.buildx-cache' \ | |
| --cache-to 'type=local,dest=/tmp/.buildx-cache' \ | |
| ${TAGS} \ | |
| --file ${{ matrix.docker }}/Dockerfile ./${{ matrix.docker }}" >> $GITHUB_OUTPUT | |
| - name: Set up QEMU | |
| uses: docker/setup-qemu-action@v2 | |
| with: | |
| platforms: all | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| with: | |
| version: latest | |
| - name: Cache Docker layers | |
| uses: actions/cache@v3 | |
| id: cache | |
| with: | |
| path: /tmp/.buildx-cache | |
| key: ${{ runner.os }}-buildx-${{ matrix.docker }}-${{ hashFiles(format('{0}/Dockerfile', matrix.docker)) }} | |
| restore-keys: | | |
| ${{ runner.os }}-buildx-${{ matrix.docker }}-${{ hashFiles(format('{0}/Dockerfile', matrix.docker)) }} | |
| ${{ runner.os }}-buildx-${{ matrix.docker }} | |
| - name: Docker Buildx (build) | |
| run: | | |
| # Pull latest version of image to help with build speed | |
| for platform in $(echo ${{ matrix.platforms }} | tr ',' '\n'); do | |
| docker pull --platform ${platform} ${DOCKER_USERNAME:-bluerobotics}/${{ matrix.project }}-${{ matrix.docker }}:master || true | |
| done | |
| docker buildx build \ | |
| --output "type=image,push=false" \ | |
| --platform ${{ matrix.platforms }} \ | |
| ${{ steps.prepare.outputs.buildx_args }} | |
| - name: Check core size | |
| if: ${{ matrix.docker == 'core' }} | |
| run: | | |
| # Check if the image size is lower than our limit | |
| docker image list | |
| IMAGE_ID=$(docker images -q ${DOCKER_USERNAME:-bluerobotics}/${{ matrix.project }} | head -n 1) | |
| LIMIT_SIZE_MB=700 | |
| IMAGE_SIZE_MB=$(( $(docker inspect $IMAGE_ID --format {{.Size}})/(2**20) )) | |
| echo "Core size is: $IMAGE_SIZE_MB MB" | |
| ((IMAGE_SIZE_MB < LIMIT_SIZE_MB)) | |
| - name: Login to DockerHub | |
| if: success() && github.event_name != 'pull_request' | |
| uses: docker/login-action@v2 | |
| with: | |
| username: ${{ secrets.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| - name: Docker Buildx (push) | |
| if: success() && github.event_name != 'pull_request' | |
| run: | | |
| docker buildx build \ | |
| --output "type=image,push=true" \ | |
| --platform ${{ matrix.platforms }} \ | |
| ${{ steps.prepare.outputs.buildx_args }} | |
| - name: Inspect image | |
| if: always() && github.event_name != 'pull_request' | |
| run: | | |
| docker buildx imagetools \ | |
| inspect ${{ steps.prepare.outputs.docker_image }}:${{ steps.prepare.outputs.version }} | |
| - name: Create image artifact | |
| if: success() && matrix.docker == 'core' | |
| run: | | |
| DOCKER_IMAGE=${DOCKER_USERNAME:-bluerobotics}/${{ matrix.project }}-${{ matrix.docker }} | |
| GIT_HASH_SHORT=$(git rev-parse --short "$GITHUB_SHA") | |
| docker buildx build \ | |
| ${{ steps.prepare.outputs.buildx_args }} \ | |
| --platform "linux/arm64/v8" \ | |
| --tag ${DOCKER_IMAGE}:${GIT_HASH_SHORT} \ | |
| --output "type=docker,dest=BlueOs-core-docker-image-${GIT_HASH_SHORT}-arm64-v8.tar" \ | |
| - name: Upload artifact arm64-v8 | |
| uses: actions/upload-artifact@v3 | |
| if: success() && matrix.docker == 'core' | |
| with: | |
| name: BlueOS-core-docker-image-arm64-v8 | |
| path: '*arm64-v8.tar' | |
| - name: Create image artifact | |
| if: success() && matrix.docker == 'core' | |
| run: | | |
| DOCKER_IMAGE=${DOCKER_USERNAME:-bluerobotics}/${{ matrix.project }}-${{ matrix.docker }} | |
| GIT_HASH_SHORT=$(git rev-parse --short "$GITHUB_SHA") | |
| docker buildx build \ | |
| ${{ steps.prepare.outputs.buildx_args }} \ | |
| --platform "linux/arm/v7" \ | |
| --tag ${DOCKER_IMAGE}:${GIT_HASH_SHORT} \ | |
| --output "type=docker,dest=BlueOs-core-docker-image-${GIT_HASH_SHORT}-arm-v7.tar" \ | |
| - name: Upload artifact arm-v7 | |
| uses: actions/upload-artifact@v3 | |
| if: success() && matrix.docker == 'core' | |
| with: | |
| name: BlueOS-core-docker-image-arm-v7 | |
| path: '*arm-v7.tar' | |
| - name: Create image artifact | |
| if: success() && matrix.docker == 'core' | |
| run: | | |
| DOCKER_IMAGE=${DOCKER_USERNAME:-bluerobotics}/${{ matrix.project }}-${{ matrix.docker }} | |
| GIT_HASH_SHORT=$(git rev-parse --short "$GITHUB_SHA") | |
| docker buildx build \ | |
| ${{ steps.prepare.outputs.buildx_args }} \ | |
| --platform "linux/amd64" \ | |
| --tag ${DOCKER_IMAGE}:${GIT_HASH_SHORT} \ | |
| --output "type=docker,dest=BlueOs-core-docker-image-${GIT_HASH_SHORT}-amd64.tar" \ | |
| - name: Upload artifact amd64 | |
| uses: actions/upload-artifact@v3 | |
| if: success() && matrix.docker == 'core' | |
| with: | |
| name: BlueOS-core-docker-image-amd64 | |
| path: '*amd64.tar' | |
| - name: Upload docker image for release | |
| uses: svenstaro/upload-release-action@v2 | |
| if: startsWith(github.ref, 'refs/tags/') && success() && matrix.docker == 'core' | |
| with: | |
| repo_token: ${{ secrets.GITHUB_TOKEN }} | |
| file: '*.tar' | |
| tag: ${{ github.ref }} | |
| overwrite: true | |
| prerelease: true | |
| file_glob: true | |
| deploy-raspberry-image: | |
| needs: deploy-docker-images | |
| if: github.event_name != 'pull_request' | |
| runs-on: self-hosted | |
| # The runner for this job is a Raspberry Pi OS Bullseye. | |
| # Just install docker (curl -sSL https://get.docker.com/ | sh) and follow the instructions for setting up a new runner in | |
| # https://github.com/bluerobotics/BlueOS-docker/settings/actions/runners/new | |
| strategy: | |
| matrix: | |
| platforms: ["linux/arm/v7"] | |
| steps: | |
| - name: Install git | |
| run: sudo apt install -y git | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| # We use our own pimod as upstream doesn't provide armv7 images | |
| - name: Pimod Build | |
| run: | | |
| VERSION=$GITHUB_REPOSITORY | |
| VERSION=${VERSION:-master} | |
| wget https://raw.githubusercontent.com/williangalvani/pimod/master/pimod.sh && chmod +x pimod.sh | |
| docker run --rm --privileged \ | |
| -v $PWD:/files \ | |
| -e PATH=/pimod:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ | |
| -e GITHUB_REPOSITORY=$GITHUB_REPOSITORY \ | |
| -e VERSION=$GITHUB_REF_NAME \ | |
| --workdir=/files \ | |
| --platform ${{ matrix.platforms }} williangalvani/pimod:latest pimod.sh deploy/pimod/blueos.Pifile | |
| # TODO: add GITHUB_REF_NAME after https://github.com/actions/upload-artifact/issues/231 is fixed | |
| # name: blueos-${{ env.GITHUB_REF_NAME }}.zip | |
| - name: Zip image | |
| if: startsWith(github.ref, 'refs/tags/') | |
| run: | | |
| sudo apt install zip | |
| zip BlueOS-raspberry.zip deploy/pimod/blueos.img | |
| - name: Upload artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| # We use the .img to avoid a zip of a zip | |
| name: BlueOS-raspberry.zip | |
| path: deploy/pimod/blueos.img | |
| if-no-files-found: error | |
| retention-days: 7 | |
| - name: Upload raspberry image for release | |
| uses: svenstaro/upload-release-action@v2 | |
| if: startsWith(github.ref, 'refs/tags/') | |
| with: | |
| repo_token: ${{ secrets.GITHUB_TOKEN }} | |
| file: BlueOS-raspberry.zip | |
| tag: ${{ github.ref }} | |
| overwrite: true | |
| prerelease: true | |
| # This is required because docker has root permissions, which means the runner is unable to clear this cache normally | |
| - name: Cleanup | |
| if: ${{ always() }} | |
| run: sudo rm -rf .cache |