Work-around systemctl makedirs race condition: FileExistsError: [Errn… #277
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI Full Run | |
| on: | |
| pull_request: | |
| branches: | |
| - main | |
| - grok/*/* | |
| push: | |
| branches: | |
| - main | |
| tags: | |
| - "v*" | |
| jobs: | |
| # Tests command-line tool using tests scaffolding. | |
| ci-storage-tool-test: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: | | |
| exec 2>&1; set -e -o xtrace | |
| tests/all.sh | |
| # Tests action itself. | |
| ci-storage-action-test: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Create dummy file | |
| run: | | |
| echo "dummy" > dummy.txt | |
| mkdir -p dir/subdir | |
| echo "layer" > dir/subdir/layer.txt | |
| echo -n "" > ~/ci-storage-host | |
| - name: Test store | |
| uses: ./ | |
| with: | |
| action: store | |
| storage-dir: ~/storage-dir | |
| - name: Test store (layer) | |
| uses: ./ | |
| with: | |
| action: store | |
| storage-dir: ~/storage-dir | |
| layer-name: my-layer | |
| layer-include: layer.txt | |
| - name: Remove dummy file | |
| run: rm dummy.txt | |
| - name: Test load | |
| uses: ./ | |
| with: | |
| action: load | |
| storage-dir: ~/storage-dir | |
| - name: Check that dummy.txt was restored | |
| run: | | |
| set -e | |
| ls -la ~/storage-dir/${{ github.repository }} | |
| [[ "$(cat dummy.txt)" == "dummy" ]] || { echo "dummy.txt must be restored"; exit 1; } | |
| - name: Remove layer.txt file and dir/subdir hierarchy | |
| run: rm -rf dir | |
| - name: Test load (layer) | |
| uses: ./ | |
| with: | |
| action: load | |
| storage-dir: ~/storage-dir | |
| layer-name: my-layer | |
| - name: Check that dir/subdir/layer.txt was restored, and dummy.txt still exists | |
| run: | | |
| set -e | |
| ls -la ~/storage-dir/${{ github.repository }}.my-layer | |
| [[ "$(cat dummy.txt)" == "dummy" ]] || { echo "dummy.txt must be kept"; exit 1; } | |
| [[ "$(cat dir/subdir/layer.txt)" == "layer" ]] || { echo "layer.txt must be restored"; exit 1; } | |
| # Tests ci-scaler logic. | |
| ci-scaler-test: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - run: | | |
| exec 2>&1; set -e -o xtrace | |
| docker/ci-scaler/guest/scaler/tests/all.sh | |
| env: | |
| GH_TOKEN: ${{ secrets.CI_PAT }} | |
| # Builds and boots a ci-runner container inside GitHub's infra. Once it's | |
| # settled, there is a running container with one self-hosted runner waiting | |
| # for jobs with ci-storage-test tag to pick up (based on Dockerfile image). | |
| build-and-boot-containers: | |
| runs-on: ubuntu-latest | |
| needs: | |
| - ci-storage-tool-test | |
| - ci-storage-action-test | |
| - ci-scaler-test | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Start test Docker containers | |
| run: | | |
| exec 2>&1; set -e -o xtrace | |
| cd docker | |
| # Build all containers. | |
| docker compose build --parallel | |
| # Boot ci-storage container in background. | |
| docker compose up ci-storage -d | |
| # Boot ci-runner container. It connects to ci-storage container and | |
| # load a test (non-existent) ci-storage slot from there, then register | |
| # a GitHub self-hosted runner and remain waiting for jobs. | |
| docker compose up ci-runner | |
| env: | |
| GH_TOKEN: ${{ secrets.CI_PAT }} | |
| GH_REPOSITORY: ${{ github.repository }} | |
| GH_LABELS: ${{ format('ci-storage-test-{0}-{1}', github.run_id, github.run_attempt) }} | |
| TZ: America/Los_Angeles | |
| FORWARD_HOST: host.docker.internal | |
| # Test the job with ci-storage-test tag which is initially queued, but then is | |
| # picked up by the ci-runner container booted in the previous job. In the end, | |
| # the test job sends SIGINT to the container entrypoint.sh PID, so the | |
| # container (based on Dockerfile image) shuts down gracefully. | |
| spawn-job-test: | |
| runs-on: | |
| - self-hosted | |
| - ${{ format('ci-storage-test-{0}-{1}', github.run_id, github.run_attempt) }} | |
| needs: | |
| - ci-storage-tool-test | |
| - ci-storage-action-test | |
| - ci-scaler-test | |
| timeout-minutes: 15 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - name: Run test job inside the self-hosted runner | |
| run: echo "Hello, world!" | |
| - name: Test store using GitHub Action | |
| uses: ./ | |
| with: | |
| action: "store" | |
| - name: Kill ci-runner container | |
| run: kill -SIGINT $(cat ~guest/.entrypoint.pid) | |
| # Publishes ci-scaler image. | |
| push-ci-scaler: | |
| runs-on: ubuntu-latest | |
| if: github.event_name != 'pull_request' | |
| needs: | |
| - ci-storage-tool-test | |
| - ci-storage-action-test | |
| - ci-scaler-test | |
| - build-and-boot-containers | |
| - spawn-job-test | |
| timeout-minutes: 15 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: docker/setup-qemu-action@v3 | |
| - uses: docker/setup-buildx-action@v3 | |
| - uses: docker/metadata-action@v5 | |
| id: meta | |
| with: | |
| images: | | |
| dimikot/ci-scaler | |
| ghcr.io/${{ github.repository_owner }}/ci-scaler | |
| - uses: docker/login-action@v3 | |
| with: | |
| username: dimikot | |
| password: ${{ secrets.DOCKERHUB_PAT }} | |
| - uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: docker/build-push-action@v5 | |
| with: | |
| context: docker/ci-scaler | |
| platforms: linux/amd64,linux/arm64,linux/arm64/v8 | |
| push: ${{ github.event_name != 'pull_request' }} | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| - uses: peter-evans/dockerhub-description@v3 | |
| with: | |
| username: dimikot | |
| password: ${{ secrets.DOCKERHUB_PAT }} | |
| readme-filepath: docker/ci-scaler/README.md | |
| repository: dimikot/ci-scaler | |
| # Publishes ci-storage image. | |
| push-ci-storage: | |
| runs-on: ubuntu-latest | |
| if: github.event_name != 'pull_request' | |
| needs: | |
| - ci-storage-tool-test | |
| - ci-storage-action-test | |
| - ci-scaler-test | |
| - build-and-boot-containers | |
| - spawn-job-test | |
| timeout-minutes: 15 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: docker/setup-qemu-action@v3 | |
| - uses: docker/setup-buildx-action@v3 | |
| - uses: docker/metadata-action@v5 | |
| id: meta | |
| with: | |
| images: | | |
| dimikot/ci-storage | |
| ghcr.io/${{ github.repository_owner }}/ci-storage | |
| - uses: docker/login-action@v3 | |
| with: | |
| username: dimikot | |
| password: ${{ secrets.DOCKERHUB_PAT }} | |
| - uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: docker/build-push-action@v5 | |
| with: | |
| context: docker/ci-storage | |
| platforms: linux/amd64,linux/arm64,linux/arm64/v8 | |
| push: ${{ github.event_name != 'pull_request' }} | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| - uses: peter-evans/dockerhub-description@v3 | |
| with: | |
| username: dimikot | |
| password: ${{ secrets.DOCKERHUB_PAT }} | |
| readme-filepath: docker/ci-storage/README.md | |
| repository: dimikot/ci-storage | |
| # Publishes ci-runner image. | |
| push-ci-runner: | |
| runs-on: ubuntu-latest | |
| if: github.event_name != 'pull_request' | |
| needs: | |
| - ci-storage-tool-test | |
| - ci-storage-action-test | |
| - ci-scaler-test | |
| - build-and-boot-containers | |
| - spawn-job-test | |
| timeout-minutes: 15 | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: docker/setup-qemu-action@v3 | |
| - uses: docker/setup-buildx-action@v3 | |
| - uses: docker/metadata-action@v5 | |
| id: meta | |
| with: | |
| images: | | |
| dimikot/ci-runner | |
| ghcr.io/${{ github.repository_owner }}/ci-runner | |
| - uses: docker/login-action@v3 | |
| with: | |
| username: dimikot | |
| password: ${{ secrets.DOCKERHUB_PAT }} | |
| - uses: docker/login-action@v3 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - uses: docker/build-push-action@v5 | |
| with: | |
| context: docker/ci-runner | |
| platforms: linux/amd64,linux/arm64,linux/arm64/v8 | |
| push: ${{ github.event_name != 'pull_request' }} | |
| tags: ${{ steps.meta.outputs.tags }} | |
| labels: ${{ steps.meta.outputs.labels }} | |
| - uses: peter-evans/dockerhub-description@v3 | |
| with: | |
| username: dimikot | |
| password: ${{ secrets.DOCKERHUB_PAT }} | |
| readme-filepath: docker/ci-runner/README.md | |
| repository: dimikot/ci-runner |