diff --git a/.github/actions/check-image-tags-exist/action.yml b/.github/actions/check-image-tags-exist/action.yml index 1caf8e001..e3ab6eb74 100644 --- a/.github/actions/check-image-tags-exist/action.yml +++ b/.github/actions/check-image-tags-exist/action.yml @@ -5,48 +5,24 @@ inputs: last_commit_tag: description: 'The tag of the last commit image' required: true - common_ancestor_tag: - description: 'The tag of the common ancestor commit image' - required: true image_name: description: 'The name of the image to check' required: true - docker_username: - description: 'The username to login to Docker Hub' - required: true - docker_password: - description: 'The password to login to Docker Hub' - required: true outputs: last_commit_tag_exists: description: 'Whether the last commit image tag exists' value: ${{ steps.last_commit_image_exists.outputs.last_commit_tag_exists }} - common_ancestor_commit_tag_exists: - description: 'Whether the common ancestor commit image tag exists' - value: ${{ steps.ancestor_commit_image_exists.outputs.common_ancestor_commit_tag_exists }} runs: using: 'composite' steps: - - name: Login to Docker Hub - uses: docker/login-action@v2 - with: - username: ${{ inputs.docker_username }} - password: ${{ inputs.docker_password }} - name: Check last commit image tag exists id: last_commit_image_exists shell: bash continue-on-error: true run: | echo last_commit_tag_exists=$(docker pull ${{ inputs.image_name }}:${{ inputs.last_commit_tag }} > /dev/null ; echo $?) >> $GITHUB_OUTPUT - - name: Check ancestor commit image tag exists - shell: bash - id: ancestor_commit_image_exists - continue-on-error: true - run: | - echo common_ancestor_commit_tag_exists=$(docker pull ${{ inputs.image_name }}:${{ inputs.common_ancestor_tag }} > /dev/null ; echo $?) >> $GITHUB_OUTPUT - name: Show outputs shell: bash run: | echo "last_commit_tag_exists: ${{ steps.last_commit_image_exists.outputs.last_commit_tag_exists }}" - echo "common_ancestor_commit_tag_exists: ${{ steps.ancestor_commit_image_exists.outputs.common_ancestor_commit_tag_exists }}" \ No newline at end of file diff --git a/.github/actions/image-tag-and-push/action.yml b/.github/actions/image-tag-and-push/action.yml index 555dc0e09..519e526ef 100644 --- a/.github/actions/image-tag-and-push/action.yml +++ b/.github/actions/image-tag-and-push/action.yml @@ -8,33 +8,24 @@ inputs: last_commit_tag: description: 'The tag of the last commit image' required: true - common_ancestor_tag: - description: 'The tag of the common ancestor commit image' - required: true develop_tag: description: 'The tag of the develop image' required: true - untested_tag_suffix: - description: 'The suffix to add to untested images' - required: true image_name: description: 'The name of the image to tag and push' required: true last_commit_tag_exists: description: 'Whether the last commit image tag exists' required: true - common_ancestor_commit_tag_exists: - description: 'Whether the common ancestor commit image tag exists' - required: true docker_username: description: 'The username to login to Docker Hub' - required: true + required: false docker_password: description: 'The password to login to Docker Hub' - required: true + required: false outputs: - image_tagged: + image_tagged: description: 'Whether the image was tagged' value: ${{ steps.set-output.outputs.image_tagged }} @@ -42,6 +33,7 @@ runs: using: 'composite' steps: - name: Login to Docker Hub + if: ${{ github.ref == 'refs/heads/main' && inputs.last_commit_tag_exists == '0' }} uses: docker/login-action@v2 with: username: ${{ inputs.docker_username }} @@ -50,29 +42,15 @@ runs: shell: bash run: | echo IMAGE_TAGGED=false >> $GITHUB_ENV - - name: Tag Docker image with last commit tag with the commit hash plus w/o "untested" suffix - shell: bash - if: ${{ inputs.last_commit_tag != '0000000' && inputs.last_commit_tag_exists == '0' }} - run: | - docker buildx imagetools create --tag ${{ inputs.image_name }}:${{ inputs.commit_tag }} ${{ inputs.image_name }}:${{ inputs.last_commit_tag }} - docker buildx imagetools create --tag ${{ inputs.image_name }}:${{ inputs.commit_tag }}-${{ inputs.untested_tag_suffix }} ${{ inputs.image_name }}:${{ inputs.last_commit_tag }} - echo IMAGE_TAGGED=true >> $GITHUB_ENV - - name: Tag Docker image with common ancestor commit tag with the commit hash plus w/o "untested" suffix - shell: bash - if: ${{ inputs.last_commit_tag == '0000000' && inputs.common_ancestor_commit_tag_exists == '0' }} - run: | - docker buildx imagetools create --tag ${{ inputs.image_name }}:${{ inputs.commit_tag }} ${{ inputs.image_name }}:${{ inputs.common_ancestor_tag }} - docker buildx imagetools create --tag ${{ inputs.image_name }}:${{ inputs.commit_tag }}-${{ inputs.untested_tag_suffix }} ${{ inputs.image_name }}:${{ inputs.common_ancestor_tag }} - echo IMAGE_TAGGED=true >> $GITHUB_ENV - name: Tag Docker image with develop if on main branch shell: bash if: ${{ github.ref == 'refs/heads/main' && inputs.last_commit_tag_exists == '0' }} run: | docker buildx imagetools create --tag ${{ inputs.image_name }}:${{ inputs.develop_tag }} ${{ inputs.image_name }}:${{ inputs.last_commit_tag }} + docker buildx imagetools create --tag ${{ inputs.image_name }}:${{ inputs.commit_tag }} ${{ inputs.image_name }}:${{ inputs.last_commit_tag }} echo IMAGE_TAGGED=true >> $GITHUB_ENV - - name: Set output from environment variable shell: bash id: set-output run: | - echo "image_tagged=$IMAGE_TAGGED" >> $GITHUB_OUTPUT \ No newline at end of file + echo "image_tagged=$IMAGE_TAGGED" >> $GITHUB_OUTPUT diff --git a/.github/workflows/all-tools.yml b/.github/workflows/all-tools.yml index edcfc7fe1..a2405eec5 100644 --- a/.github/workflows/all-tools.yml +++ b/.github/workflows/all-tools.yml @@ -50,10 +50,7 @@ jobs: uses: ./.github/actions/check-image-tags-exist with: last_commit_tag: ${{ needs.store_image_name_and_tags.outputs.last_commit_tag }} - common_ancestor_tag: ${{ needs.store_image_name_and_tags.outputs.common_ancestor_tag }} image_name: consensys/linea-alltools - docker_username: ${{ secrets.DOCKERHUB_USERNAME }} - docker_password: ${{ secrets.DOCKERHUB_TOKEN }} all-tools-tag-only: runs-on: [self-hosted, ubuntu-20.04, X64, small] @@ -71,12 +68,9 @@ jobs: with: commit_tag: ${{ needs.store_image_name_and_tags.outputs.commit_tag }} last_commit_tag: ${{ needs.store_image_name_and_tags.outputs.last_commit_tag }} - common_ancestor_tag: ${{ needs.store_image_name_and_tags.outputs.common_ancestor_tag }} develop_tag: ${{ needs.store_image_name_and_tags.outputs.develop_tag }} - untested_tag_suffix: ${{ needs.store_image_name_and_tags.outputs.untested_tag_suffix }} image_name: consensys/linea-alltools last_commit_tag_exists: ${{ needs.check_image_tags_exist.outputs.last_commit_tag_exists }} - common_ancestor_commit_tag_exists: ${{ needs.check_image_tags_exist.outputs.common_ancestor_commit_tag_exists }} docker_username: ${{ secrets.DOCKERHUB_USERNAME }} docker_password: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/build-and-publish.yml b/.github/workflows/build-and-publish.yml index c6dad7628..b1298f8f1 100644 --- a/.github/workflows/build-and-publish.yml +++ b/.github/workflows/build-and-publish.yml @@ -6,18 +6,13 @@ on: commit_tag: required: true type: string - last_commit_tag: - required: true - type: string - common_ancestor_tag: - required: true - type: string develop_tag: required: true type: string - untested_tag_suffix: - required: true - type: string + push_image: + required: false + type: boolean + default: false coordinator_changed: required: true type: string @@ -50,9 +45,9 @@ on: type: string secrets: DOCKERHUB_USERNAME: - required: true + required: false DOCKERHUB_TOKEN: - required: true + required: false jobs: coordinator: @@ -60,11 +55,9 @@ jobs: if: ${{ always() && (inputs.coordinator_changed == 'true' || inputs.coordinator_image_tagged != 'true') }} with: commit_tag: ${{ inputs.commit_tag }} - last_commit_tag: ${{ inputs.last_commit_tag }} - common_ancestor_tag: ${{ inputs.common_ancestor_tag }} develop_tag: ${{ inputs.develop_tag }} - untested_tag_suffix: ${{ inputs.untested_tag_suffix }} image_name: consensys/linea-coordinator + push_image: ${{ inputs.push_image }} secrets: inherit prover: @@ -72,11 +65,9 @@ jobs: if: ${{ always() && (inputs.prover_changed == 'true' || inputs.prover_image_tagged != 'true') }} with: commit_tag: ${{ inputs.commit_tag }} - last_commit_tag: ${{ inputs.last_commit_tag }} - common_ancestor_tag: ${{ inputs.common_ancestor_tag }} develop_tag: ${{ inputs.develop_tag }} - untested_tag_suffix: ${{ inputs.untested_tag_suffix }} image_name: consensys/linea-prover + push_image: ${{ inputs.push_image }} secrets: inherit postman: @@ -84,11 +75,9 @@ jobs: if: ${{ always() && (inputs.postman_changed == 'true' || inputs.postman_image_tagged != 'true') }} with: commit_tag: ${{ inputs.commit_tag }} - last_commit_tag: ${{ inputs.last_commit_tag }} - common_ancestor_tag: ${{ inputs.common_ancestor_tag }} develop_tag: ${{ inputs.develop_tag }} - untested_tag_suffix: ${{ inputs.untested_tag_suffix }} image_name: consensys/linea-postman + push_image: ${{ inputs.push_image }} secrets: inherit traces-api-facade: @@ -96,11 +85,9 @@ jobs: if: ${{ always() && (inputs.traces_api_facade_changed == 'true' || inputs.traces_api_facade_image_tagged != 'true') }} with: commit_tag: ${{ inputs.commit_tag }} - last_commit_tag: ${{ inputs.last_commit_tag }} - common_ancestor_tag: ${{ inputs.common_ancestor_tag }} develop_tag: ${{ inputs.develop_tag }} - untested_tag_suffix: ${{ inputs.untested_tag_suffix }} image_name: consensys/linea-traces-api-facade + push_image: ${{ inputs.push_image }} secrets: inherit transaction_exclusion_api: @@ -108,9 +95,7 @@ jobs: if: ${{ always() && (inputs.transaction_exclusion_api_changed == 'true' || inputs.transaction_exclusion_api_image_tagged != 'true') }} with: commit_tag: ${{ inputs.commit_tag }} - last_commit_tag: ${{ inputs.last_commit_tag }} - common_ancestor_tag: ${{ inputs.common_ancestor_tag }} develop_tag: ${{ inputs.develop_tag }} - untested_tag_suffix: ${{ inputs.untested_tag_suffix }} image_name: consensys/linea-transaction-exclusion-api + push_image: ${{ inputs.push_image }} secrets: inherit diff --git a/.github/workflows/coordinator-build-and-publish.yml b/.github/workflows/coordinator-build-and-publish.yml index 02a4f4bce..240344be1 100644 --- a/.github/workflows/coordinator-build-and-publish.yml +++ b/.github/workflows/coordinator-build-and-publish.yml @@ -6,26 +6,28 @@ on: commit_tag: required: true type: string - last_commit_tag: - required: true - type: string - common_ancestor_tag: - required: true - type: string develop_tag: required: true type: string - untested_tag_suffix: - required: true - type: string image_name: required: true type: string + push_image: + required: false + type: boolean + default: false secrets: DOCKERHUB_USERNAME: required: true DOCKERHUB_TOKEN: required: true + workflow_dispatch: + inputs: + push_image: + description: 'Toggle whether to push image to docker registry' + required: false + type: boolean + default: true concurrency: group: coordinator-build-and-publish-${{ github.workflow }}-${{ github.ref }} @@ -38,10 +40,16 @@ jobs: env: COMMIT_TAG: ${{ inputs.commit_tag }} DEVELOP_TAG: ${{ inputs.develop_tag }} - UNTESTED_TAG_SUFFIX: ${{ inputs.untested_tag_suffix }} IMAGE_NAME: ${{ inputs.image_name }} - GITHUB_TOKEN: ${{ secrets._GITHUB_TOKEN_RELEASE_ACCESS }} + PUSH_IMAGE: ${{ inputs.push_image }} + TAGS: ${{ inputs.image_name }}:${{ inputs.commit_tag }} + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} steps: + - name: Set develop tag if main branch + if: ${{ github.ref == 'refs/heads/main' }} + run: | + echo "TAGS=${{ env.IMAGE_NAME }}:${{ env.COMMIT_TAG }},${{ env.IMAGE_NAME }}:${{ env.DEVELOP_TAG }}" >> $GITHUB_ENV - name: Checkout uses: actions/checkout@v4 - uses: actions/setup-java@v4 @@ -54,13 +62,14 @@ jobs: run: | ./gradlew coordinator:app:distZip --no-daemon - name: Login to Docker Hub + if: ${{ env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != '' }} uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Set up QEMU uses: docker/setup-qemu-action@v3 - - name: Set up Docker Buildx + - name: Set up Docker Buildx - local id: buildx uses: docker/setup-buildx-action@v3 - name: Docker meta @@ -68,15 +77,38 @@ jobs: uses: docker/metadata-action@v3 with: images: ${{ env.IMAGE_NAME }} + - name: Build for testing + uses: docker/build-push-action@v6 + if: ${{ env.PUSH_IMAGE == 'false' && github.event_name == 'pull_request' }} + with: + context: . + build-contexts: zip=./coordinator/app/build/distributions/ + file: ./coordinator/Dockerfile + platforms: linux/amd64 + load: true + push: false + tags: ${{ env.TAGS }} + - name: Save Docker image as artifact + if: ${{ env.PUSH_IMAGE == 'false' && github.event_name == 'pull_request' }} + run: | + docker save ${{ env.IMAGE_NAME }}:${{ env.COMMIT_TAG }} | gzip > linea-coordinator-docker-image.tar.gz + shell: bash + - name: Upload Docker image artifact + if: ${{ env.PUSH_IMAGE == 'false' && github.event_name == 'pull_request' }} + uses: actions/upload-artifact@v4 + with: + name: linea-coordinator + path: linea-coordinator-docker-image.tar.gz - name: Build & push uses: docker/build-push-action@v6 + if: ${{ env.PUSH_IMAGE == 'true' || github.event_name == 'workflow_dispatch' }} with: context: . build-contexts: zip=./coordinator/app/build/distributions/ file: ./coordinator/Dockerfile platforms: linux/amd64,linux/arm64 push: true - tags: | - ${{ env.IMAGE_NAME }}:${{ env.COMMIT_TAG }}-${{ env.UNTESTED_TAG_SUFFIX }} + tags: ${{ env.TAGS }} cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:buildcache cache-to: type=registry,ref=${{ env.IMAGE_NAME }}:buildcache,mode=max + diff --git a/.github/workflows/coordinator-testing.yml b/.github/workflows/coordinator-testing.yml index 58abc2ad4..96ad4a3b8 100644 --- a/.github/workflows/coordinator-testing.yml +++ b/.github/workflows/coordinator-testing.yml @@ -8,9 +8,9 @@ on: type: string secrets: DOCKERHUB_USERNAME: - required: true + required: false DOCKERHUB_TOKEN: - required: true + required: false concurrency: group: coordinator-testing-${{ github.workflow }}-${{ github.ref }} @@ -24,6 +24,8 @@ jobs: env: COMMIT_TAG: ${{ inputs.commit_tag }} GITHUB_TOKEN: ${{ secrets._GITHUB_TOKEN_RELEASE_ACCESS }} + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} runs-on: [self-hosted, ubuntu-22.04, X64, medium] name: Coordinator tests steps: @@ -52,6 +54,7 @@ jobs: run: | ./gradlew -V coordinator:app:buildNeeded - name: Login to Docker Hub + if: ${{ env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != '' }} uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 94241b453..d4f1ab769 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -118,9 +118,7 @@ jobs: with: commit_tag: ${{ needs.store-image-name-and-tags.outputs.commit_tag }} last_commit_tag: ${{ needs.store-image-name-and-tags.outputs.last_commit_tag }} - common_ancestor_tag: ${{ needs.store-image-name-and-tags.outputs.common_ancestor_tag }} develop_tag: ${{ needs.store-image-name-and-tags.outputs.develop_tag }} - untested_tag_suffix: ${{ needs.store-image-name-and-tags.outputs.untested_tag_suffix }} coordinator_changed: ${{ needs.filter-commit-changes.outputs.coordinator }} postman_changed: ${{ needs.filter-commit-changes.outputs.postman }} prover_changed: ${{ needs.filter-commit-changes.outputs.prover }} @@ -132,6 +130,9 @@ jobs: runs-on: [self-hosted, ubuntu-20.04, X64, small] needs: [ store-image-name-and-tags, filter-commit-changes, check-and-tag-images ] environment: ${{ github.ref != 'refs/heads/main' && 'docker-build-and-e2e' || '' }} + concurrency: + group: manual-docker-build-and-e2e-tests-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} steps: - name: Deploy environment run: | @@ -142,10 +143,7 @@ jobs: uses: ./.github/workflows/build-and-publish.yml with: commit_tag: ${{ needs.store-image-name-and-tags.outputs.commit_tag }} - last_commit_tag: ${{ needs.store-image-name-and-tags.outputs.last_commit_tag }} - common_ancestor_tag: ${{ needs.store-image-name-and-tags.outputs.common_ancestor_tag }} develop_tag: ${{ needs.store-image-name-and-tags.outputs.develop_tag }} - untested_tag_suffix: ${{ needs.store-image-name-and-tags.outputs.untested_tag_suffix }} coordinator_changed: ${{ needs.filter-commit-changes.outputs.coordinator }} postman_changed: ${{ needs.filter-commit-changes.outputs.postman }} prover_changed: ${{ needs.filter-commit-changes.outputs.prover }} @@ -193,7 +191,6 @@ jobs: uses: ./.github/workflows/reuse-run-e2e-tests.yml with: commit_tag: ${{ needs.store-image-name-and-tags.outputs.commit_tag }} - untested_tag_suffix: ${{ needs.store-image-name-and-tags.outputs.untested_tag_suffix }} tracing-engine: 'geth' e2e-tests-logs-dump: true secrets: inherit @@ -207,20 +204,28 @@ jobs: uses: ./.github/workflows/reuse-run-e2e-tests.yml with: commit_tag: ${{ needs.store-image-name-and-tags.outputs.commit_tag }} - untested_tag_suffix: ${{ needs.store-image-name-and-tags.outputs.untested_tag_suffix }} tracing-engine: 'besu' e2e-tests-logs-dump: true secrets: inherit - tag-after-run-tests-success: + publish-images-after-run-tests-success-on-main: needs: [ store-image-name-and-tags, testing, run-e2e-tests, run-e2e-tests-geth-tracing ] - if: ${{ always() && needs.testing.result == 'success' && needs.run-e2e-tests.outputs.tests_outcome == 'success' && needs.run-e2e-tests-geth-tracing.outputs.tests_outcome == 'success' }} - uses: ./.github/workflows/reuse-tag-without-untested-suffix.yml + if: ${{ always() && github.ref == 'refs/heads/main' && needs.testing.result == 'success' && needs.run-e2e-tests.outputs.tests_outcome == 'success' && needs.run-e2e-tests-geth-tracing.outputs.tests_outcome == 'success' }} + uses: ./.github/workflows/build-and-publish.yml with: + push_image: true commit_tag: ${{ needs.store-image-name-and-tags.outputs.commit_tag }} develop_tag: ${{ needs.store-image-name-and-tags.outputs.develop_tag }} - untested_tag_suffix: ${{ needs.store-image-name-and-tags.outputs.untested_tag_suffix }} - image_names: '["consensys/linea-coordinator", "consensys/linea-postman", "consensys/linea-prover", "consensys/linea-traces-api-facade", "consensys/linea-transaction-exclusion-api"]' + coordinator_changed: ${{ needs.filter-commit-changes.outputs.coordinator }} + postman_changed: ${{ needs.filter-commit-changes.outputs.postman }} + prover_changed: ${{ needs.filter-commit-changes.outputs.prover }} + traces_api_facade_changed: ${{ needs.filter-commit-changes.outputs.traces-api-facade }} + transaction_exclusion_api_changed: ${{ needs.filter-commit-changes.outputs.transaction-exclusion-api }} + coordinator_image_tagged: ${{ needs.check-and-tag-images.outputs.image_tagged_coordinator }} + postman_image_tagged: ${{ needs.check-and-tag-images.outputs.image_tagged_postman }} + prover_image_tagged: ${{ needs.check-and-tag-images.outputs.image_tagged_prover }} + traces_api_facade_image_tagged: ${{ needs.check-and-tag-images.outputs.image_tagged_traces_api_facade }} + transaction_exclusion_api_image_tagged: ${{ needs.check-and-tag-images.outputs.image_tagged_transaction_exclusion_api }} secrets: inherit cleanup-deployments: diff --git a/.github/workflows/postman-build-and-publish.yml b/.github/workflows/postman-build-and-publish.yml index ab2246112..ea021f8ba 100644 --- a/.github/workflows/postman-build-and-publish.yml +++ b/.github/workflows/postman-build-and-publish.yml @@ -6,26 +6,28 @@ on: commit_tag: required: true type: string - last_commit_tag: - required: true - type: string - common_ancestor_tag: - required: true - type: string develop_tag: required: true type: string - untested_tag_suffix: - required: true - type: string image_name: required: true type: string + push_image: + required: false + type: boolean + default: false secrets: DOCKERHUB_USERNAME: - required: true + required: false DOCKERHUB_TOKEN: - required: true + required: false + workflow_dispatch: + inputs: + push_image: + description: 'Toggle whether to push image to docker registry' + required: false + type: boolean + default: true concurrency: group: postman-build-and-publish-${{ github.workflow }}-${{ github.ref }} @@ -38,9 +40,16 @@ jobs: env: COMMIT_TAG: ${{ inputs.commit_tag }} DEVELOP_TAG: ${{ inputs.develop_tag }} - UNTESTED_TAG_SUFFIX: ${{ inputs.untested_tag_suffix }} IMAGE_NAME: ${{ inputs.image_name }} + PUSH_IMAGE: ${{ inputs.push_image }} + TAGS: ${{ inputs.image_name }}:${{ inputs.commit_tag }} + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} steps: + - name: Set develop tag if main branch + if: ${{ github.ref == 'refs/heads/main' }} + run: | + echo "TAGS=${{ env.IMAGE_NAME }}:${{ env.COMMIT_TAG }},${{ env.IMAGE_NAME }}:${{ env.DEVELOP_TAG }}" >> $GITHUB_ENV - name: Checkout uses: actions/checkout@v3 with: @@ -48,6 +57,7 @@ jobs: submodules: true persist-credentials: false - name: Login to Docker Hub + if: ${{ env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != '' }} uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} @@ -62,15 +72,38 @@ jobs: run: | echo "We inject the commit tag in the docker image ${{ env.COMMIT_TAG }}" echo COMMIT_TAG=${{ env.COMMIT_TAG }} >> $GITHUB_ENV + - name: Build postman image for testing + uses: docker/build-push-action@v6 + if: ${{ env.PUSH_IMAGE == 'false' && github.event_name == 'pull_request' }} + with: + context: ./ + file: ./sdk/Dockerfile + platforms: linux/amd64 + load: true + push: false + tags: ${{ env.TAGS }} + build-args: | + NATIVE_LIBS_RELEASE_TAG=blob-libs-v1.0.1 + - name: Save Docker image as artifact + if: ${{ env.PUSH_IMAGE == 'false' && github.event_name == 'pull_request' }} + run: | + docker save ${{ env.IMAGE_NAME }}:${{ env.COMMIT_TAG }} | gzip > linea-postman-docker-image.tar.gz + shell: bash + - name: Upload Docker image artifact + if: ${{ env.PUSH_IMAGE == 'false' && github.event_name == 'pull_request' }} + uses: actions/upload-artifact@v4 + with: + name: linea-postman + path: linea-postman-docker-image.tar.gz - name: Build and push postman image uses: docker/build-push-action@v6 + if: ${{ env.PUSH_IMAGE == 'true' || github.event_name == 'workflow_dispatch' }} with: context: ./ file: ./sdk/Dockerfile platforms: linux/amd64,linux/arm64 push: true - tags: | - ${{ env.IMAGE_NAME }}:${{ env.COMMIT_TAG }}-${{ env.UNTESTED_TAG_SUFFIX }} + tags: ${{ env.TAGS }} cache-from: | type=registry,ref=${{ env.IMAGE_NAME }}:buildcache-amd64,platform=linux/amd64 type=registry,ref=${{ env.IMAGE_NAME }}:buildcache-arm64,platform=linux/arm64 diff --git a/.github/workflows/postman-testing.yml b/.github/workflows/postman-testing.yml index 64071e2f1..a352614ef 100644 --- a/.github/workflows/postman-testing.yml +++ b/.github/workflows/postman-testing.yml @@ -2,11 +2,6 @@ name: postman-and-SDK-testing on: workflow_call: - secrets: - DOCKERHUB_USERNAME: - required: true - DOCKERHUB_TOKEN: - required: true concurrency: group: postman-testing-${{ github.workflow }}-${{ github.ref }} diff --git a/.github/workflows/prover-build-and-publish.yml b/.github/workflows/prover-build-and-publish.yml index bae07ab60..70c8cc68e 100644 --- a/.github/workflows/prover-build-and-publish.yml +++ b/.github/workflows/prover-build-and-publish.yml @@ -6,26 +6,28 @@ on: commit_tag: required: true type: string - last_commit_tag: - required: true - type: string - common_ancestor_tag: - required: true - type: string develop_tag: required: true type: string - untested_tag_suffix: - required: true - type: string image_name: required: true type: string + push_image: + required: false + type: boolean + default: false secrets: DOCKERHUB_USERNAME: - required: true + required: false DOCKERHUB_TOKEN: - required: true + required: false + workflow_dispatch: + inputs: + push_image: + description: 'Toggle whether to push image to docker registry' + required: false + type: boolean + default: true concurrency: group: prover-build-and-publish-${{ github.workflow }}-${{ github.ref }} @@ -41,9 +43,16 @@ jobs: env: COMMIT_TAG: ${{ inputs.commit_tag }} DEVELOP_TAG: ${{ inputs.develop_tag }} - UNTESTED_TAG_SUFFIX: ${{ inputs.untested_tag_suffix }} IMAGE_NAME: ${{ inputs.image_name }} + PUSH_IMAGE: ${{ inputs.push_image }} + TAGS: ${{ inputs.image_name }}:${{ inputs.commit_tag }} + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} steps: + - name: Set develop tag if main branch + if: ${{ github.ref == 'refs/heads/main' }} + run: | + echo "TAGS=${{ env.IMAGE_NAME }}:${{ env.COMMIT_TAG }},${{ env.IMAGE_NAME }}:${{ env.DEVELOP_TAG }}" >> $GITHUB_ENV - name: Checkout uses: actions/checkout@v4 with: @@ -51,6 +60,7 @@ jobs: submodules: true persist-credentials: false - name: Login to Docker Hub + if: ${{ env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != '' }} uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} @@ -63,6 +73,34 @@ jobs: echo COMMIT_TAG=${{ env.COMMIT_TAG }} >> $GITHUB_ENV - name: Build and push prover image uses: docker/build-push-action@v6 + if: ${{ env.PUSH_IMAGE == 'false' && github.event_name == 'pull_request' }} + with: + context: . + file: ./prover/Dockerfile + build-args: | + RUSTFLAGS="-C target-cpu=x86-64-v3" + build-contexts: | + prover=prover/ + corset=corset/ + constraints=constraints/ + platforms: linux/amd64 + load: true + push: false + tags: ${{ env.TAGS }} + - name: Save Docker image as artifact + if: ${{ env.PUSH_IMAGE == 'false' && github.event_name == 'pull_request' }} + run: | + docker save ${{ env.IMAGE_NAME }}:${{ env.COMMIT_TAG }} | gzip > linea-prover-docker-image.tar.gz + shell: bash + - name: Upload Docker image artifact + if: ${{ env.PUSH_IMAGE == 'false' && github.event_name == 'pull_request' }} + uses: actions/upload-artifact@v4 + with: + name: linea-prover + path: linea-prover-docker-image.tar.gz + - name: Build and push prover image + uses: docker/build-push-action@v6 + if: ${{ env.PUSH_IMAGE == 'true' || github.event_name == 'workflow_dispatch' }} with: context: . file: ./prover/Dockerfile @@ -73,8 +111,8 @@ jobs: corset=corset/ constraints=constraints/ platforms: linux/amd64 + load: false push: true - tags: | - ${{ env.IMAGE_NAME }}:${{ env.COMMIT_TAG }}-${{ env.UNTESTED_TAG_SUFFIX }} + tags: ${{ env.TAGS }} cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:buildcache cache-to: type=registry,ref=${{ env.IMAGE_NAME }}:buildcache,mode=max diff --git a/.github/workflows/reuse-check-images-tags-and-push.yml b/.github/workflows/reuse-check-images-tags-and-push.yml index 4d9ae8d22..e0e5b70d5 100644 --- a/.github/workflows/reuse-check-images-tags-and-push.yml +++ b/.github/workflows/reuse-check-images-tags-and-push.yml @@ -8,15 +8,9 @@ on: last_commit_tag: required: true type: string - common_ancestor_tag: - required: true - type: string develop_tag: required: true type: string - untested_tag_suffix: - required: true - type: string coordinator_changed: required: true type: string @@ -45,9 +39,9 @@ on: value: ${{ jobs.image_tag_push.outputs.image_tagged_transaction_exclusion_api }} secrets: DOCKERHUB_USERNAME: - required: true + required: false DOCKERHUB_TOKEN: - required: true + required: false concurrency: group: check-images-tags-and-push-${{ github.workflow }}-${{ github.ref }} @@ -78,10 +72,7 @@ jobs: id: check_image_tags_exist_coordinator with: last_commit_tag: ${{ inputs.last_commit_tag }} - common_ancestor_tag: ${{ inputs.common_ancestor_tag }} image_name: consensys/linea-coordinator - docker_username: ${{ secrets.DOCKERHUB_USERNAME }} - docker_password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Check image tags exist for postman uses: ./.github/actions/check-image-tags-exist @@ -89,10 +80,7 @@ jobs: id: check_image_tags_exist_postman with: last_commit_tag: ${{ inputs.last_commit_tag }} - common_ancestor_tag: ${{ inputs.common_ancestor_tag }} image_name: consensys/linea-postman - docker_username: ${{ secrets.DOCKERHUB_USERNAME }} - docker_password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Check image tags exist for prover uses: ./.github/actions/check-image-tags-exist @@ -100,10 +88,7 @@ jobs: id: check_image_tags_exist_prover with: last_commit_tag: ${{ inputs.last_commit_tag }} - common_ancestor_tag: ${{ inputs.common_ancestor_tag }} image_name: consensys/linea-prover - docker_username: ${{ secrets.DOCKERHUB_USERNAME }} - docker_password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Check image tags exist for traces-api-facade uses: ./.github/actions/check-image-tags-exist @@ -111,10 +96,7 @@ jobs: id: check_image_tags_exist_traces_api_facade with: last_commit_tag: ${{ inputs.last_commit_tag }} - common_ancestor_tag: ${{ inputs.common_ancestor_tag }} image_name: consensys/linea-traces-api-facade - docker_username: ${{ secrets.DOCKERHUB_USERNAME }} - docker_password: ${{ secrets.DOCKERHUB_TOKEN }} - name: Check image tags exist for transaction-exclusion-api uses: ./.github/actions/check-image-tags-exist @@ -122,13 +104,11 @@ jobs: id: check_image_tags_exist_transaction_exclusion_api with: last_commit_tag: ${{ inputs.last_commit_tag }} - common_ancestor_tag: ${{ inputs.common_ancestor_tag }} image_name: consensys/linea-transaction-exclusion-api - docker_username: ${{ secrets.DOCKERHUB_USERNAME }} - docker_password: ${{ secrets.DOCKERHUB_TOKEN }} image_tag_push: runs-on: [self-hosted, ubuntu-20.04, X64, small] + if: ${{ github.ref == 'refs/heads/main' }} name: Tag and push images needs: [ check_image_tags_exist ] outputs: @@ -148,12 +128,9 @@ jobs: with: commit_tag: ${{ inputs.commit_tag }} last_commit_tag: ${{ inputs.last_commit_tag }} - common_ancestor_tag: ${{ inputs.common_ancestor_tag }} develop_tag: ${{ inputs.develop_tag }} - untested_tag_suffix: ${{ inputs.untested_tag_suffix }} image_name: consensys/linea-coordinator last_commit_tag_exists: ${{ needs.check_image_tags_exist.outputs.last_commit_tag_exists_coordinator }} - common_ancestor_commit_tag_exists: ${{ needs.check_image_tags_exist.outputs.common_ancestor_commit_tag_exists_coordinator }} docker_username: ${{ secrets.DOCKERHUB_USERNAME }} docker_password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -164,12 +141,9 @@ jobs: with: commit_tag: ${{ inputs.commit_tag }} last_commit_tag: ${{ inputs.last_commit_tag }} - common_ancestor_tag: ${{ inputs.common_ancestor_tag }} develop_tag: ${{ inputs.develop_tag }} - untested_tag_suffix: ${{ inputs.untested_tag_suffix }} image_name: consensys/linea-postman last_commit_tag_exists: ${{ needs.check_image_tags_exist.outputs.last_commit_tag_exists_postman }} - common_ancestor_commit_tag_exists: ${{ needs.check_image_tags_exist.outputs.common_ancestor_commit_tag_exists_postman }} docker_username: ${{ secrets.DOCKERHUB_USERNAME }} docker_password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -180,12 +154,9 @@ jobs: with: commit_tag: ${{ inputs.commit_tag }} last_commit_tag: ${{ inputs.last_commit_tag }} - common_ancestor_tag: ${{ inputs.common_ancestor_tag }} develop_tag: ${{ inputs.develop_tag }} - untested_tag_suffix: ${{ inputs.untested_tag_suffix }} image_name: consensys/linea-prover last_commit_tag_exists: ${{ needs.check_image_tags_exist.outputs.last_commit_tag_exists_prover }} - common_ancestor_commit_tag_exists: ${{ needs.check_image_tags_exist.outputs.common_ancestor_commit_tag_exists_prover }} docker_username: ${{ secrets.DOCKERHUB_USERNAME }} docker_password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -196,12 +167,9 @@ jobs: with: commit_tag: ${{ inputs.commit_tag }} last_commit_tag: ${{ inputs.last_commit_tag }} - common_ancestor_tag: ${{ inputs.common_ancestor_tag }} develop_tag: ${{ inputs.develop_tag }} - untested_tag_suffix: ${{ inputs.untested_tag_suffix }} image_name: consensys/linea-traces-api-facade last_commit_tag_exists: ${{ needs.check_image_tags_exist.outputs.last_commit_tag_exists_traces_api_facade }} - common_ancestor_commit_tag_exists: ${{ needs.check_image_tags_exist.outputs.common_ancestor_commit_tag_exists_traces_api_facade }} docker_username: ${{ secrets.DOCKERHUB_USERNAME }} docker_password: ${{ secrets.DOCKERHUB_TOKEN }} @@ -212,11 +180,8 @@ jobs: with: commit_tag: ${{ inputs.commit_tag }} last_commit_tag: ${{ inputs.last_commit_tag }} - common_ancestor_tag: ${{ inputs.common_ancestor_tag }} develop_tag: ${{ inputs.develop_tag }} - untested_tag_suffix: ${{ inputs.untested_tag_suffix }} image_name: consensys/linea-transaction-exclusion-api last_commit_tag_exists: ${{ needs.check_image_tags_exist.outputs.last_commit_tag_exists_transaction_exclusion_api }} - common_ancestor_commit_tag_exists: ${{ needs.check_image_tags_exist.outputs.common_ancestor_commit_tag_exists_transaction_exclusion_api }} docker_username: ${{ secrets.DOCKERHUB_USERNAME }} docker_password: ${{ secrets.DOCKERHUB_TOKEN }} diff --git a/.github/workflows/reuse-run-e2e-tests.yml b/.github/workflows/reuse-run-e2e-tests.yml index a5cb81beb..3c1df5333 100644 --- a/.github/workflows/reuse-run-e2e-tests.yml +++ b/.github/workflows/reuse-run-e2e-tests.yml @@ -6,10 +6,6 @@ on: description: 'The commit tag to use' required: true type: string - untested_tag_suffix: - description: 'The untested tag suffix to use' - required: true - type: string tracing-engine: description: Variable option for running tests against [besu] or [geth] stack required: true @@ -34,9 +30,6 @@ on: commit_tag: required: true type: string - untested_tag_suffix: - required: true - type: string tracing-engine: description: Variable option for running tests against [besu] or [geth] stack required: true @@ -61,19 +54,21 @@ on: value: ${{ jobs.run-e2e-tests.outputs.tests_outcome }} secrets: DOCKERHUB_USERNAME: - required: true + required: false DOCKERHUB_TOKEN: - required: true + required: false jobs: run-e2e-tests: env: - COORDINATOR_TAG: ${{ inputs.commit_tag }}-${{ inputs.untested_tag_suffix }} - POSTMAN_TAG: ${{ inputs.commit_tag }}-${{ inputs.untested_tag_suffix }} - PROVER_TAG: ${{ inputs.commit_tag }}-${{ inputs.untested_tag_suffix }} - TRACES_API_TAG: ${{ inputs.commit_tag }}-${{ inputs.untested_tag_suffix }} - TRANSACTION_EXCLUSION_API_TAG: ${{ inputs.commit_tag }}-${{ inputs.untested_tag_suffix }} + COORDINATOR_TAG: ${{ inputs.commit_tag }} + POSTMAN_TAG: ${{ inputs.commit_tag }} + PROVER_TAG: ${{ inputs.commit_tag }} + TRACES_API_TAG: ${{ inputs.commit_tag }} + TRANSACTION_EXCLUSION_API_TAG: ${{ inputs.commit_tag }} GITHUB_TOKEN: ${{ secrets._GITHUB_TOKEN_RELEASE_ACCESS }} + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} outputs: tests_outcome: ${{ steps.run_e2e_tests.outcome }} runs-on: [self-hosted, ubuntu-22.04, X64, large] @@ -88,6 +83,7 @@ jobs: with: pnpm-install-options: '--frozen-lockfile --prefer-offline' - name: Login to Docker Hub + if: ${{ env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != '' }} uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} @@ -112,7 +108,17 @@ jobs: retry_wait_seconds: 30 timeout_minutes: 10 command: | - make pull-all-images + make pull-images-external-to-monorepo + - name: Download local docker image artifacts + uses: actions/download-artifact@v4 + - name: Load Docker images + run: | + gunzip -c /runner/_work/linea-monorepo/linea-monorepo/linea-coordinator/linea-coordinator-docker-image.tar.gz | docker load && + gunzip -c /runner/_work/linea-monorepo/linea-monorepo/linea-postman/linea-postman-docker-image.tar.gz | docker load && + gunzip -c /runner/_work/linea-monorepo/linea-monorepo/linea-prover/linea-prover-docker-image.tar.gz | docker load && + gunzip -c /runner/_work/linea-monorepo/linea-monorepo/linea-traces-api-facade/linea-traces-api-facade-docker-image.tar.gz | docker load && + gunzip -c /runner/_work/linea-monorepo/linea-monorepo/linea-transaction-exclusion-api/linea-transaction-exclusion-api-docker-image.tar.gz | docker load + shell: bash - name: Spin up fresh environment with geth tracing with retry if: ${{ inputs.tracing-engine == 'geth' }} uses: nick-fields/retry@v2 diff --git a/.github/workflows/reuse-store-image-name-and-tags.yml b/.github/workflows/reuse-store-image-name-and-tags.yml index bad0023fc..05a61e23e 100644 --- a/.github/workflows/reuse-store-image-name-and-tags.yml +++ b/.github/workflows/reuse-store-image-name-and-tags.yml @@ -8,10 +8,6 @@ on: value: ${{ jobs.store_image_name_and_tags.outputs.last_commit_tag }} develop_tag: value: ${{ jobs.store_image_name_and_tags.outputs.develop_tag }} - common_ancestor_tag: - value: ${{ jobs.store_image_name_and_tags.outputs.common_ancestor_tag }} - untested_tag_suffix: - value: ${{ jobs.store_image_name_and_tags.outputs.untested_tag_suffix }} concurrency: group: store-image-name-and-tags-${{ github.workflow }}-${{ github.ref }} @@ -23,35 +19,24 @@ jobs: name: Compute version tags env: # REF_NAME: ${{ github.ref_name }} - BRANCH_NAME: ${{ github.head_ref || github.ref_name }} EVENT_BEFORE: ${{ github.event.before }} outputs: - commit_tag: ${{ steps.step2.outputs.COMMIT_TAG }} - last_commit_tag: ${{ steps.step2.outputs.LAST_COMMIT_TAG }} - common_ancestor_tag: ${{ steps.step2.outputs.COMMON_ANCESTOR_TAG }} - develop_tag: ${{ steps.step2.outputs.DEVELOP_TAG }} - untested_tag_suffix: ${{ steps.step2.outputs.UNTESTED_TAG_SUFFIX }} + commit_tag: ${{ steps.step1.outputs.COMMIT_TAG }} + last_commit_tag: ${{ steps.step1.outputs.LAST_COMMIT_TAG }} + develop_tag: ${{ steps.step1.outputs.DEVELOP_TAG }} steps: - name: Checkout uses: actions/checkout@v3 - - name: Get common ancestor commit - id: step1 - run: | - git fetch --no-tags --depth=100 origin main ${{ env.BRANCH_NAME }} - echo COMMON_ANCESTOR=$(git merge-base refs/remotes/origin/main refs/remotes/origin/${{ env.BRANCH_NAME }}) >> $GITHUB_ENV - name: Compute version tags - id: step2 + id: step1 run: | echo COMMIT_TAG=$(git rev-parse --short "$GITHUB_SHA") >> $GITHUB_OUTPUT echo LAST_COMMIT_TAG=$(git rev-parse --short "${{ env.EVENT_BEFORE }}") >> $GITHUB_OUTPUT echo DEVELOP_TAG=develop >> $GITHUB_OUTPUT - echo COMMON_ANCESTOR_TAG=$(git rev-parse --short "${{ env.COMMON_ANCESTOR }}") >> $GITHUB_OUTPUT - echo UNTESTED_TAG_SUFFIX=untested >> $GITHUB_OUTPUT - name: Show version tags - id: step3 + id: step2 run: | - echo "COMMIT_TAG: ${{ steps.step2.outputs.COMMIT_TAG }}" - echo "LAST_COMMIT_TAG: ${{ steps.step2.outputs.LAST_COMMIT_TAG }}" - echo "DEVELOP_TAG: ${{ steps.step2.outputs.DEVELOP_TAG }}" - echo "COMMON_ANCESTOR_TAG: ${{ steps.step2.outputs.COMMON_ANCESTOR_TAG }}" - echo "UNTESTED_TAG_SUFFIX: ${{ steps.step2.outputs.UNTESTED_TAG_SUFFIX }}" + echo "COMMIT_TAG: ${{ steps.step1.outputs.COMMIT_TAG }}" + echo "LAST_COMMIT_TAG: ${{ steps.step1.outputs.LAST_COMMIT_TAG }}" + echo "DEVELOP_TAG: ${{ steps.step1.outputs.DEVELOP_TAG }}" + echo "github.ref: ${{ github.ref }}" diff --git a/.github/workflows/reuse-tag-without-untested-suffix.yml b/.github/workflows/reuse-tag-without-untested-suffix.yml deleted file mode 100644 index 2eb3f943f..000000000 --- a/.github/workflows/reuse-tag-without-untested-suffix.yml +++ /dev/null @@ -1,42 +0,0 @@ -name: Reusable tag without untested suffix -on: - workflow_call: - inputs: - commit_tag: - required: true - type: string - develop_tag: - required: true - type: string - untested_tag_suffix: - required: true - type: string - image_names: - required: true - type: string - secrets: - DOCKERHUB_USERNAME: - required: true - DOCKERHUB_TOKEN: - required: true - -jobs: - tag-without-untested-suffix: - runs-on: [self-hosted, ubuntu-20.04, X64, small] - name: tag without untested suffix - strategy: - matrix: - image_name: ${{ fromJSON(inputs.image_names) }} - steps: - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKERHUB_USERNAME }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Tag Docker image with develop if on main branch - if: ${{ github.ref == 'refs/heads/main' }} - run: | - docker buildx imagetools create --tag ${{ matrix.image_name }}:${{ inputs.develop_tag }} ${{ matrix.image_name }}:${{ inputs.commit_tag }}-${{ inputs.untested_tag_suffix }} - - name: Tag Docker image with the commit hash (without the "untested" suffix) - run: | - docker buildx imagetools create --tag ${{ matrix.image_name }}:${{ inputs.commit_tag }} ${{ matrix.image_name }}:${{ inputs.commit_tag }}-${{ inputs.untested_tag_suffix }} diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml index 571fc7cc8..b7c20c1c5 100644 --- a/.github/workflows/testing.yml +++ b/.github/workflows/testing.yml @@ -33,11 +33,6 @@ on: transaction_exclusion_api_image_tagged: required: true type: string - secrets: - DOCKERHUB_USERNAME: - required: true - DOCKERHUB_TOKEN: - required: true jobs: coordinator: @@ -60,9 +55,7 @@ jobs: traces-api-facade: uses: ./.github/workflows/traces-api-facade-testing.yml if: ${{ always() && (inputs.traces_api_facade_changed == 'true' || inputs.traces_api_facade_image_tagged != 'true') }} - secrets: inherit transaction-exclusion-api: uses: ./.github/workflows/transaction-exclusion-api-testing.yml if: ${{ always() && (inputs.transaction_exclusion_api_changed == 'true' || inputs.transaction_exclusion_api_image_tagged != 'true') }} - secrets: inherit diff --git a/.github/workflows/traces-api-facade-build-and-publish.yml b/.github/workflows/traces-api-facade-build-and-publish.yml index e2dd28bbd..1656594c4 100644 --- a/.github/workflows/traces-api-facade-build-and-publish.yml +++ b/.github/workflows/traces-api-facade-build-and-publish.yml @@ -6,26 +6,28 @@ on: commit_tag: required: true type: string - last_commit_tag: - required: true - type: string - common_ancestor_tag: - required: true - type: string develop_tag: required: true type: string - untested_tag_suffix: - required: true - type: string image_name: required: true type: string + push_image: + required: false + type: boolean + default: false secrets: DOCKERHUB_USERNAME: - required: true + required: false DOCKERHUB_TOKEN: - required: true + required: false + workflow_dispatch: + inputs: + push_image: + description: 'Toggle whether to push image to docker registry' + required: false + type: boolean + default: true concurrency: group: traces-api-facade-build-and-publish-${{ github.workflow }}-${{ github.ref }} @@ -38,9 +40,16 @@ jobs: env: COMMIT_TAG: ${{ inputs.commit_tag }} DEVELOP_TAG: ${{ inputs.develop_tag }} - UNTESTED_TAG_SUFFIX: ${{ inputs.untested_tag_suffix }} IMAGE_NAME: ${{ inputs.image_name }} + PUSH_IMAGE: ${{ inputs.push_image }} + TAGS: ${{ inputs.image_name }}:${{ inputs.commit_tag }} + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} steps: + - name: Set develop tag if main branch + if: ${{ github.ref == 'refs/heads/main' }} + run: | + echo "TAGS=${{ env.IMAGE_NAME }}:${{ env.COMMIT_TAG }},${{ env.IMAGE_NAME }}:${{ env.DEVELOP_TAG }}" >> $GITHUB_ENV - name: Checkout uses: actions/checkout@v3 with: @@ -57,6 +66,7 @@ jobs: ./gradlew traces-api-facade:app:shadowJar echo ${{ github.workspace }} - name: Login to Docker Hub + if: ${{ env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != '' }} uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} @@ -71,15 +81,37 @@ jobs: uses: docker/metadata-action@v3 with: images: consensys/linea-traces-api-facade + - name: Build for testing + uses: docker/build-push-action@v6 + if: ${{ env.PUSH_IMAGE == 'false' && github.event_name == 'pull_request' }} + with: + context: . + build-contexts: jar=./traces-api-facade/app/build/libs/ + file: ./traces-api-facade/Dockerfile + platforms: linux/amd64 + load: true + push: false + tags: ${{ env.TAGS }} + - name: Save Docker image as artifact + if: ${{ env.PUSH_IMAGE == 'false' && github.event_name == 'pull_request' }} + run: | + docker save ${{ env.IMAGE_NAME }}:${{ env.COMMIT_TAG }} | gzip > linea-traces-api-facade-docker-image.tar.gz + shell: bash + - name: Upload Docker image artifact + if: ${{ env.PUSH_IMAGE == 'false' && github.event_name == 'pull_request' }} + uses: actions/upload-artifact@v4 + with: + name: linea-traces-api-facade + path: linea-traces-api-facade-docker-image.tar.gz - name: Build & push uses: docker/build-push-action@v6 + if: ${{ env.PUSH_IMAGE == 'true' || github.event_name == 'workflow_dispatch' }} with: context: . build-contexts: jar=./traces-api-facade/app/build/libs/ file: ./traces-api-facade/Dockerfile platforms: linux/amd64,linux/arm64 push: true - tags: | - ${{ env.IMAGE_NAME }}:${{ env.COMMIT_TAG }}-${{ env.UNTESTED_TAG_SUFFIX }} + tags: ${{ env.TAGS }} cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:buildcache cache-to: type=registry,ref=${{ env.IMAGE_NAME }}:buildcache,mode=max diff --git a/.github/workflows/traces-api-facade-testing.yml b/.github/workflows/traces-api-facade-testing.yml index 6df13304c..d532eb15f 100644 --- a/.github/workflows/traces-api-facade-testing.yml +++ b/.github/workflows/traces-api-facade-testing.yml @@ -2,11 +2,7 @@ name: traces-api-facade-testing on: workflow_call: - secrets: - DOCKERHUB_USERNAME: - required: true - DOCKERHUB_TOKEN: - required: true + workflow_dispatch: inputs: coverage: diff --git a/.github/workflows/transaction-exclusion-api-build-and-publish.yml b/.github/workflows/transaction-exclusion-api-build-and-publish.yml index 3a1a3ce0a..fe837591e 100644 --- a/.github/workflows/transaction-exclusion-api-build-and-publish.yml +++ b/.github/workflows/transaction-exclusion-api-build-and-publish.yml @@ -6,26 +6,32 @@ on: commit_tag: required: true type: string - last_commit_tag: - required: true - type: string - common_ancestor_tag: - required: true - type: string develop_tag: required: true type: string - untested_tag_suffix: - required: true - type: string image_name: required: true type: string + push_image: + required: false + type: boolean + default: false secrets: DOCKERHUB_USERNAME: - required: true + required: false DOCKERHUB_TOKEN: - required: true + required: false + workflow_dispatch: + inputs: + push_image: + description: 'Toggle whether to push image to docker registry' + required: false + type: boolean + default: true + +concurrency: + group: transaction-exclusion-api-build-and-publish-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: build-and-publish: @@ -34,9 +40,16 @@ jobs: env: COMMIT_TAG: ${{ inputs.commit_tag }} DEVELOP_TAG: ${{ inputs.develop_tag }} - UNTESTED_TAG_SUFFIX: ${{ inputs.untested_tag_suffix }} IMAGE_NAME: ${{ inputs.image_name }} + PUSH_IMAGE: ${{ inputs.push_image }} + TAGS: ${{ inputs.image_name }}:${{ inputs.commit_tag }} + DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} + DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }} steps: + - name: Set develop tag if main branch + if: ${{ github.ref == 'refs/heads/main' }} + run: | + echo "TAGS=${{ env.IMAGE_NAME }}:${{ env.COMMIT_TAG }},${{ env.IMAGE_NAME }}:${{ env.DEVELOP_TAG }}" >> $GITHUB_ENV - name: Checkout uses: actions/checkout@v3 with: @@ -51,30 +64,53 @@ jobs: - name: Build dist run: | ./gradlew transaction-exclusion-api:app:distZip --no-daemon - - name: Set up QEMU - uses: docker/setup-qemu-action@v2 - - name: Set up Docker Buildx - id: buildx - uses: docker/setup-buildx-action@v2 - name: Login to Docker Hub - uses: docker/login-action@v1 + if: ${{ env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != '' }} + uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx - local + id: buildx + uses: docker/setup-buildx-action@v3 - name: Docker meta id: transaction-exclusion-api uses: docker/metadata-action@v3 with: - images: consensys/linea-transaction-exclusion-api + images: ${{ env.IMAGE_NAME }} + - name: Build for testing + uses: docker/build-push-action@v6 + if: ${{ env.PUSH_IMAGE == 'false' && github.event_name == 'pull_request' }} + with: + context: . + build-contexts: zip=./transaction-exclusion-api/app/build/distributions/ + file: ./transaction-exclusion-api/Dockerfile + platforms: linux/amd64 + load: true + push: false + tags: ${{ env.TAGS }} + - name: Save Docker image as artifact + if: ${{ env.PUSH_IMAGE == 'false' && github.event_name == 'pull_request' }} + run: | + docker save ${{ env.IMAGE_NAME }}:${{ env.COMMIT_TAG }} | gzip > linea-transaction-exclusion-api-docker-image.tar.gz + shell: bash + - name: Upload Docker image artifact + if: ${{ env.PUSH_IMAGE == 'false' && github.event_name == 'pull_request' }} + uses: actions/upload-artifact@v4 + with: + name: linea-transaction-exclusion-api + path: linea-transaction-exclusion-api-docker-image.tar.gz - name: Build & push uses: docker/build-push-action@v6 + if: ${{ env.PUSH_IMAGE == 'true' || github.event_name == 'workflow_dispatch' }} with: context: . build-contexts: zip=./transaction-exclusion-api/app/build/distributions/ file: ./transaction-exclusion-api/Dockerfile platforms: linux/amd64,linux/arm64 push: true - tags: | - ${{ env.IMAGE_NAME }}:${{ env.COMMIT_TAG }}-${{ env.UNTESTED_TAG_SUFFIX }} + tags: ${{ env.TAGS }} cache-from: type=registry,ref=${{ env.IMAGE_NAME }}:buildcache cache-to: type=registry,ref=${{ env.IMAGE_NAME }}:buildcache,mode=max diff --git a/.github/workflows/transaction-exclusion-api-testing.yml b/.github/workflows/transaction-exclusion-api-testing.yml index 77fd024da..cb076b35d 100644 --- a/.github/workflows/transaction-exclusion-api-testing.yml +++ b/.github/workflows/transaction-exclusion-api-testing.yml @@ -2,11 +2,6 @@ name: transaction-exclusion-api-testing on: workflow_call: - secrets: - DOCKERHUB_USERNAME: - required: true - DOCKERHUB_TOKEN: - required: true workflow_dispatch: inputs: coverage: @@ -14,6 +9,10 @@ on: required: false type: boolean default: false + +concurrency: + group: transaction-exclusion-api-testing-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} jobs: run-tests: diff --git a/Makefile b/Makefile index bca9ec1b5..cda135314 100644 --- a/Makefile +++ b/Makefile @@ -54,6 +54,9 @@ start-whole-environment-traces-v2: pull-all-images: docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile l1 --profile l2 pull +pull-images-external-to-monorepo: + docker compose -f docker/compose.yml -f docker/compose-local-dev-traces-v2.overrides.yml --profile external-to-monorepo pull + compile-contracts: cd contracts; \ make compile diff --git a/docker/compose-local-dev-traces-v2.overrides.yml b/docker/compose-local-dev-traces-v2.overrides.yml index e4397f965..8e78f6ec7 100644 --- a/docker/compose-local-dev-traces-v2.overrides.yml +++ b/docker/compose-local-dev-traces-v2.overrides.yml @@ -27,7 +27,7 @@ services: hostname: traces-node-v2 container_name: traces-node-v2 image: consensys/linea-besu:24.9-delivery32 - profiles: [ "l2", "l2-bc", "debug" ] + profiles: [ "l2", "l2-bc", "debug", "external-to-monorepo" ] depends_on: traces-node-v2-plugin-downloader: condition: service_completed_successfully diff --git a/docker/compose.yml b/docker/compose.yml index fa8bccecc..5900ac1f8 100644 --- a/docker/compose.yml +++ b/docker/compose.yml @@ -24,7 +24,7 @@ services: hostname: sequencer container_name: sequencer image: consensys/linea-besu:${SEQUENCER_TAG:-24.10-delivery34} - profiles: [ "l2", "l2-bc", "debug" ] + profiles: [ "l2", "l2-bc", "debug", "external-to-monorepo" ] depends_on: linea-besu-sequencer-plugin-downloader: condition: service_completed_successfully @@ -90,7 +90,7 @@ services: hostname: l2-node image: consensys/linea-geth:${ZKGETH_TAG:-0588665} platform: linux/amd64 - profiles: [ "l2", "l2-bc", "debug" ] + profiles: [ "l2", "l2-bc", "debug", "external-to-monorepo" ] depends_on: sequencer: condition: service_healthy @@ -123,7 +123,7 @@ services: hostname: l2-node-besu container_name: l2-node-besu image: consensys/linea-besu:24.10-delivery34 - profiles: [ "l2", "l2-bc", "debug" ] + profiles: [ "l2", "l2-bc", "debug", "external-to-monorepo" ] depends_on: sequencer: condition: service_healthy @@ -172,7 +172,7 @@ services: hostname: traces-node image: consensys/linea-geth:${ZKGETH_TAG:-0588665} platform: linux/amd64 - profiles: [ "l2", "l2-bc", "debug" ] + profiles: [ "l2", "l2-bc", "debug", "external-to-monorepo" ] depends_on: sequencer: condition: service_healthy @@ -308,7 +308,7 @@ services: transaction-exclusion-api: hostname: transaction-exclusion-api container_name: transaction-exclusion-api - image: consensys/linea-transaction-exclusion-api:${TRANSACTION_EXCLUSION_API_TAG:-d227ddf} + image: consensys/linea-transaction-exclusion-api:${TRANSACTION_EXCLUSION_API_TAG:-cd58916} profiles: [ "l2", "debug" ] restart: on-failure depends_on: @@ -367,7 +367,7 @@ services: hostname: web3signer container_name: web3signer image: consensys/web3signer:23.3-jdk17 - profiles: [ "l2", "debug" ] + profiles: [ "l2", "debug", "external-to-monorepo" ] ports: - "9000:9000" command: @@ -396,7 +396,7 @@ services: image: postgres:14.5 hostname: postgres container_name: postgres - profiles: [ "l2", "debug" ] + profiles: [ "l2", "debug", "external-to-monorepo" ] environment: POSTGRES_USER: ${POSTGRES_USER:-postgres} POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} @@ -426,7 +426,7 @@ services: container_name: l1-el-node hostname: l1-el-node image: hyperledger/besu:24.6.0 - profiles: [ "l1", "debug" ] + profiles: [ "l1", "debug", "external-to-monorepo" ] depends_on: l1-node-genesis-generator: condition: service_completed_successfully @@ -458,7 +458,7 @@ services: container_name: l1-cl-node hostname: l1-cl-node image: consensys/teku:24.2.0 - profiles: [ "l1", "debug" ] + profiles: [ "l1", "debug", "external-to-monorepo" ] depends_on: l1-el-node: condition: service_healthy @@ -484,7 +484,7 @@ services: l1-node-genesis-generator: build: context: ./config/l1-node/ - profiles: [ "l1", "debug" ] + profiles: [ "l1", "debug", "external-to-monorepo" ] entrypoint: - /bin/sh - -c @@ -532,7 +532,7 @@ services: image: consensys/linea-besu:linea-delivery-27 hostname: zkbesu-shomei container_name: zkbesu-shomei - profiles: [ "l2", "l2-bc" ] + profiles: [ "l2", "l2-bc", "external-to-monorepo" ] depends_on: shomei-plugin-downloader: condition: service_completed_successfully @@ -578,7 +578,7 @@ services: image: consensys/linea-shomei:2.2.0 hostname: shomei container_name: shomei - profiles: [ "l2", "l2-bc" ] + profiles: [ "l2", "l2-bc", "external-to-monorepo" ] depends_on: zkbesu-shomei: condition: service_started @@ -614,7 +614,7 @@ services: image: consensys/linea-shomei:2.2.0 hostname: shomei-frontend container_name: shomei-frontend - profiles: [ "l2", "l2-bc" ] + profiles: [ "l2", "l2-bc", "external-to-monorepo" ] depends_on: zkbesu-shomei: condition: service_started @@ -655,7 +655,7 @@ services: shomei-plugin-downloader: container_name: shomei-plugin-downloader - profiles: ["l2", "l2-bc"] + profiles: [ "l2", "l2-bc", "external-to-monorepo" ] image: busybox:1.36.1 command: [ "sh", "/file-downloader.sh", "https://github.com/Consensys/besu-shomei-plugin/releases/download/v0.3.1/besu-shomei-plugin-v0.3.1.jar", "/shomei-plugin" ] volumes: