From 00884c570d1f18cc09f4d806e5d3becfe674a5f9 Mon Sep 17 00:00:00 2001 From: tison Date: Sat, 18 May 2024 11:03:56 +0800 Subject: [PATCH] ci: report CI failures with creating issues (#3976) * ci: report CI failures with creating issues Signed-off-by: tison * integrate with CI workflows Signed-off-by: tison * mention db-approver Signed-off-by: tison --------- Signed-off-by: tison --- .github/workflows/dev-build.yml | 14 ++++- .github/workflows/nightly-build.yml | 16 +++++- .github/workflows/nightly-ci.yml | 25 ++++++++- .github/workflows/release.yml | 20 +++++-- cyborg/bin/report-ci-failure.ts | 83 +++++++++++++++++++++++++++++ 5 files changed, 149 insertions(+), 9 deletions(-) create mode 100644 cyborg/bin/report-ci-failure.ts diff --git a/.github/workflows/dev-build.yml b/.github/workflows/dev-build.yml index 6fb8ae93a712..f88202e14460 100644 --- a/.github/workflows/dev-build.yml +++ b/.github/workflows/dev-build.yml @@ -82,6 +82,9 @@ env: # The source code will check out in the following path: '${WORKING_DIR}/dev/greptime'. CHECKOUT_GREPTIMEDB_PATH: dev/greptimedb +permissions: + issues: write + jobs: allocate-runners: name: Allocate runners @@ -330,6 +333,15 @@ jobs: env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_DEVELOP_CHANNEL }} steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup-cyborg + - name: Report CI status + id: report-ci-status + working-directory: cyborg + run: pnpm tsx bin/report-ci-failure.ts + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + CI_REPORT_STATUS: ${{ needs.release-images-to-dockerhub.outputs.build-result == 'success' }} - name: Notifiy dev build successful result uses: slackapi/slack-github-action@v1.23.0 if: ${{ needs.release-images-to-dockerhub.outputs.build-result == 'success' }} @@ -342,4 +354,4 @@ jobs: if: ${{ needs.release-images-to-dockerhub.outputs.build-result != 'success' }} with: payload: | - {"text": "GreptimeDB's ${{ env.NEXT_RELEASE_VERSION }} build has failed, please check 'https://github.com/GreptimeTeam/greptimedb/actions/workflows/${{ env.NEXT_RELEASE_VERSION }}-build.yml'."} + {"text": "GreptimeDB's ${{ env.NEXT_RELEASE_VERSION }} build has failed, please check ${{ steps.report-ci-status.outputs.html_url }}."} diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index 562323ce76b5..46c472f211c9 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -66,6 +66,9 @@ env: NIGHTLY_RELEASE_PREFIX: nightly +permissions: + issues: write + jobs: allocate-runners: name: Allocate runners @@ -285,7 +288,7 @@ jobs: github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} notification: - if: ${{ always() }} # Not requiring successful dependent jobs, always run. + if: ${{ github.repository == 'GreptimeTeam/greptimedb' && always() }} # Not requiring successful dependent jobs, always run. name: Send notification to Greptime team needs: [ release-images-to-dockerhub @@ -294,6 +297,15 @@ jobs: env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_DEVELOP_CHANNEL }} steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup-cyborg + - name: Report CI status + id: report-ci-status + working-directory: cyborg + run: pnpm tsx bin/report-ci-failure.ts + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + CI_REPORT_STATUS: ${{ needs.release-images-to-dockerhub.outputs.build-result == 'success' }} - name: Notifiy nightly build successful result uses: slackapi/slack-github-action@v1.23.0 if: ${{ needs.release-images-to-dockerhub.outputs.nightly-build-result == 'success' }} @@ -306,4 +318,4 @@ jobs: if: ${{ needs.release-images-to-dockerhub.outputs.nightly-build-result != 'success' }} with: payload: | - {"text": "GreptimeDB's ${{ env.NEXT_RELEASE_VERSION }} build has failed, please check 'https://github.com/GreptimeTeam/greptimedb/actions/workflows/${{ env.NEXT_RELEASE_VERSION }}-build.yml'."} + {"text": "GreptimeDB's ${{ env.NEXT_RELEASE_VERSION }} build has failed, please check ${{ steps.report-ci-status.outputs.html_url }}."} diff --git a/.github/workflows/nightly-ci.yml b/.github/workflows/nightly-ci.yml index 601c74c64e4f..c15822f73d88 100644 --- a/.github/workflows/nightly-ci.yml +++ b/.github/workflows/nightly-ci.yml @@ -12,6 +12,9 @@ concurrency: env: RUST_TOOLCHAIN: nightly-2024-04-20 +permissions: + issues: write + jobs: sqlness-test: name: Run sqlness test @@ -39,6 +42,7 @@ jobs: timeout-minutes: 60 steps: - uses: actions/checkout@v4 + - uses: ./.github/actions/setup-cyborg - uses: arduino/setup-protoc@v3 with: repo-token: ${{ secrets.GITHUB_TOKEN }} @@ -49,6 +53,14 @@ jobs: uses: Swatinem/rust-cache@v2 - name: Run sqlness run: cargo sqlness + - name: Report CI status + if: always() + id: report-ci-status + working-directory: cyborg + run: pnpm tsx bin/report-ci-failure.ts + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + CI_REPORT_STATUS: ${{ success() }} - name: Notify slack if failed if: failure() uses: slackapi/slack-github-action@v1.23.0 @@ -56,7 +68,7 @@ jobs: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_DEVELOP_CHANNEL }} with: payload: | - {"text": "Nightly CI failed for sqlness tests"} + {"text": "Nightly CI failed for sqlness tests, please check ${{ steps.report-ci-status.outputs.html_url }}"} - name: Upload sqlness logs if: always() uses: actions/upload-artifact@v4 @@ -73,6 +85,7 @@ jobs: steps: - run: git config --global core.autocrlf false - uses: actions/checkout@v4 + - uses: ./.github/actions/setup-cyborg - uses: arduino/setup-protoc@v3 with: repo-token: ${{ secrets.GITHUB_TOKEN }} @@ -110,6 +123,14 @@ jobs: GT_S3_ACCESS_KEY: ${{ secrets.AWS_CI_TEST_SECRET_ACCESS_KEY }} GT_S3_REGION: ${{ vars.AWS_CI_TEST_BUCKET_REGION }} UNITTEST_LOG_DIR: "__unittest_logs" + - name: Report CI status + if: always() + id: report-ci-status + working-directory: cyborg + run: pnpm tsx bin/report-ci-failure.ts + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + CI_REPORT_STATUS: ${{ success() }} - name: Notify slack if failed if: failure() uses: slackapi/slack-github-action@v1.23.0 @@ -117,4 +138,4 @@ jobs: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_DEVELOP_CHANNEL }} with: payload: | - {"text": "Nightly CI failed for cargo test"} + {"text": "Nightly CI failed for cargo test, please check ${{ steps.report-ci-status.outputs.html_url }}"} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 13ac982e68f8..309f76d08c5f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -93,6 +93,9 @@ env: # Note: The NEXT_RELEASE_VERSION should be modified manually by every formal release. NEXT_RELEASE_VERSION: v0.9.0 +permissions: + issues: write + jobs: allocate-runners: name: Allocate runners @@ -245,7 +248,7 @@ jobs: - name: Set build macos result id: set-build-macos-result run: | - echo "build-macos-result=success" >> $GITHUB_OUTPUT + echo "build-macos-result=success" >> $GITHUB_OUTPUT build-windows-artifacts: name: Build Windows artifacts @@ -318,7 +321,7 @@ jobs: - name: Set build image result id: set-build-image-result run: | - echo "build-image-result=success" >> $GITHUB_OUTPUT + echo "build-image-result=success" >> $GITHUB_OUTPUT release-cn-artifacts: name: Release artifacts to CN region @@ -436,7 +439,7 @@ jobs: github-token: ${{ secrets.GH_PERSONAL_ACCESS_TOKEN }} notification: - if: ${{ always() || github.repository == 'GreptimeTeam/greptimedb' }} + if: ${{ github.repository == 'GreptimeTeam/greptimedb' && always() }} name: Send notification to Greptime team needs: [ release-images-to-dockerhub, @@ -447,6 +450,15 @@ jobs: env: SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_DEVELOP_CHANNEL }} steps: + - uses: actions/checkout@v4 + - uses: ./.github/actions/setup-cyborg + - name: Report CI status + id: report-ci-status + working-directory: cyborg + run: pnpm tsx bin/report-ci-failure.ts + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + CI_REPORT_STATUS: ${{ needs.release-images-to-dockerhub.outputs.build-image-result == 'success' && needs.build-windows-artifacts.outputs.build-windows-result == 'success' && needs.build-macos-artifacts.outputs.build-macos-result == 'success' }} - name: Notifiy release successful result uses: slackapi/slack-github-action@v1.25.0 if: ${{ needs.release-images-to-dockerhub.outputs.build-image-result == 'success' && needs.build-windows-artifacts.outputs.build-windows-result == 'success' && needs.build-macos-artifacts.outputs.build-macos-result == 'success' }} @@ -459,4 +471,4 @@ jobs: if: ${{ needs.release-images-to-dockerhub.outputs.build-image-result != 'success' || needs.build-windows-artifacts.outputs.build-windows-result != 'success' || needs.build-macos-artifacts.outputs.build-macos-result != 'success' }} with: payload: | - {"text": "GreptimeDB's release version has failed, please check 'https://github.com/GreptimeTeam/greptimedb/actions/workflows/release.yml'."} + {"text": "GreptimeDB's release version has failed, please check ${{ steps.report-ci-status.outputs.html_url }}."} diff --git a/cyborg/bin/report-ci-failure.ts b/cyborg/bin/report-ci-failure.ts new file mode 100644 index 000000000000..bc8e69b9eaa2 --- /dev/null +++ b/cyborg/bin/report-ci-failure.ts @@ -0,0 +1,83 @@ +/* + * Copyright 2023 Greptime Team + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as core from '@actions/core' +import {handleError, obtainClient} from "@/common" +import {context} from "@actions/github" +import _ from "lodash" + +async function main() { + const success = process.env["CI_REPORT_STATUS"] === "true" + core.info(`CI_REPORT_STATUS=${process.env["CI_REPORT_STATUS"]}, resolved to ${success}`) + + const client = obtainClient("GITHUB_TOKEN") + const title = `Workflow run '${context.action}' failed` + const url = `${process.env["GITHUB_SERVER_URL"]}/${process.env["GITHUB_REPOSITORY"]}/actions/runs/${process.env["GITHUB_RUN_ID"]}` + const failure_comment = `@GreptimeTeam/db-approver\nNew failure: ${url} ` + const success_comment = `@GreptimeTeam/db-approver\nBack to success: ${url}` + + const {owner, repo} = context.repo + const labels = ['O-ci-failure'] + + const issues = await client.paginate(client.rest.issues.listForRepo, { + owner, + repo, + labels: labels.join(','), + state: "open", + sort: "created", + direction: "desc", + }); + const issue = _.find(issues, (i) => i.title === title); + + if (issue) { // exist issue + core.info(`Found previous issue ${issue.html_url}`) + if (!success) { + await client.rest.issues.createComment({ + owner, + repo, + issue_number: issue.number, + body: failure_comment, + }) + } else { + await client.rest.issues.createComment({ + owner, + repo, + issue_number: issue.number, + body: success_comment, + }) + await client.rest.issues.update({ + owner, + repo, + issue_number: issue.number, + state: "closed", + state_reason: "completed", + }) + } + core.setOutput("html_url", issue.html_url) + } else if (!success) { // create new issue for failure + const issue = await client.rest.issues.create({ + owner, + repo, + title, + labels, + body: failure_comment, + }) + core.info(`Created issue ${issue.data.html_url}`) + core.setOutput("html_url", issue.data.html_url) + } +} + +main().catch(handleError)