From 08bc800f861bf87039af0429da95a258af79e2ad Mon Sep 17 00:00:00 2001 From: driazati Date: Wed, 3 Aug 2022 11:23:58 -0700 Subject: [PATCH 1/5] [ci][tvmbot] Enable re-run for GitHub Actions --- .github/workflows/main.yml | 7 ++- .github/workflows/tvmbot.yml | 4 +- tests/scripts/git_utils.py | 6 +- tests/scripts/github_tvmbot.py | 100 +++++++++++++++++++++++++-------- 4 files changed, 87 insertions(+), 30 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 55fe5f1441cb..077fbb7ce68a 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -36,12 +36,15 @@ concurrency: jobs: MacOS: - if: ${{ github.repository == 'apache/tvm' }} - runs-on: macOS-latest + if: ${{ github.repository == 'driazati/tvm' }} + runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 with: submodules: 'recursive' + - name: fail + run: | + exit 1 - name: Set up environment uses: ./.github/actions/setup - name: Conda Build diff --git a/.github/workflows/tvmbot.yml b/.github/workflows/tvmbot.yml index 792977f92ee5..06e4f3b80142 100644 --- a/.github/workflows/tvmbot.yml +++ b/.github/workflows/tvmbot.yml @@ -1,7 +1,6 @@ name: tvm-bot on: - status: pull_request_review: types: - submitted @@ -21,13 +20,14 @@ jobs: issues: write pull-requests: write statuses: write - if: ${{ github.event.issue.pull_request && github.repository == 'apache/tvm' }} + if: ${{ github.event.issue.pull_request && github.repository == 'driazati/tvm' }} runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - name: Run tvm-bot env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_API_TOKEN: ${{ secrets.GH_API_TOKEN }} TVM_BOT_JENKINS_TOKEN: ${{ secrets.TVM_BOT_JENKINS_TOKEN }} PR_NUMBER: ${{ github.event.issue.number }} ISSUE_COMMENT: ${{ toJson(github.event.comment) }} diff --git a/tests/scripts/git_utils.py b/tests/scripts/git_utils.py index f0d300e2f0b8..cb639178c3f9 100644 --- a/tests/scripts/git_utils.py +++ b/tests/scripts/git_utils.py @@ -89,9 +89,9 @@ def _request(self, full_url: str, body: Dict[str, Any], method: str) -> Dict[str with request.urlopen(req, data) as response: content = response.read() except error.HTTPError as e: - logging.info(f"Error response: {e.read().decode()}") - e.seek(0) - raise e + msg = str(e) + error_data = e.read().decode() + raise RuntimeError(f"Error response: {msg}\n{error_data}") logging.info(f"Got response from {full_url}: {content}") try: diff --git a/tests/scripts/github_tvmbot.py b/tests/scripts/github_tvmbot.py index e83318e18e51..f977884c72a3 100755 --- a/tests/scripts/github_tvmbot.py +++ b/tests/scripts/github_tvmbot.py @@ -37,6 +37,7 @@ EXPECTED_JOBS = ["tvm-ci/pr-head"] TVM_BOT_JENKINS_TOKEN = os.environ["TVM_BOT_JENKINS_TOKEN"] +GH_API_TOKEN = os.environ["GH_API_TOKEN"] JENKINS_URL = "https://ci.tlcpack.ai/" THANKS_MESSAGE = r"(\s*)Thanks for contributing to TVM! Please refer to guideline https://tvm.apache.org/docs/contribute/ for useful information and tips. After the pull request is submitted, please request code reviews from \[Reviewers\]\(https://github.com/apache/incubator-tvm/blob/master/CONTRIBUTORS.md#reviewers\) by them in the pull request thread.(\s*)" @@ -106,6 +107,7 @@ def to_json_str(obj: Any) -> str: nodes { ... on CheckRun { name + databaseId checkSuite { workflowRun { workflow { @@ -221,12 +223,12 @@ def checker(obj, parent_key): def __repr__(self): return json.dumps(self.raw, indent=2) - def plus_one(self, comment: Dict[str, Any]): + def react(self, comment: Dict[str, Any], content: str): """ React with a thumbs up to a comment """ url = f"issues/comments/{comment['id']}/reactions" - data = {"content": "+1"} + data = {"content": content} if self.dry_run: logging.info(f"Dry run, would have +1'ed to {url} with {data}") else: @@ -503,6 +505,27 @@ def rerun_jenkins_ci(self) -> None: else: post(url, auth=("tvm-bot", TVM_BOT_JENKINS_TOKEN)) + def rerun_github_actions(self) -> None: + job_ids = [] + for item in self.head_commit()["statusCheckRollup"]["contexts"]["nodes"]: + if "checkSuite" in item: + job_ids.append(item["databaseId"]) + + logging.info(f"Rerunning GitHub Actions jobs with IDs: {job_ids}") + actions_github = GitHubRepo( + user=self.github.user, repo=self.github.repo, token=GH_API_TOKEN + ) + for job_id in job_ids: + try: + actions_github.post(f"actions/jobs/{job_id}/rerun", data={}) + except RuntimeError as e: + # Ignore errors about jobs that are part of the same workflow to avoid + # having to figure out which jobs are in which workflows ahead of time + if "The workflow run containing this job is already running" in str(e): + pass + else: + raise e + def comment_failure(self, msg: str, exception: Exception): if not self.dry_run: exception_msg = traceback.format_exc() @@ -514,6 +537,40 @@ def comment_failure(self, msg: str, exception: Exception): return exception +def check_author(pr, triggering_comment, args): + comment_author = triggering_comment["user"]["login"] + if pr.author() == comment_author: + logging.info("Comment user is PR author, continuing") + return True + return False + + +def check_collaborator(pr, triggering_comment, args): + logging.info("Checking collaborators") + # Get the list of collaborators for the repo filtered by the comment + # author + if args.testing_collaborators_json: + collaborators = json.loads(args.testing_collaborators_json) + else: + collaborators = pr.search_collaborator(triggering_comment["user"]["login"]) + logging.info(f"Found collaborators: {collaborators}") + + return len(collaborators) > 0 + + +def check_anyone(pr, triggering_comment, args): + return True + + +AUTH_CHECKS = { + "anyone": check_anyone, + "collaborators": check_collaborator, + "author": check_author, +} +# Stash the keys so they're accessible from the values +AUTH_CHECKS = {k: (k, v) for k, v in AUTH_CHECKS.items()} + + class Merge: triggers = [ "merge", @@ -521,6 +578,8 @@ class Merge: "merge this pr", ] + auth = [AUTH_CHECKS["collaborators"], AUTH_CHECKS["author"]] + @staticmethod def run(pr: PR): info = None @@ -548,9 +607,15 @@ class Rerun: "run ci", ] + auth = [AUTH_CHECKS["anyone"]] + @staticmethod def run(pr: PR): - pr.rerun_jenkins_ci() + try: + pr.rerun_jenkins_ci() + pr.rerun_github_actions() + except Exception as e: + pr.comment_failure("Failed to re-run CI", e) if __name__ == "__main__": @@ -615,29 +680,18 @@ def run(pr: PR): else: pr = PR(number=int(args.pr), owner=owner, repo=repo, dry_run=args.dry_run) - # Acknowledge the comment with a react - pr.plus_one(comment) - - # Check the comment author - comment_author = comment["user"]["login"] - if pr.author() == comment_author: - logging.info("Comment user is PR author, continuing") - else: - logging.info("Comment is not from PR author, checking collaborators") - # Get the list of collaborators for the repo filtered by the comment - # author - if args.testing_collaborators_json: - collaborators = json.loads(args.testing_collaborators_json) - else: - collaborators = pr.search_collaborator(comment_author) - logging.info(f"Found collaborators: {collaborators}") - - if len(collaborators) > 0: - logging.info("Comment is from collaborator") + for name, check in command_to_run.auth: + if check(pr, comment, args): + logging.info(f"Passed auth check '{name}', continuing") else: - logging.info("Comment is not from from PR author or collaborator, quitting") + logging.info(f"Failed auth check '{name}', quitting") + # Add a sad face + pr.react(comment, "confused") exit(0) + # Acknowledge the comment with a react + pr.react(comment, "+1") + state = pr.state() if state != "OPEN": From 38ec51a50507f23fc1eeafb67c7f7ea623eec4ff Mon Sep 17 00:00:00 2001 From: driazati Date: Wed, 3 Aug 2022 12:37:06 -0700 Subject: [PATCH 2/5] Fix up for merging --- .github/workflows/main.yml | 7 ++---- .github/workflows/tvmbot.yml | 4 ++-- tests/scripts/github_tvmbot.py | 41 ++++++++++++++++++++++++++++------ 3 files changed, 38 insertions(+), 14 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 077fbb7ce68a..55fe5f1441cb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -36,15 +36,12 @@ concurrency: jobs: MacOS: - if: ${{ github.repository == 'driazati/tvm' }} - runs-on: ubuntu-latest + if: ${{ github.repository == 'apache/tvm' }} + runs-on: macOS-latest steps: - uses: actions/checkout@v2 with: submodules: 'recursive' - - name: fail - run: | - exit 1 - name: Set up environment uses: ./.github/actions/setup - name: Conda Build diff --git a/.github/workflows/tvmbot.yml b/.github/workflows/tvmbot.yml index 06e4f3b80142..87292ec211d1 100644 --- a/.github/workflows/tvmbot.yml +++ b/.github/workflows/tvmbot.yml @@ -20,14 +20,14 @@ jobs: issues: write pull-requests: write statuses: write - if: ${{ github.event.issue.pull_request && github.repository == 'driazati/tvm' }} + if: ${{ github.event.issue.pull_request && github.repository == 'apache/tvm' }} runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 - name: Run tvm-bot env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - GH_API_TOKEN: ${{ secrets.GH_API_TOKEN }} + GH_ACTIONS_TOKEN: ${{ secrets.GH_ACTIONS_TOKEN }} TVM_BOT_JENKINS_TOKEN: ${{ secrets.TVM_BOT_JENKINS_TOKEN }} PR_NUMBER: ${{ github.event.issue.number }} ISSUE_COMMENT: ${{ toJson(github.event.comment) }} diff --git a/tests/scripts/github_tvmbot.py b/tests/scripts/github_tvmbot.py index f977884c72a3..7f28a795dd07 100755 --- a/tests/scripts/github_tvmbot.py +++ b/tests/scripts/github_tvmbot.py @@ -37,7 +37,7 @@ EXPECTED_JOBS = ["tvm-ci/pr-head"] TVM_BOT_JENKINS_TOKEN = os.environ["TVM_BOT_JENKINS_TOKEN"] -GH_API_TOKEN = os.environ["GH_API_TOKEN"] +GH_ACTIONS_TOKEN = os.environ["GH_ACTIONS_TOKEN"] JENKINS_URL = "https://ci.tlcpack.ai/" THANKS_MESSAGE = r"(\s*)Thanks for contributing to TVM! Please refer to guideline https://tvm.apache.org/docs/contribute/ for useful information and tips. After the pull request is submitted, please request code reviews from \[Reviewers\]\(https://github.com/apache/incubator-tvm/blob/master/CONTRIBUTORS.md#reviewers\) by them in the pull request thread.(\s*)" @@ -58,6 +58,18 @@ def to_json_str(obj: Any) -> str: } """ +MENTIONABLE_QUERY = """ +query ($owner: String!, $name: String!, $user: String!) { + repository(owner: $owner, name: $name) { + mentionableUsers(query: $user, first: 1) { + nodes { + login + } + } + } +} +""" + PR_QUERY = """ query ($owner: String!, $name: String!, $number: Int!) { @@ -328,8 +340,11 @@ def search_collaborator(self, user: str) -> List[Dict[str, Any]]: """ Query GitHub for collaborators matching 'user' """ + return self.search_users(user, COLLABORATORS_QUERY)["collaborators"]["nodes"] + + def search_users(self, user: str, query: str) -> List[Dict[str, Any]]: return self.github.graphql( - query=COLLABORATORS_QUERY, + query=query, variables={ "owner": self.owner, "name": self.repo_name, @@ -337,6 +352,9 @@ def search_collaborator(self, user: str) -> List[Dict[str, Any]]: }, )["data"]["repository"]["collaborators"]["nodes"] + def search_mentionable_users(self, user: str) -> List[Dict[str, Any]]: + return self.search_users(user, MENTIONABLE_QUERY)["mentionableUsers"]["nodes"] + def comment(self, text: str) -> None: """ Leave the comment 'text' on this PR @@ -513,7 +531,7 @@ def rerun_github_actions(self) -> None: logging.info(f"Rerunning GitHub Actions jobs with IDs: {job_ids}") actions_github = GitHubRepo( - user=self.github.user, repo=self.github.repo, token=GH_API_TOKEN + user=self.github.user, repo=self.github.repo, token=GH_ACTIONS_TOKEN ) for job_id in job_ids: try: @@ -558,12 +576,21 @@ def check_collaborator(pr, triggering_comment, args): return len(collaborators) > 0 -def check_anyone(pr, triggering_comment, args): - return True +def check_mentionable_users(pr, triggering_comment, args): + logging.info("Checking mentionable users") + # Get the list of collaborators for the repo filtered by the comment + # author + if args.testing_mentionable_users_json: + mentionable_users = json.loads(args.testing_mentionable_users_json) + else: + mentionable_users = pr.search_mentionable_users(triggering_comment["user"]["login"]) + logging.info(f"Found mentionable_users: {mentionable_users}") + + return len(mentionable_users) > 0 AUTH_CHECKS = { - "anyone": check_anyone, + "metionable_users": check_mentionable_users, "collaborators": check_collaborator, "author": check_author, } @@ -607,7 +634,7 @@ class Rerun: "run ci", ] - auth = [AUTH_CHECKS["anyone"]] + auth = [AUTH_CHECKS["metionable_users"]] @staticmethod def run(pr: PR): From 51ad835ca71ff74c15133c644c871fb29f4401f0 Mon Sep 17 00:00:00 2001 From: driazati Date: Wed, 3 Aug 2022 14:07:14 -0700 Subject: [PATCH 3/5] test code --- .github/workflows/main.yml | 7 +++++-- .github/workflows/tvmbot.yml | 2 +- tests/scripts/github_tvmbot.py | 5 ++++- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 55fe5f1441cb..54d47ca6ee74 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -36,9 +36,12 @@ concurrency: jobs: MacOS: - if: ${{ github.repository == 'apache/tvm' }} - runs-on: macOS-latest + if: ${{ github.repository == 'driazati/tvm' }} + runs-on: ubuntu-latest steps: + - name: Start + run: | + exit 1 - uses: actions/checkout@v2 with: submodules: 'recursive' diff --git a/.github/workflows/tvmbot.yml b/.github/workflows/tvmbot.yml index 87292ec211d1..b54b737f7767 100644 --- a/.github/workflows/tvmbot.yml +++ b/.github/workflows/tvmbot.yml @@ -20,7 +20,7 @@ jobs: issues: write pull-requests: write statuses: write - if: ${{ github.event.issue.pull_request && github.repository == 'apache/tvm' }} + if: ${{ github.event.issue.pull_request && github.repository == 'driazati/tvm' }} runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 diff --git a/tests/scripts/github_tvmbot.py b/tests/scripts/github_tvmbot.py index 7f28a795dd07..7229f23092e8 100755 --- a/tests/scripts/github_tvmbot.py +++ b/tests/scripts/github_tvmbot.py @@ -350,7 +350,7 @@ def search_users(self, user: str, query: str) -> List[Dict[str, Any]]: "name": self.repo_name, "user": user, }, - )["data"]["repository"]["collaborators"]["nodes"] + )["data"]["repository"] def search_mentionable_users(self, user: str) -> List[Dict[str, Any]]: return self.search_users(user, MENTIONABLE_QUERY)["mentionableUsers"]["nodes"] @@ -658,6 +658,9 @@ def run(pr: PR): parser.add_argument( "--testing-collaborators-json", help="(testing only) manual data for testing" ) + parser.add_argument( + "--testing-mentionable-users-json", help="(testing only) manual data for testing" + ) parser.add_argument( "--dry-run", action="store_true", From 1ed6a8900bdb3fa9edfa5f59732f68ce51c29bfc Mon Sep 17 00:00:00 2001 From: driazati Date: Wed, 3 Aug 2022 14:14:35 -0700 Subject: [PATCH 4/5] Revert "test code" This reverts commit 51ad835ca71ff74c15133c644c871fb29f4401f0. --- .github/workflows/main.yml | 7 ++----- .github/workflows/tvmbot.yml | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 54d47ca6ee74..55fe5f1441cb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -36,12 +36,9 @@ concurrency: jobs: MacOS: - if: ${{ github.repository == 'driazati/tvm' }} - runs-on: ubuntu-latest + if: ${{ github.repository == 'apache/tvm' }} + runs-on: macOS-latest steps: - - name: Start - run: | - exit 1 - uses: actions/checkout@v2 with: submodules: 'recursive' diff --git a/.github/workflows/tvmbot.yml b/.github/workflows/tvmbot.yml index b54b737f7767..87292ec211d1 100644 --- a/.github/workflows/tvmbot.yml +++ b/.github/workflows/tvmbot.yml @@ -20,7 +20,7 @@ jobs: issues: write pull-requests: write statuses: write - if: ${{ github.event.issue.pull_request && github.repository == 'driazati/tvm' }} + if: ${{ github.event.issue.pull_request && github.repository == 'apache/tvm' }} runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v2 From 0d35fbd227a14b2e9cbe7da58814e2a4857d310c Mon Sep 17 00:00:00 2001 From: driazati Date: Wed, 3 Aug 2022 14:29:37 -0700 Subject: [PATCH 5/5] Fix tests --- .../ci/{test_mergebot.py => test_tvmbot.py} | 11 ++++--- tests/scripts/github_tvmbot.py | 33 ++++++++++--------- 2 files changed, 25 insertions(+), 19 deletions(-) rename tests/python/ci/{test_mergebot.py => test_tvmbot.py} (95%) diff --git a/tests/python/ci/test_mergebot.py b/tests/python/ci/test_tvmbot.py similarity index 95% rename from tests/python/ci/test_mergebot.py rename to tests/python/ci/test_tvmbot.py index ccdfdc653901..0b55bdaa29ee 100644 --- a/tests/python/ci/test_mergebot.py +++ b/tests/python/ci/test_tvmbot.py @@ -81,7 +81,7 @@ "invalid-author": { "number": 10786, "filename": "pr10786-invalid-author.json", - "expected": "Comment is not from from PR author or collaborator, quitting", + "expected": "Failed auth check 'collaborators', quitting", "comment": "@tvm-bot merge", "user": "not-abc", "detail": "Merge requester is not a committer and cannot merge", @@ -89,7 +89,7 @@ "unauthorized-comment": { "number": 11244, "filename": "pr11244-unauthorized-comment.json", - "expected": "Comment is not from from PR author or collaborator, quitting", + "expected": "Failed auth check 'collaborators'", "comment": "@tvm-bot merge", "user": "not-abc2", "detail": "Check that a merge comment not from a CONTRIBUTOR is rejected", @@ -135,7 +135,7 @@ [tuple(d.values()) for d in TEST_DATA.values()], ids=TEST_DATA.keys(), ) -def test_mergebot(tmpdir_factory, number, filename, expected, comment, user, detail): +def test_tvmbot(tmpdir_factory, number, filename, expected, comment, user, detail): """ Test the mergebot test cases """ @@ -156,7 +156,7 @@ def test_mergebot(tmpdir_factory, number, filename, expected, comment, user, det "login": user, }, } - collaborators = [] + collaborators = ["abc"] proc = subprocess.run( [ @@ -170,6 +170,8 @@ def test_mergebot(tmpdir_factory, number, filename, expected, comment, user, det json.dumps(test_data), "--testing-collaborators-json", json.dumps(collaborators), + "--testing-mentionable-users-json", + json.dumps(collaborators), "--trigger-comment-json", json.dumps(comment), ], @@ -178,6 +180,7 @@ def test_mergebot(tmpdir_factory, number, filename, expected, comment, user, det encoding="utf-8", env={ "TVM_BOT_JENKINS_TOKEN": "123", + "GH_ACTIONS_TOKEN": "123", }, cwd=git.cwd, check=False, diff --git a/tests/scripts/github_tvmbot.py b/tests/scripts/github_tvmbot.py index 7229f23092e8..3f7db60d3384 100755 --- a/tests/scripts/github_tvmbot.py +++ b/tests/scripts/github_tvmbot.py @@ -534,15 +534,18 @@ def rerun_github_actions(self) -> None: user=self.github.user, repo=self.github.repo, token=GH_ACTIONS_TOKEN ) for job_id in job_ids: - try: - actions_github.post(f"actions/jobs/{job_id}/rerun", data={}) - except RuntimeError as e: - # Ignore errors about jobs that are part of the same workflow to avoid - # having to figure out which jobs are in which workflows ahead of time - if "The workflow run containing this job is already running" in str(e): - pass - else: - raise e + if self.dry_run: + try: + actions_github.post(f"actions/jobs/{job_id}/rerun", data={}) + except RuntimeError as e: + # Ignore errors about jobs that are part of the same workflow to avoid + # having to figure out which jobs are in which workflows ahead of time + if "The workflow run containing this job is already running" in str(e): + pass + else: + raise e + else: + logging.info(f"Dry run, not restarting {job_id}") def comment_failure(self, msg: str, exception: Exception): if not self.dry_run: @@ -567,26 +570,26 @@ def check_collaborator(pr, triggering_comment, args): logging.info("Checking collaborators") # Get the list of collaborators for the repo filtered by the comment # author + commment_author = triggering_comment["user"]["login"] if args.testing_collaborators_json: collaborators = json.loads(args.testing_collaborators_json) else: - collaborators = pr.search_collaborator(triggering_comment["user"]["login"]) + collaborators = pr.search_collaborator(commment_author) logging.info(f"Found collaborators: {collaborators}") - return len(collaborators) > 0 + return len(collaborators) > 0 and commment_author in collaborators def check_mentionable_users(pr, triggering_comment, args): logging.info("Checking mentionable users") - # Get the list of collaborators for the repo filtered by the comment - # author + commment_author = triggering_comment["user"]["login"] if args.testing_mentionable_users_json: mentionable_users = json.loads(args.testing_mentionable_users_json) else: - mentionable_users = pr.search_mentionable_users(triggering_comment["user"]["login"]) + mentionable_users = pr.search_mentionable_users(commment_author) logging.info(f"Found mentionable_users: {mentionable_users}") - return len(mentionable_users) > 0 + return len(mentionable_users) > 0 and commment_author in mentionable_users AUTH_CHECKS = {