diff --git a/.github/workflows/rerun_failed.yml b/.github/workflows/rerun_failed.yml
new file mode 100644
index 00000000000000..51c54b83fac32b
--- /dev/null
+++ b/.github/workflows/rerun_failed.yml
@@ -0,0 +1,87 @@
+# Please ping @DilumAluthge when making any changes to this file.
+
+# Here are some steps that we take in this workflow file for security reasons:
+# 1. We do not checkout any code.
+# 2. We only run actions that are defined in a repository in the `JuliaLang` GitHub organization.
+# 3. We do not give the `GITHUB_TOKEN` any permissions.
+# 4. We only give the Buildkite API token (`BUILDKITE_API_TOKEN_RETRY`) the minimum necessary
+#    set of permissions.
+
+# Important note to Buildkite maintainers:
+# In order to make this work, you need to tell Buildkite that it should NOT create a brand-new
+# build when someone closes and reopens a pull request. To do so:
+# 1. Go to the relevant pipeline (e.g. https://buildkite.com/julialang/julia-master).
+# 2. Click on the "Pipeline Settings" button.
+# 3. In the left sidebar, under "Pipeline Settings", click on "GitHub".
+# 4. In the "GitHub Settings", under "Build Pull Requests", make sure that the "Skip pull
+#    request builds for existing commits" checkbox is checked. This is the setting that tells
+#    Buildkite that it should NOT create a brand-new build when someone closes and reopens a
+#    pull request.
+# 5. At the bottom of the page, click the "Save GitHub Settings" button.
+
+name: Rerun Failed Buildkite Jobs
+
+# There are two ways that a user can rerun the failed Buildkite jobs:
+# 1. Close and reopen the pull request.
+#    In order to use this approach, the user must be in one of the following three categories:
+#        (i)   Author of the pull request
+#        (ii)  Commit permissions
+#        (iii) Triage permissions
+# 2. Post a comment on the pull request with exactly the following contents: /buildkite rerun failed
+#    In order to use this approach, the user must be in the following category:
+#        - A member of the JuliaLang GitHub organization (the membership must be publicized)
+
+on:
+  # When using the `pull_request_target` event, all PRs will get access to secret environment
+  # variables (such as the `BUILDKITE_API_TOKEN_RETRY` secret environment variable), even if
+  # the PR is from a fork. Therefore, for security reasons, we do not checkout any code in
+  # this workflow.
+  pull_request_target:
+    types: [ reopened ]
+  issue_comment:
+    types: [ created ]
+
+# We do not give the `GITHUB_TOKEN` any permissions.
+# Therefore, the `GITHUB_TOKEN` only has the same access as any member of the public.
+permissions:
+  contents: none
+
+jobs:
+  rerun-failed-buildkite-jobs:
+    name: Rerun Failed Buildkite Jobs
+    runs-on: ubuntu-latest
+    if: (github.repository == 'DilumAluthge/julia') && ((github.event_name == 'pull_request_target' && github.event.action == 'reopened') || (github.event_name == 'issue_comment' && github.event.issue.pull_request && github.event.comment.body == '/buildkite rerun failed'))
+    steps:
+      - name: Check organization membership
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          if [[ "${{ github.event_name }}" == "pull_request_target" ]]; then
+            if [[ "${{ github.event.action }}" == "reopened" ]]; then
+              echo "This is a \"reopened\" event, so we do not need to check the user's organization membership."
+              echo "GOOD_TO_PROCEED=yes" >> ${GITHUB_ENV:?}
+            else
+              echo "ERROR: The github.event_name is \"pull_request_target\", but the github.event.action is not \"reopened\"."
+              exit 1
+            fi
+          else
+            curl -H "Authorization: token ${GITHUB_TOKEN:?}" "https://api.github.com/users/${{ github.event.sender.login }}"
+            curl -H "Authorization: token ${GITHUB_TOKEN:?}" "https://api.github.com/users/${{ github.event.sender.login }}/orgs"
+            export USER_IS_ORGANIZATION_MEMBER=`curl -H "Authorization: token ${GITHUB_TOKEN:?}" "https://api.github.com/users/${{ github.event.sender.login }}/orgs" | jq '[.[] | .login] | index("JuliaLang") != null' | tr -s ' '`
+            if [[ "${USER_IS_ORGANIZATION_MEMBER:?}"   == "true" ]]; then
+              echo "The \"${{ github.event.sender.login }}\" user is a public member of the JuliaLang organization."
+              echo "GOOD_TO_PROCEED=yes" >> ${GITHUB_ENV:?}
+            else
+              echo "ERROR: the \"${{ github.event.sender.login }}\" user is NOT a public member of the JuliaLang organization."
+              echo "If you are a member, please make sure that you have publicized your membership."
+              exit 1
+            fi
+          fi
+      # For security reasons, we do not checkout any code in this workflow.
+      - uses: JuliaLang/buildkite-rerun-failed@057f6f2d37aa29a57b7679fd2af0df1d9f9188b4
+        if: env.GOOD_TO_PROCEED == 'yes'
+        with:
+          buildkite_api_token: ${{ secrets.BUILDKITE_API_TOKEN_RETRY }}
+          buildkite_organization_slug: 'julialang'
+          buildkite_pipeline_slug: 'julia-master'
+          pr_number: ${{github.event.number}}
diff --git a/.github/workflows/retry.yml b/.github/workflows/retry.yml
deleted file mode 100644
index d07eeef393ab76..00000000000000
--- a/.github/workflows/retry.yml
+++ /dev/null
@@ -1,54 +0,0 @@
-# Please ping @DilumAluthge when making any changes to this file.
-
-# Here are some steps that we take in this workflow file for security reasons:
-# 1. We do not checkout any code.
-# 2. We only run actions that are defined in a repository in the `JuliaLang` GitHub organization.
-# 3. We do not give the `GITHUB_TOKEN` any permissions.
-# 4. We only give the Buildkite API token (`BUILDKITE_API_TOKEN_RETRY`) the minimum necessary
-#    set of permissions.
-
-# Important note to Buildkite maintainers:
-# In order to make this work, you need to tell Buildkite that it should NOT create a brand-new
-# build when someone closes and reopens a pull request. To do so:
-# 1. Go to the relevant pipeline (e.g. https://buildkite.com/julialang/julia-master).
-# 2. Click on the "Pipeline Settings" button.
-# 3. In the left sidebar, under "Pipeline Settings", click on "GitHub".
-# 4. In the "GitHub Settings", under "Build Pull Requests", make sure that the "Skip pull
-#    request builds for existing commits" checkbox is checked. This is the setting that tells
-#    Buildkite that it should NOT create a brand-new build when someone closes and reopens a
-#    pull request.
-# 5. At the bottom of the page, click the "Save GitHub Settings" button.
-
-name: Retry Failed Buildkite Jobs
-
-on:
-  # When using the `pull_request_target` event, all PRs will get access to secret environment
-  # variables (such as the `BUILDKITE_API_TOKEN_RETRY` secret environment variable), even if
-  # the PR is from a fork. Therefore, for security reasons, we do not checkout any code in
-  # this workflow.
-  pull_request_target:
-    types: [ reopened ]
-
-# We do not give the `GITHUB_TOKEN` any permissions.
-permissions:
-  statuses: none
-
-jobs:
-  retry-failed-buildkite-jobs:
-    name: Retry Failed Buildkite Jobs
-    runs-on: ubuntu-latest
-
-    # TODO: delete the following line (once we have completely transitioned from Buildbot to Buildkite)
-    if: github.repository == 'JuliaLang/julia' && (github.event.label.name == 'Buildkite - retry failed jobs' || github.event.action == 'reopened')
-
-    # TODO: uncomment the following line (once we have completely transitioned from Buildbot to Buildkite)
-    # if: github.repository == 'JuliaLang/julia'
-
-    steps:
-      # For security reasons, we do not checkout any code in this workflow.
-      - uses: JuliaLang/retry-buildkite@057f6f2d37aa29a57b7679fd2af0df1d9f9188b4
-        with:
-          buildkite_api_token: ${{ secrets.BUILDKITE_API_TOKEN_RETRY }}
-          buildkite_organization_slug: 'julialang'
-          buildkite_pipeline_slug: 'julia-master'
-          pr_number: ${{github.event.number}}