diff --git a/.github/actions/build-test/action.yml b/.github/actions/build-test/action.yml
deleted file mode 100644
index a3cb5f6a..00000000
--- a/.github/actions/build-test/action.yml
+++ /dev/null
@@ -1,24 +0,0 @@
-name: "Build and Test"
-runs:
-  using: "composite"
-  steps:
-    - uses: actions/setup-python@v4
-      with:
-        python-version: "3.10"
-    - name: Install Python Packages
-      shell: bash
-      run: pip install build pytest
-    - name: Build Python Package
-      shell: bash
-      run: python -m build .
-    - name: Test Python Packages
-      shell: bash
-      run: |
-        #!/bin/bash
-        for PKG in dist/*; do
-          echo "$PKG"
-          pip uninstall -y rapids-dependency-file-generator
-          pip install "$PKG"
-          pytest
-          rapids-dependency-file-generator -h # test CLI output
-        done
diff --git a/.github/actions/setup-python/action.yml b/.github/actions/setup-python/action.yml
new file mode 100644
index 00000000..96d127df
--- /dev/null
+++ b/.github/actions/setup-python/action.yml
@@ -0,0 +1,7 @@
+name: "Setup Python"
+runs:
+  using: "composite"
+  steps:
+    - uses: actions/setup-python@v4
+      with:
+        python-version: "3.10"
diff --git a/.github/workflows/prs.yaml b/.github/workflows/prs.yaml
index 8d412f10..4fc2d4ea 100644
--- a/.github/workflows/prs.yaml
+++ b/.github/workflows/prs.yaml
@@ -15,4 +15,6 @@ jobs:
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      - uses: ./.github/actions/build-test
+      - uses: ./.github/actions/setup-python
+      - name: Build & Test
+        run: ./ci/build-test.sh
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
index 23db7ab9..33b367eb 100644
--- a/.github/workflows/release.yaml
+++ b/.github/workflows/release.yaml
@@ -19,7 +19,7 @@ jobs:
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
-      - uses: ./.github/actions/build-test
+      - uses: ./.github/actions/setup-python
       - name: Setup Node.js
         uses: actions/setup-node@v3
         with:
diff --git a/.releaserc.yaml b/.releaserc.yaml
index 13549f84..4548f2a3 100644
--- a/.releaserc.yaml
+++ b/.releaserc.yaml
@@ -5,5 +5,5 @@ plugins:
   - "@semantic-release/release-notes-generator"
   - "@semantic-release/github"
   - - "@semantic-release/exec"
-    - prepareCmd: ./ci/update-version.sh ${nextRelease.version}
+    - verifyReleaseCmd: ./ci/update-version.sh ${nextRelease.version} && ./ci/build-test.sh
       publishCmd: ./ci/pypi-publish.sh
diff --git a/ci/build-test.sh b/ci/build-test.sh
new file mode 100755
index 00000000..8038f416
--- /dev/null
+++ b/ci/build-test.sh
@@ -0,0 +1,15 @@
+#!/bin/bash
+# Builds and tests Python package
+set -ue
+
+pip install build pytest
+
+python -m build .
+
+for PKG in dist/*; do
+  echo "$PKG"
+  pip uninstall -y rapids-dependency-file-generator
+  pip install "$PKG"
+  pytest
+  rapids-dependency-file-generator -h # test CLI output
+done
diff --git a/ci/pypi-publish.sh b/ci/pypi-publish.sh
old mode 100644
new mode 100755
index e1d73b83..80323433
--- a/ci/pypi-publish.sh
+++ b/ci/pypi-publish.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Simple script to upload packages to PyPI
+# Uploads packages to PyPI
 set -ue
 
 twine upload \
diff --git a/ci/update-version.sh b/ci/update-version.sh
index fdbd8870..2fb681dc 100755
--- a/ci/update-version.sh
+++ b/ci/update-version.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-# Simple script to update the version string in `_version.py`
+# Updates the version string in `_version.py`
 set -ue
 
 NEXT_VERSION=$1