diff --git a/lib/change_set.rb b/lib/change_set.rb
index 74e4bb1..9ce2ebb 100644
--- a/lib/change_set.rb
+++ b/lib/change_set.rb
@@ -9,6 +9,8 @@ def internal?
   end
 end
 
+class UnexpectedCommitMessage < StandardError; end
+
 Change = Struct.new(:dependency, :type) do
   def self.type_from_dependabot_type(dependabot_type)
     case dependabot_type
@@ -21,7 +23,7 @@ def self.type_from_dependabot_type(dependabot_type)
     else
       # As of March 2024, these are the only options Dependabot can return
       # If they add more in the future, we will need to update this
-      raise "Unrecognised update-type: #{dependabot_type}"
+      raise(UnexpectedCommitMessage, "Unrecognised update-type: #{dependabot_type}")
     end
   end
 end
@@ -47,6 +49,6 @@ def self.from_commit_message(commit_message)
       }
       .then { |changes| ChangeSet.new changes }
   rescue StandardError
-    raise "Commit message is not in the expected format"
+    raise(UnexpectedCommitMessage, "Commit message is not in the expected format")
   end
 end
diff --git a/lib/policy_manager.rb b/lib/policy_manager.rb
index 3538219..56a3b0b 100644
--- a/lib/policy_manager.rb
+++ b/lib/policy_manager.rb
@@ -58,6 +58,8 @@ def reasons_not_to_merge(pull_request)
     end
 
     reasons_not_to_merge
+  rescue UnexpectedCommitMessage => e
+    [e.message]
   end
 
   def change_allowed?(dependency_name, change_type)
diff --git a/spec/lib/change_set_spec.rb b/spec/lib/change_set_spec.rb
index e716594..21f7742 100644
--- a/spec/lib/change_set_spec.rb
+++ b/spec/lib/change_set_spec.rb
@@ -43,7 +43,7 @@
       expect(Change.type_from_dependabot_type("version-update:semver-major")).to eq(:major)
       expect(Change.type_from_dependabot_type("version-update:semver-minor")).to eq(:minor)
       expect(Change.type_from_dependabot_type("version-update:semver-patch")).to eq(:patch)
-      expect { Change.type_from_dependabot_type("foo") }.to raise_error(RuntimeError, "Unrecognised update-type: foo")
+      expect { Change.type_from_dependabot_type("foo") }.to raise_error(UnexpectedCommitMessage, "Unrecognised update-type: foo")
     end
   end
 end
@@ -124,11 +124,11 @@ def multiple_dependencies_commit
     it "raises an error if the commit message is not in the expected format" do
       expect {
         ChangeSet.from_commit_message("Hello world!")
-      }.to raise_error(RuntimeError, "Commit message is not in the expected format")
+      }.to raise_error(UnexpectedCommitMessage, "Commit message is not in the expected format")
 
       expect {
         ChangeSet.from_commit_message("foo\n---\nsyntax: error:\n...\nbar")
-      }.to raise_error(RuntimeError, "Commit message is not in the expected format")
+      }.to raise_error(UnexpectedCommitMessage, "Commit message is not in the expected format")
     end
   end
 end
diff --git a/spec/lib/policy_manager_spec.rb b/spec/lib/policy_manager_spec.rb
index b6a6849..386efe5 100644
--- a/spec/lib/policy_manager_spec.rb
+++ b/spec/lib/policy_manager_spec.rb
@@ -343,6 +343,23 @@
       ])
     end
 
+    it "should return reasons not to merge when commit message is not in the expected format" do
+      mock_pr = instance_double("PullRequest")
+      allow(mock_pr).to receive(:commit_message).and_return(
+        <<~COMMIT_MESSAGE,
+          ---
+          updated-dependencies:
+          - dependency-name: #{external_dependency}
+            dependency-type: direct:production
+        COMMIT_MESSAGE
+      )
+
+      expect(PolicyManager.new(remote_config).is_auto_mergeable?(mock_pr)).to eq(false)
+      expect(PolicyManager.new(remote_config).reasons_not_to_merge(mock_pr)).to eq([
+        "Commit message is not in the expected format",
+      ])
+    end
+
     it "should return empty array if nothing wrong" do
       mock_pr = instance_double("PullRequest")
       allow(mock_pr).to receive(:commit_message).and_return(