diff --git a/evalml/tests/automl_tests/test_automl.py b/evalml/tests/automl_tests/test_automl.py
index 359bfa01b3..155d131fc2 100644
--- a/evalml/tests/automl_tests/test_automl.py
+++ b/evalml/tests/automl_tests/test_automl.py
@@ -1530,9 +1530,12 @@ def test_results_getter(AutoMLTestEnv, X_y_binary):
 
     assert automl.results["pipeline_results"][0]["mean_cv_score"] == 1.0
 
-    with pytest.raises(AttributeError, match="set attribute"):
+    with pytest.raises(AttributeError) as atr_error:
         automl.results = 2.0
 
+    assert "set attribute" in str(atr_error.value) or "has no setter" in str(
+        atr_error.value,
+    )
     automl.results["pipeline_results"][0]["mean_cv_score"] = 2.0
     assert automl.results["pipeline_results"][0]["mean_cv_score"] == 1.0