diff --git a/android_world/task_evals/common_validators/sms_validators.py b/android_world/task_evals/common_validators/sms_validators.py index 92ea931..ab2e2e9 100644 --- a/android_world/task_evals/common_validators/sms_validators.py +++ b/android_world/task_evals/common_validators/sms_validators.py @@ -261,6 +261,10 @@ def is_successful(self, env: interface.AsyncEnv) -> float: ) == "com.simplemobiletools.smsmessenger" ) + if _check_if_stuck_at_sending(env): + raise ValueError( + "Message could not be sent due to Android/emulator issue." + ) return 1.0 if sms_was_sent and in_correct_app else 0.0 @classmethod @@ -272,3 +276,12 @@ def generate_random_params(cls) -> dict[str, str | int]: "number": number, "message": message, } + + +def _check_if_stuck_at_sending(env: interface.AsyncEnv) -> bool: + """Checks if the app is stuck at the sending screen.""" + state = env.get_state() + for element in state.ui_elements: + if element.text is not None and element.text.startswith("Sending"): + return True + return False