Add support for real devices in AndroidWorld.

android_world/agents/random_agent.py

@@ -68,6 +68,8 @@ def _generate_random_action(
       action_details['text'] = ''.join(
           random.choices(text_characters, k=10)
       )  # Random text of length 10
+    elif action_type == json_action.SWIPE:
+      action_details['direction'] = random.choice(scroll_directions)
   elif action_type == json_action.SCROLL:
     action_details['direction'] = random.choice(scroll_directions)
 

android_world/env/android_world_controller.py

@@ -287,22 +287,39 @@ def get_controller(
     console_port: int = 5554,
     adb_path: str = DEFAULT_ADB_PATH,
     grpc_port: int = 8554,
+    real_device_name: str | None = None,
 ) -> AndroidWorldController:
   """Creates a controller by connecting to an existing Android environment."""
 
-  config = config_classes.AndroidEnvConfig(
-      task=config_classes.FilesystemTaskConfig(
-          path=_write_default_task_proto()
-      ),
-      simulator=config_classes.EmulatorConfig(
-          emulator_launcher=config_classes.EmulatorLauncherConfig(
-              emulator_console_port=console_port,
-              adb_port=console_port + 1,
-              grpc_port=grpc_port,
-          ),
-          adb_controller=config_classes.AdbControllerConfig(adb_path=adb_path),
-      ),
-  )
+  if real_device_name:
+    config = config_classes.AndroidEnvConfig(
+        task=config_classes.FilesystemTaskConfig(
+            path=_write_default_task_proto()
+        ),
+        simulator=config_classes.RealDeviceConfig(
+            device_name=real_device_name,
+            adb_controller_config=config_classes.AdbControllerConfig(
+                adb_path=adb_path,
+                device_name=real_device_name,
+            ),
+        ),
+    )
+  else:
+    config = config_classes.AndroidEnvConfig(
+        task=config_classes.FilesystemTaskConfig(
+            path=_write_default_task_proto()
+        ),
+        simulator=config_classes.EmulatorConfig(
+            emulator_launcher=config_classes.EmulatorLauncherConfig(
+                emulator_console_port=console_port,
+                adb_port=console_port + 1,
+                grpc_port=grpc_port,
+            ),
+            adb_controller=config_classes.AdbControllerConfig(
+                adb_path=adb_path
+            ),
+        ),
+    )
   android_env_instance = loader.load(config)
   logging.info('Setting up AndroidWorldController.')
   return AndroidWorldController(android_env_instance)

android_world/env/env_launcher.py

@@ -30,11 +30,14 @@
 
 
 def _get_env(
-    console_port: int, adb_path: str, grpc_port: int
+    console_port: int,
+    adb_path: str,
+    grpc_port: int,
+    real_device_name: str | None = None,
 ) -> interface.AsyncEnv:
   """Creates an AsyncEnv by connecting to an existing Android environment."""
   controller = android_world_controller.get_controller(
-      console_port, adb_path, grpc_port
+      console_port, adb_path, grpc_port, real_device_name
   )
   return interface.AsyncAndroidEnv(controller)
 
@@ -99,6 +102,7 @@ def load_and_setup_env(
     freeze_datetime: bool = True,
     adb_path: str = android_world_controller.DEFAULT_ADB_PATH,
     grpc_port: int = 8554,
+    real_device_name: str | None = None,
 ) -> interface.AsyncEnv:
   """Create environment with `get_env()` and perform env setup and validation.
 
@@ -118,10 +122,11 @@ def load_and_setup_env(
       2023, to ensure consistent benchmarking.
     adb_path: The location of the adb binary.
     grpc_port: The port for gRPC communication with the emulator.
+    real_device_name: The name of the real device to use.
 
   Returns:
     An interactable Android environment.
   """
-  env = _get_env(console_port, adb_path, grpc_port)
+  env = _get_env(console_port, adb_path, grpc_port, real_device_name)
   setup_env(env, emulator_setup, freeze_datetime)
   return env

run.py

@@ -149,6 +149,12 @@ def _find_adb_directory() -> str:
     ),
 ]
 
+_REAL_DEVICE_NAME = flags.DEFINE_string(
+    'real_device_name',
+    None,
+    'The real device name as shown in `adb devices` to run the agent on.',
+)
+
 
 def _get_agent(
     env: interface.AsyncEnv,
@@ -200,6 +206,7 @@ def _main() -> None:
       console_port=_DEVICE_CONSOLE_PORT.value,
       emulator_setup=_EMULATOR_SETUP.value,
       adb_path=_ADB_PATH.value,
+      real_device_name=_REAL_DEVICE_NAME.value,
   )
   env_launcher.verify_api_level(env)