[RLlib] MultiAgentEnv would NOT call env.close() on a failed sub-env (if restart_failed_sub_environments is True).

rllib/env/env_runner.py

@@ -103,7 +103,11 @@ def set_state(self, state: Dict[str, Any]) -> None:
         pass
 
     def stop(self) -> None:
-        """Releases all resources used by this EnvRunner."""
+        """Releases all resources used by this EnvRunner.
+
+        For example, when using a gym.Env in this EnvRunner, you should make sure
+        that its `close()` method is called.
+        """
         pass
 
     def __del__(self) -> None:

rllib/env/multi_agent_env.py

@@ -731,7 +731,17 @@ def try_restart(self, env_id: Optional[EnvID] = None) -> None:
         if env_id is None:
             env_id = list(range(len(self.envs)))
         for idx in env_id:
-            # Recreate the sub-env.
+            # Try closing down the old (possibly faulty) sub-env, but ignore errors.
+            try:
+                self.envs[idx].close()
+            except Exception as e:
+                if log_once("close_sub_env"):
+                    logger.warning(
+                        "Trying to close old and replaced sub-environment (at vector "
+                        f"index={idx}), but closing resulted in error:\n{e}"
+                    )
+
+            # Try recreating the sub-env.
             logger.warning(f"Trying to restart sub-environment at index {idx}.")
             self.env_states[idx].env = self.envs[idx] = self.make_env(idx)
             logger.warning(f"Sub-environment at index {idx} restarted successfully.")

rllib/env/multi_agent_env_runner.py

@@ -671,8 +671,6 @@ def assert_healthy(self):
 
     @override(EnvRunner)
     def stop(self):
-        """Closes this `EnvRunner` by running necessary closing operations."""
-
         # Note, `MultiAgentEnv` inherits `close()`-method from `gym.Env`.
         self.env.close()