[wptrunner] Clear screenshot cache between retries (#49685)

Otherwise, the same incorrect result might be retrieved each attempt,
which defeats the retry's purpose.

Workaround for https://crbug.com/383612546

tools/wptrunner/wptrunner/environment.py

@@ -1,5 +1,6 @@
 # mypy: allow-untyped-defs
 
+import collections
 import contextlib
 import errno
 import json
@@ -114,6 +115,7 @@ def __init__(self, test_paths, testharness_timeout_multipler,
         mp_context = mpcontext.get_context()
         self._stack = contextlib.ExitStack()
         self.cache_manager = mp_context.Manager()
+        self.screenshot_caches = collections.defaultdict(self.cache_manager.dict)
         self.stash = serve.stash.StashServer(mp_context=mp_context)
         self.env_extras = env_extras
         self.env_extras_cms = None
@@ -165,6 +167,12 @@ def __exit__(self, exc_type, exc_val, exc_tb):
         self._stack.__exit__(exc_type, exc_val, exc_tb)
         self.env_extras_cms = None
 
+    def reset(self):
+        """Reset state between retry attempts to isolate failures."""
+        for cache in self.screenshot_caches.values():
+            cache.clear()
+        # TODO: Clear the stash.
+
     @contextlib.contextmanager
     def ignore_interrupts(self):
         prev_handler = signal.signal(signal.SIGINT, signal.SIG_IGN)

tools/wptrunner/wptrunner/executors/base.py

@@ -34,7 +34,8 @@ def executor_kwargs(test_type, test_environment, run_info_data, subsuite, **kwar
                        "target_platform": run_info_data["os"]}
 
     if test_type in ("reftest", "print-reftest"):
-        executor_kwargs["screenshot_cache"] = test_environment.cache_manager.dict()
+        screenshot_cache = test_environment.screenshot_caches[test_type, subsuite.name]
+        executor_kwargs["screenshot_cache"] = screenshot_cache
         executor_kwargs["reftest_screenshot"] = kwargs["reftest_screenshot"]
 
     if test_type == "wdspec":
@@ -417,7 +418,7 @@ def logger(self):
         return self.executor.logger
 
     def get_hash(self, test, viewport_size, dpi, page_ranges):
-        key = (self.subsuite, test.url, viewport_size, dpi)
+        key = (test.url, viewport_size, dpi)
 
         if key not in self.screenshot_cache:
             success, data = self.get_screenshot_list(test, viewport_size, dpi, page_ranges)

tools/wptrunner/wptrunner/wptrunner.py

@@ -299,6 +299,7 @@ def run_test_iteration(test_status, test_loader, test_queue_builder,
                             test_loader.subsuites,
                             kwargs["run_by_dir"])
 
+        test_environment.reset()
         with ManagerGroup("web-platform-tests",
                           test_queue_builder,
                           test_implementations,