diff --git a/tests/e2e/compare/compare.js b/tests/e2e/compare/compare.js
index 9d40ab2bf312..63d8697e4053 100644
--- a/tests/e2e/compare/compare.js
+++ b/tests/e2e/compare/compare.js
@@ -24,7 +24,7 @@ const PROBABILITY_CONSIDERED_SIGNIFICANCE = 0.02;
  * This is additional filter, in addition to probability threshold above.
  * Too small duration difference might be result of measurement grain of 1 ms.
  */
-const DURATION_DIFF_THRESHOLD_SIGNIFICANCE = 50;
+const DURATION_DIFF_THRESHOLD_SIGNIFICANCE = 100;
 
 const loadFile = (path) =>
     fs.readFile(path, 'utf8').then((data) => {
diff --git a/tests/e2e/config.js b/tests/e2e/config.js
index 6095927d0174..b386daa634b6 100644
--- a/tests/e2e/config.js
+++ b/tests/e2e/config.js
@@ -32,7 +32,7 @@ module.exports = {
     SERVER_PORT: 4723,
 
     // The amount of times a test should be executed for average performance metrics
-    RUNS: 60,
+    RUNS: 80,
 
     DEFAULT_BASELINE_BRANCH: 'main',
 
@@ -47,6 +47,9 @@ module.exports = {
     // The time in milliseconds after which an operation fails due to timeout
     INTERACTION_TIMEOUT: 300000,
 
+    // Period we wait between each test runs, to let the device cool down
+    COOL_DOWN: 90 * 1000,
+
     TEST_NAMES,
 
     /**
diff --git a/tests/e2e/testRunner.js b/tests/e2e/testRunner.js
index 00cf46a6eaea..00edec1defad 100644
--- a/tests/e2e/testRunner.js
+++ b/tests/e2e/testRunner.js
@@ -258,6 +258,15 @@ const runTests = async () => {
             }
         }
         testLog.done();
+
+        // If we still have tests add a cool down period
+        if (testIndex < numOfTests - 1) {
+            const coolDownLogs = Logger.progressInfo(`Cooling down for ${config.COOL_DOWN / 1000}s`);
+            coolDownLogs.updateText(`Cooling down for ${config.COOL_DOWN / 1000}s`);
+            // eslint-disable-next-line no-loop-func
+            await new Promise((resolve) => setTimeout(resolve, config.COOL_DOWN));
+            coolDownLogs.done();
+        }
     }
 
     // Calculate statistics and write them to our work file