diff --git a/packages/gatsby-core-utils/src/__tests__/fetch-remote-file.js b/packages/gatsby-core-utils/src/__tests__/fetch-remote-file.js
index e16a8268e2f54..57191a7ded0de 100644
--- a/packages/gatsby-core-utils/src/__tests__/fetch-remote-file.js
+++ b/packages/gatsby-core-utils/src/__tests__/fetch-remote-file.js
@@ -1,4 +1,5 @@
 // @ts-check
+
 import path from "path"
 import zlib from "zlib"
 import os from "os"
@@ -347,8 +348,6 @@ describe(`fetch-remote-file`, () => {
     jest.runAllTimers()
     await requests[0]
 
-    jest.useRealTimers()
-
     // we still expect 2 fetches because cache can't save fast enough
     expect(gotStream).toBeCalledTimes(2)
     expect(fsMove).toBeCalledTimes(1)
@@ -408,20 +407,12 @@ describe(`fetch-remote-file`, () => {
     jest.runAllTimers()
     await requests[0]
 
-    jest.useRealTimers()
-
     // we still expect 4 fetches because cache can't save fast enough
     expect(gotStream).toBeCalledTimes(4)
     expect(fsMove).toBeCalledTimes(2)
   })
 
   it(`doesn't keep lock when file download failed`, async () => {
-    jest.setTimeout(7000)
-
-    // we don't want to wait for polling to finish
-    jest.useFakeTimers()
-    jest.runAllTimers()
-
     const cacheInternals = new Map()
     const workerCache = {
       get(key) {
@@ -443,8 +434,6 @@ describe(`fetch-remote-file`, () => {
       })
     ).rejects.toThrow()
 
-    jest.runAllTimers()
-
     await expect(
       fetchRemoteFileInstanceTwo({
         url: `http://external.com/500.jpg`,
@@ -452,8 +441,6 @@ describe(`fetch-remote-file`, () => {
       })
     ).rejects.toThrow()
 
-    jest.useRealTimers()
-
     expect(gotStream).toBeCalledTimes(3)
     expect(fsMove).toBeCalledTimes(0)
   })
@@ -468,8 +455,6 @@ describe(`fetch-remote-file`, () => {
   })
 
   it(`fails when 500 is triggered`, async () => {
-    jest.setTimeout(7000)
-
     await expect(
       fetchRemoteFile({
         url: `http://external.com/500.jpg`,
@@ -529,13 +514,13 @@ Fetch details:
     })
 
     it(`Retries when server returns 503 error till server returns 200`, async () => {
-      jest.setTimeout(7000)
-
-      const filePath = await fetchRemoteFile({
+      const fetchRemoteFileInstance = fetchRemoteFile({
         url: `http://external.com/503-twice.svg`,
         cache,
       })
 
+      const filePath = await fetchRemoteFileInstance
+
       expect(path.basename(filePath)).toBe(`503-twice.svg`)
       expect(getFileSize(filePath)).resolves.toBe(
         await getFileSize(path.join(__dirname, `./fixtures/gatsby-logo.svg`))
@@ -544,8 +529,6 @@ Fetch details:
     })
 
     it(`Stops retry when maximum attempts is reached`, async () => {
-      jest.setTimeout(7000)
-
       await expect(
         fetchRemoteFile({
           url: `http://external.com/503-forever.svg`,
@@ -579,9 +562,7 @@ Fetch details:
       expect(gotStream).toBeCalledTimes(3)
     })
     // @todo retry on network errors
-    it.skip(`Retries on network errors`, async () => {
-      jest.setTimeout(7000)
-
+    it(`Retries on network errors`, async () => {
       await expect(
         fetchRemoteFile({
           url: `http://external.com/network-error.svg`,
diff --git a/packages/gatsby-core-utils/src/fetch-remote-file.ts b/packages/gatsby-core-utils/src/fetch-remote-file.ts
index c149bc755cc99..a5204a35aec8d 100644
--- a/packages/gatsby-core-utils/src/fetch-remote-file.ts
+++ b/packages/gatsby-core-utils/src/fetch-remote-file.ts
@@ -48,6 +48,9 @@ const INCOMPLETE_RETRY_LIMIT = process.env.GATSBY_INCOMPLETE_RETRY_LIMIT
   ? parseInt(process.env.GATSBY_INCOMPLETE_RETRY_LIMIT, 10)
   : 3
 
+// jest doesn't allow us to run all timings infinitely, so we set it 0  in tests
+const BACKOFF_TIME = process.env.NODE_ENV === `test` ? 0 : 1000
+
 function range(start: number, end: number): Array<number> {
   return Array(end - start)
     .fill(null)
@@ -396,16 +399,17 @@ function requestRemoteNode(
         // (error.code && ERROR_CODES_TO_RETRY.includes(error.code))
       ) {
         if (attempt < INCOMPLETE_RETRY_LIMIT) {
-          // @todo The tests don't like the delay. I tried several ways and positions.
-          new Promise(resolve => setTimeout(resolve, 1000 * attempt)).then(() =>
-            requestRemoteNode(
-              url,
-              headers,
-              tmpFilename,
-              httpOptions,
-              attempt + 1
+          setTimeout(() => {
+            resolve(
+              requestRemoteNode(
+                url,
+                headers,
+                tmpFilename,
+                httpOptions,
+                attempt + 1
+              )
             )
-          )
+          }, BACKOFF_TIME * attempt)
 
           return undefined
         } else {