From f3af8a69b4aac0b1c81239b027887f34975080a9 Mon Sep 17 00:00:00 2001 From: Peter van der Zee Date: Mon, 16 Nov 2020 10:42:59 +0100 Subject: [PATCH 1/6] chore(gatsby): add try/catch to debug flaky test --- packages/gatsby/src/cache/json-file-store.ts | 62 ++++++++++++-------- 1 file changed, 36 insertions(+), 26 deletions(-) diff --git a/packages/gatsby/src/cache/json-file-store.ts b/packages/gatsby/src/cache/json-file-store.ts index af927067dc881..33169ad7f6836 100644 --- a/packages/gatsby/src/cache/json-file-store.ts +++ b/packages/gatsby/src/cache/json-file-store.ts @@ -104,32 +104,42 @@ exports.read = async function (path, options): Promise { } const externalBuffers = [] - const data = JSON.parse(dataString, function bufferReceiver(k, value) { - if (value && value.type === `Buffer` && value.data) { - return Buffer.from(value.data) - } else if ( - value && - value.type === `ExternalBuffer` && - typeof value.index === `number` && - typeof value.size === `number` - ) { - //JSON.parse is sync so we need to return a buffer sync, we will fill the buffer later - const buffer = Buffer.alloc(value.size) - externalBuffers.push({ - index: +value.index, - buffer: buffer, - }) - return buffer - } else if ( - value && - value.type === `Infinity` && - typeof value.sign === `number` - ) { - return Infinity * value.sign - } else { - return value - } - }) + let data + try { + data = JSON.parse(dataString, function bufferReceiver(k, value) { + if (value && value.type === `Buffer` && value.data) { + return Buffer.from(value.data) + } else if ( + value && + value.type === `ExternalBuffer` && + typeof value.index === `number` && + typeof value.size === `number` + ) { + //JSON.parse is sync so we need to return a buffer sync, we will fill the buffer later + const buffer = Buffer.alloc(value.size) + externalBuffers.push({ + index: +value.index, + buffer: buffer, + }) + return buffer + } else if ( + value && + value.type === `Infinity` && + typeof value.sign === `number` + ) { + return Infinity * value.sign + } else { + return value + } + }) + } catch (e) { + throw new Error( + "json-file-store failed to JSON.parse this string: `" + + dataString.replace(/\n/g, `⏎`) + + "`\n" + + e.message + ) + } //read external buffers await Promise.all( From a5dc27e7118462b717e92cde184b0f34eaccd261 Mon Sep 17 00:00:00 2001 From: Peter van der Zee Date: Tue, 17 Nov 2020 11:41:49 +0100 Subject: [PATCH 2/6] debugging more --- packages/gatsby/src/cache/cache-fs.ts | 24 ++++++++++++++++++-- packages/gatsby/src/cache/json-file-store.ts | 7 +++++- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/packages/gatsby/src/cache/cache-fs.ts b/packages/gatsby/src/cache/cache-fs.ts index f172463c5e278..9ff546bb8d8cd 100644 --- a/packages/gatsby/src/cache/cache-fs.ts +++ b/packages/gatsby/src/cache/cache-fs.ts @@ -38,6 +38,8 @@ import reporter from "gatsby-cli/lib/reporter" // we currently don't support two concurrent builds at the same time anyways. const globalGatsbyCacheLock = new Map() +global.debugging = [] + /** * construction of the disk storage * @param {object} [args] options of disk store @@ -221,6 +223,8 @@ DiskStore.prototype.reset = wrapCallback(async function (): Promise { } }) +let lockRid = 0 + /** * locks a file so other forks that want to use the same file have to wait * @param {string} filePath @@ -228,17 +232,30 @@ DiskStore.prototype.reset = wrapCallback(async function (): Promise { * @private */ DiskStore.prototype._lock = function _lock(filePath): Promise { + const rid = ++lockRid + global.debugging.push(rid + ` lock(` + filePath + `) requested`) return new Promise((resolve, reject) => innerLock(resolve, reject, filePath) ).then(() => { + global.debugging.push(rid + ` lock(` + filePath + `) received`) globalGatsbyCacheLock.set(filePath, Date.now()) }) } -function innerLock(resolve, reject, filePath): void { +function innerLock(resolve, reject, filePath, rid): void { try { let lockTime = globalGatsbyCacheLock.get(filePath) ?? 0 + global.debugging.push( + rid + + ` lock time for ` + + filePath + + ` is ` + + lockTime + + `, cond: ` + + (lockTime > 0 && Date.now() - lockTime > 10 * 1000) + ) if (lockTime > 0 && Date.now() - lockTime > 10 * 1000) { + global.debugging.push(rid + ` lock timeout for ` + filePath) reporter.verbose( `Warning: lock file older than 10s, ignoring it... There is a possibility this leads to caching problems later.` ) @@ -247,10 +264,12 @@ function innerLock(resolve, reject, filePath): void { } if (lockTime > 0) { + global.debugging.push(rid + ` waiting 50ms for ` + filePath) setTimeout(() => { - innerLock(resolve, reject, filePath) + innerLock(resolve, reject, filePath, rid) }, 50) } else { + global.debugging.push(rid + ` have lock for ` + filePath) resolve() } } catch (e) { @@ -266,6 +285,7 @@ function innerLock(resolve, reject, filePath): void { * @private */ DiskStore.prototype._unlock = function _unlock(filePath): Promise { + global.debugging.push(`lock(` + filePath + `) released`) globalGatsbyCacheLock.delete(filePath) } diff --git a/packages/gatsby/src/cache/json-file-store.ts b/packages/gatsby/src/cache/json-file-store.ts index 33169ad7f6836..2d0ad344e5a1d 100644 --- a/packages/gatsby/src/cache/json-file-store.ts +++ b/packages/gatsby/src/cache/json-file-store.ts @@ -28,6 +28,7 @@ const fs = require(`fs`) const zlib = require(`zlib`) exports.write = async function (path, data, options): Promise { + global.debugging.push(`writing to ` + path) const externalBuffers = [] let dataString = JSON.stringify(data, function replacerFunction(k, value) { //Buffers searilize to {data: [...], type: "Buffer"} @@ -84,6 +85,8 @@ exports.write = async function (path, data, options): Promise { } exports.read = async function (path, options): Promise { + global.debugging.push(`reading from ` + path) + let zipExtension = `` if (options.zip) { zipExtension = `.gz` @@ -137,7 +140,9 @@ exports.read = async function (path, options): Promise { "json-file-store failed to JSON.parse this string: `" + dataString.replace(/\n/g, `⏎`) + "`\n" + - e.message + e.message + + `\ndebugging:` + + global.debugging.map(s => `- ` + s).join(`\n`) ) } From 8c0bf2f99b9a71231072b7df0423192824cb22c6 Mon Sep 17 00:00:00 2001 From: Peter van der Zee Date: Tue, 17 Nov 2020 13:22:27 +0100 Subject: [PATCH 3/6] missing arg --- packages/gatsby/src/cache/cache-fs.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/gatsby/src/cache/cache-fs.ts b/packages/gatsby/src/cache/cache-fs.ts index 9ff546bb8d8cd..689121d9b9f24 100644 --- a/packages/gatsby/src/cache/cache-fs.ts +++ b/packages/gatsby/src/cache/cache-fs.ts @@ -235,7 +235,7 @@ DiskStore.prototype._lock = function _lock(filePath): Promise { const rid = ++lockRid global.debugging.push(rid + ` lock(` + filePath + `) requested`) return new Promise((resolve, reject) => - innerLock(resolve, reject, filePath) + innerLock(resolve, reject, filePath, rid) ).then(() => { global.debugging.push(rid + ` lock(` + filePath + `) received`) globalGatsbyCacheLock.set(filePath, Date.now()) From 86b1b302afad3d65943e57ff2ce9b2593b76a5f6 Mon Sep 17 00:00:00 2001 From: Peter van der Zee Date: Tue, 17 Nov 2020 13:26:02 +0100 Subject: [PATCH 4/6] more debug --- packages/gatsby/src/cache/cache-fs.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/gatsby/src/cache/cache-fs.ts b/packages/gatsby/src/cache/cache-fs.ts index 689121d9b9f24..70d6bfa302bdd 100644 --- a/packages/gatsby/src/cache/cache-fs.ts +++ b/packages/gatsby/src/cache/cache-fs.ts @@ -266,6 +266,7 @@ function innerLock(resolve, reject, filePath, rid): void { if (lockTime > 0) { global.debugging.push(rid + ` waiting 50ms for ` + filePath) setTimeout(() => { + global.debugging.push(rid + ` trying again for ` + filePath) innerLock(resolve, reject, filePath, rid) }, 50) } else { From 5196c7c32eb6b36637706f49b77273d54fe5b428 Mon Sep 17 00:00:00 2001 From: Peter van der Zee Date: Tue, 17 Nov 2020 13:30:32 +0100 Subject: [PATCH 5/6] set promise sync, not in a .then --- packages/gatsby/src/cache/cache-fs.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/gatsby/src/cache/cache-fs.ts b/packages/gatsby/src/cache/cache-fs.ts index 70d6bfa302bdd..89176bf0fd025 100644 --- a/packages/gatsby/src/cache/cache-fs.ts +++ b/packages/gatsby/src/cache/cache-fs.ts @@ -238,7 +238,6 @@ DiskStore.prototype._lock = function _lock(filePath): Promise { innerLock(resolve, reject, filePath, rid) ).then(() => { global.debugging.push(rid + ` lock(` + filePath + `) received`) - globalGatsbyCacheLock.set(filePath, Date.now()) }) } @@ -271,6 +270,8 @@ function innerLock(resolve, reject, filePath, rid): void { }, 50) } else { global.debugging.push(rid + ` have lock for ` + filePath) + // set sync + globalGatsbyCacheLock.set(filePath, Date.now()) resolve() } } catch (e) { From 34e79db59c8bd2d22b641c548411fddba5eb7ee7 Mon Sep 17 00:00:00 2001 From: Peter van der Zee Date: Tue, 17 Nov 2020 14:35:58 +0100 Subject: [PATCH 6/6] Remove the debugging --- packages/gatsby/src/cache/cache-fs.ts | 30 ++------------------ packages/gatsby/src/cache/json-file-store.ts | 9 +----- 2 files changed, 4 insertions(+), 35 deletions(-) diff --git a/packages/gatsby/src/cache/cache-fs.ts b/packages/gatsby/src/cache/cache-fs.ts index 89176bf0fd025..42e3e590737a1 100644 --- a/packages/gatsby/src/cache/cache-fs.ts +++ b/packages/gatsby/src/cache/cache-fs.ts @@ -38,8 +38,6 @@ import reporter from "gatsby-cli/lib/reporter" // we currently don't support two concurrent builds at the same time anyways. const globalGatsbyCacheLock = new Map() -global.debugging = [] - /** * construction of the disk storage * @param {object} [args] options of disk store @@ -223,8 +221,6 @@ DiskStore.prototype.reset = wrapCallback(async function (): Promise { } }) -let lockRid = 0 - /** * locks a file so other forks that want to use the same file have to wait * @param {string} filePath @@ -232,29 +228,13 @@ let lockRid = 0 * @private */ DiskStore.prototype._lock = function _lock(filePath): Promise { - const rid = ++lockRid - global.debugging.push(rid + ` lock(` + filePath + `) requested`) - return new Promise((resolve, reject) => - innerLock(resolve, reject, filePath, rid) - ).then(() => { - global.debugging.push(rid + ` lock(` + filePath + `) received`) - }) + return new Promise((resolve, reject) => innerLock(resolve, reject, filePath)) } -function innerLock(resolve, reject, filePath, rid): void { +function innerLock(resolve, reject, filePath): void { try { let lockTime = globalGatsbyCacheLock.get(filePath) ?? 0 - global.debugging.push( - rid + - ` lock time for ` + - filePath + - ` is ` + - lockTime + - `, cond: ` + - (lockTime > 0 && Date.now() - lockTime > 10 * 1000) - ) if (lockTime > 0 && Date.now() - lockTime > 10 * 1000) { - global.debugging.push(rid + ` lock timeout for ` + filePath) reporter.verbose( `Warning: lock file older than 10s, ignoring it... There is a possibility this leads to caching problems later.` ) @@ -263,13 +243,10 @@ function innerLock(resolve, reject, filePath, rid): void { } if (lockTime > 0) { - global.debugging.push(rid + ` waiting 50ms for ` + filePath) setTimeout(() => { - global.debugging.push(rid + ` trying again for ` + filePath) - innerLock(resolve, reject, filePath, rid) + innerLock(resolve, reject, filePath) }, 50) } else { - global.debugging.push(rid + ` have lock for ` + filePath) // set sync globalGatsbyCacheLock.set(filePath, Date.now()) resolve() @@ -287,7 +264,6 @@ function innerLock(resolve, reject, filePath, rid): void { * @private */ DiskStore.prototype._unlock = function _unlock(filePath): Promise { - global.debugging.push(`lock(` + filePath + `) released`) globalGatsbyCacheLock.delete(filePath) } diff --git a/packages/gatsby/src/cache/json-file-store.ts b/packages/gatsby/src/cache/json-file-store.ts index 2d0ad344e5a1d..d86b195c8814d 100644 --- a/packages/gatsby/src/cache/json-file-store.ts +++ b/packages/gatsby/src/cache/json-file-store.ts @@ -28,7 +28,6 @@ const fs = require(`fs`) const zlib = require(`zlib`) exports.write = async function (path, data, options): Promise { - global.debugging.push(`writing to ` + path) const externalBuffers = [] let dataString = JSON.stringify(data, function replacerFunction(k, value) { //Buffers searilize to {data: [...], type: "Buffer"} @@ -85,8 +84,6 @@ exports.write = async function (path, data, options): Promise { } exports.read = async function (path, options): Promise { - global.debugging.push(`reading from ` + path) - let zipExtension = `` if (options.zip) { zipExtension = `.gz` @@ -138,11 +135,7 @@ exports.read = async function (path, options): Promise { } catch (e) { throw new Error( "json-file-store failed to JSON.parse this string: `" + - dataString.replace(/\n/g, `⏎`) + - "`\n" + - e.message + - `\ndebugging:` + - global.debugging.map(s => `- ` + s).join(`\n`) + dataString.replace(/\n/g, `⏎`) ) }