-
Notifications
You must be signed in to change notification settings - Fork 27.2k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Memory] Add option to reduce memory usage caused by duplicate string…
…s in webpack-sources (#66003) This PR adds a flag to Next.js to enable Webpack options to improve memory usage. See webpack/webpack-sources#155 for a full description of the changes and impact on memory. This PR adds a patch to `webpack-sources` temporarily that contains the fixes as the real changes are iterated on to merge upstream in the `webpack/webpack-sources` repository. After that is done, the patch will be reverted and the latest `webpack-sources` version will be updated in Next.js.
- Loading branch information
1 parent
6a469cc
commit 79ec837
Showing
8 changed files
with
264 additions
and
5 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Large diffs are not rendered by default.
Oops, something went wrong.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,218 @@ | ||
| diff --git a/lib/RawSource.js b/lib/RawSource.js | ||
| index 098d317..06e6b8f 100644 | ||
| --- a/lib/RawSource.js | ||
| +++ b/lib/RawSource.js | ||
| @@ -6,6 +6,10 @@ | ||
| "use strict"; | ||
|
|
||
| const streamChunksOfRawSource = require("./helpers/streamChunksOfRawSource"); | ||
| +const { | ||
| + internString, | ||
| + isDualStringBufferCachingEnabled | ||
| +} = require("./helpers/stringBufferUtils"); | ||
| const Source = require("./Source"); | ||
|
|
||
| class RawSource extends Source { | ||
| @@ -17,8 +21,13 @@ class RawSource extends Source { | ||
| } | ||
| this._valueIsBuffer = !convertToString && isBuffer; | ||
| - this._value = convertToString && isBuffer ? undefined : value; | ||
| + this._value = | ||
| + convertToString && isBuffer | ||
| + ? undefined | ||
| + : typeof value === "string" | ||
| + ? internString(value) | ||
| + : value; | ||
| this._valueAsBuffer = isBuffer ? value : undefined; | ||
| - this._valueAsString = isBuffer ? undefined : value; | ||
| + this._valueAsString = isBuffer ? undefined : internString(value); | ||
| } | ||
|
|
||
| isBuffer() { | ||
| @@ -27,14 +36,22 @@ class RawSource extends Source { | ||
|
|
||
| source() { | ||
| if (this._value === undefined) { | ||
| - this._value = this._valueAsBuffer.toString("utf-8"); | ||
| + const value = internString(this._valueAsBuffer.toString("utf-8")); | ||
| + if (isDualStringBufferCachingEnabled()) { | ||
| + this._value = value; | ||
| + } | ||
| + return value; | ||
| } | ||
| return this._value; | ||
| } | ||
|
|
||
| buffer() { | ||
| if (this._valueAsBuffer === undefined) { | ||
| - this._valueAsBuffer = Buffer.from(this._value, "utf-8"); | ||
| + const value = Buffer.from(this._value, "utf-8"); | ||
| + if (isDualStringBufferCachingEnabled()) { | ||
| + this._valueAsBuffer = value; | ||
| + } | ||
| + return value; | ||
| } | ||
| return this._valueAsBuffer; | ||
| } | ||
| @@ -51,17 +68,21 @@ class RawSource extends Source { | ||
| * @returns {void} | ||
| */ | ||
| streamChunks(options, onChunk, onSource, onName) { | ||
| - if (this._value === undefined) { | ||
| + if (this._value === undefined && isDualStringBufferCachingEnabled()) { | ||
| this._value = Buffer.from(this._valueAsBuffer, "utf-8"); | ||
| } | ||
| - if (this._valueAsString === undefined) { | ||
| - this._valueAsString = | ||
| + let strValue = this._valueAsString; | ||
| + if (strValue === undefined) { | ||
| + strValue = | ||
| typeof this._value === "string" | ||
| ? this._value | ||
| - : this._value.toString("utf-8"); | ||
| + : internString(this._value.toString("utf-8")); | ||
| + if (isDualStringBufferCachingEnabled()) { | ||
| + this._valueAsString = strValue; | ||
| + } | ||
| } | ||
| return streamChunksOfRawSource( | ||
| - this._valueAsString, | ||
| + strValue, | ||
| onChunk, | ||
| onSource, | ||
| onName, | ||
| @@ -70,11 +91,8 @@ class RawSource extends Source { | ||
| } | ||
|
|
||
| updateHash(hash) { | ||
| - if (this._valueAsBuffer === undefined) { | ||
| - this._valueAsBuffer = Buffer.from(this._value, "utf-8"); | ||
| - } | ||
| hash.update("RawSource"); | ||
| - hash.update(this._valueAsBuffer); | ||
| + hash.update(this.buffer()); | ||
| } | ||
| } | ||
|
|
||
| diff --git a/lib/helpers/stringBufferUtils.js b/lib/helpers/stringBufferUtils.js | ||
| new file mode 100644 | ||
| index 0000000..3b210f1 | ||
| --- /dev/null | ||
| +++ b/lib/helpers/stringBufferUtils.js | ||
| @@ -0,0 +1,107 @@ | ||
| +/* | ||
| + MIT License http://www.opensource.org/licenses/mit-license.php | ||
| + Author Mark Knichel @mknichel | ||
| +*/ | ||
| + | ||
| +"use strict"; | ||
| + | ||
| +let dualStringBufferCaching = true; | ||
| + | ||
| +/** | ||
| + * @returns {boolean} Whether the optimization to cache copies of both the | ||
| + * string and buffer version of source content is enabled. This is enabled by | ||
| + * default to improve performance but can consume more memory since values are | ||
| + * stored twice. | ||
| + */ | ||
| +function isDualStringBufferCachingEnabled() { | ||
| + return dualStringBufferCaching; | ||
| +} | ||
| + | ||
| +/** | ||
| + * Enables an optimization to save both string and buffer in memory to avoid | ||
| + * repeat conversions between the two formats when they are requested. This | ||
| + * is enabled by default. This option can improve performance but can consume | ||
| + * additional memory since values are stored twice. | ||
| + * | ||
| + * @returns {void} | ||
| + */ | ||
| +function enableDualStringBufferCaching() { | ||
| + dualStringBufferCaching = true; | ||
| +} | ||
| + | ||
| +/** | ||
| + * Disables the optimization to save both string and buffer in memory. This | ||
| + * may increase performance but should reduce memory usage in the Webpack | ||
| + * compiler. | ||
| + * | ||
| + * @returns {void} | ||
| + */ | ||
| +function disableDualStringBufferCaching() { | ||
| + dualStringBufferCaching = false; | ||
| +} | ||
| + | ||
| +const interningStringMap = new Map(); | ||
| + | ||
| +/** | ||
| + * Saves the string in a map to ensure that only one copy of the string exists | ||
| + * in memory at a given time. This is controlled by {@link enableStringInterning} | ||
| + * and {@link disableStringInterning}. Callers are expect to manage the memory | ||
| + * of the interned strings by calling {@link disableStringInterning} after the | ||
| + * compiler no longer needs to save the interned memory. | ||
| + * | ||
| + * @param {string} str A string to be interned. | ||
| + * @returns {string} The original string or a reference to an existing string | ||
| + * of the same value if it has already been interned. | ||
| + */ | ||
| +function internString(str) { | ||
| + if (!isStringInterningEnabled() || !str || typeof str !== "string") { | ||
| + return str; | ||
| + } | ||
| + let internedString = interningStringMap.get(str); | ||
| + if (internedString === undefined) { | ||
| + internedString = str; | ||
| + interningStringMap.set(str, internedString); | ||
| + } | ||
| + return internedString; | ||
| +} | ||
| + | ||
| +let enableStringInterningRefCount = 0; | ||
| + | ||
| +function isStringInterningEnabled() { | ||
| + return enableStringInterningRefCount > 0; | ||
| +} | ||
| + | ||
| +/** | ||
| + * Enables a memory optimization to avoid repeat copies of the same string in | ||
| + * memory by caching a single reference to the string. This can reduce memory | ||
| + * usage if the same string is repeated many times in the compiler, such as | ||
| + * when Webpack layers are used with the same files. | ||
| + * | ||
| + * @returns {void} | ||
| + */ | ||
| +function enableStringInterning() { | ||
| + enableStringInterningRefCount++; | ||
| +} | ||
| + | ||
| +/** | ||
| + * Disables string interning. This should be called to free the memory used by | ||
| + * the interned strings after the compiler no longer needs to reuse the | ||
| + * interned strings such as at the end of the compilation. | ||
| + * | ||
| + * @returns {void} | ||
| + */ | ||
| +function disableStringInterning() { | ||
| + if (--enableStringInterningRefCount <= 0) { | ||
| + interningStringMap.clear(); | ||
| + enableStringInterningRefCount = 0; | ||
| + } | ||
| +} | ||
| + | ||
| +module.exports = { | ||
| + disableDualStringBufferCaching, | ||
| + disableStringInterning, | ||
| + enableDualStringBufferCaching, | ||
| + enableStringInterning, | ||
| + internString, | ||
| + isDualStringBufferCachingEnabled | ||
| +}; | ||
| diff --git a/lib/index.js b/lib/index.js | ||
| index 0c11c2f..86a7234 100644 | ||
| --- a/lib/index.js | ||
| +++ b/lib/index.js | ||
| @@ -28,3 +28,4 @@ defineExport("ReplaceSource", () => require("./ReplaceSource")); | ||
| defineExport("PrefixSource", () => require("./PrefixSource")); | ||
| defineExport("SizeOnlySource", () => require("./SizeOnlySource")); | ||
| defineExport("CompatSource", () => require("./CompatSource")); | ||
| +defineExport("stringBufferUtils", () => require("./helpers/stringBufferUtils")); |
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
Oops, something went wrong.