Merge pull request #13184 from webpack/memory/usage

reduce memory usage when using the filesystem cache
webpack · Apr 20, 2021 · 950aed2 · 950aed2
2 parents 85fe6ac + be66dc6
commit 950aed2
Show file tree

Hide file tree

Showing 11 changed files with 155 additions and 31 deletions.
diff --git a/declarations/WebpackOptions.d.ts b/declarations/WebpackOptions.d.ts
@@ -916,6 +916,10 @@ export interface MemoryCacheOptions {
  * Options object for persistent file-based caching.
  */
 export interface FileCacheOptions {
+	/**
+	 * Allows to collect unused memory allocated during deserialization. This requires copying data into smaller buffers and has a performance cost.
+	 */
+	allowCollectingMemory?: boolean;
 	/**
 	 * Dependencies the build depends on (in multiple categories, default categories: 'defaultWebpack').
 	 */

diff --git a/lib/WebpackOptionsApply.js b/lib/WebpackOptionsApply.js
@@ -568,7 +568,8 @@ class WebpackOptionsApply extends OptionsApply {
 										"webpack.cache.PackFileCacheStrategy"
 									),
 									snapshot: options.snapshot,
-									maxAge: cacheOptions.maxAge
+									maxAge: cacheOptions.maxAge,
+									allowCollectingMemory: cacheOptions.allowCollectingMemory
 								}),
 								cacheOptions.idleTimeout,
 								cacheOptions.idleTimeoutForInitialStore

diff --git a/lib/cache/PackFileCacheStrategy.js b/lib/cache/PackFileCacheStrategy.js
@@ -768,6 +768,14 @@ class PackContent {
 	}
 }
 
+const allowCollectingMemory = buf => {
+	const wasted = buf.buffer.byteLength - buf.byteLength;
+	if ((wasted > 8192 && wasted > 1048576) || wasted > buf.byteLength) {
+		return Buffer.from(buf);
+	}
+	return buf;
+};
+
 class PackFileCacheStrategy {
 	/**
 	 * @param {Object} options options
@@ -779,6 +787,7 @@ class PackFileCacheStrategy {
 	 * @param {Logger} options.logger a logger
 	 * @param {SnapshotOptions} options.snapshot options regarding snapshotting
 	 * @param {number} options.maxAge max age of cache items
+	 * @param {boolean} options.allowCollectingMemory allow to collect unused memory created during deserialization
 	 */
 	constructor({
 		compiler,
@@ -788,7 +797,8 @@ class PackFileCacheStrategy {
 		version,
 		logger,
 		snapshot,
-		maxAge
+		maxAge,
+		allowCollectingMemory
 	}) {
 		this.fileSerializer = createFileSerializer(fs);
 		this.fileSystemInfo = new FileSystemInfo(fs, {
@@ -802,6 +812,7 @@ class PackFileCacheStrategy {
 		this.version = version;
 		this.logger = logger;
 		this.maxAge = maxAge;
+		this.allowCollectingMemory = allowCollectingMemory;
 		this.snapshot = snapshot;
 		/** @type {Set<string>} */
 		this.buildDependencies = new Set();
@@ -845,7 +856,10 @@ class PackFileCacheStrategy {
 			.deserialize(null, {
 				filename: `${cacheLocation}/index.pack`,
 				extension: ".pack",
-				logger
+				logger,
+				retainedBuffer: this.allowCollectingMemory
+					? allowCollectingMemory
+					: undefined
 			})
 			.catch(err => {
 				if (err.code !== "ENOENT") {

diff --git a/lib/config/defaults.js b/lib/config/defaults.js
@@ -295,8 +295,9 @@ const applyCacheDefaults = (cache, { name, mode, development }) => {
 			D(cache, "store", "pack");
 			D(cache, "idleTimeout", 60000);
 			D(cache, "idleTimeoutForInitialStore", 0);
-			D(cache, "maxMemoryGenerations", development ? 10 : Infinity);
+			D(cache, "maxMemoryGenerations", development ? 5 : Infinity);
 			D(cache, "maxAge", 1000 * 60 * 60 * 24 * 60); // 1 month
+			D(cache, "allowCollectingMemory", development);
 			D(cache.buildDependencies, "defaultWebpack", [
 				path.resolve(__dirname, "..") + path.sep
 			]);

diff --git a/lib/serialization/BinaryMiddleware.js b/lib/serialization/BinaryMiddleware.js
@@ -123,6 +123,12 @@ class BinaryMiddleware extends SerializerMiddleware {
 		return this._serialize(data, context);
 	}
 
+	_serializeLazy(fn, context) {
+		return SerializerMiddleware.serializeLazy(fn, data =>
+			this._serialize(data, context)
+		);
+	}
+
 	/**
 	 * @param {DeserializedType} data data
 	 * @param {Object} context context object
@@ -135,7 +141,7 @@ class BinaryMiddleware extends SerializerMiddleware {
 		let leftOverBuffer = null;
 		let currentPosition = 0;
 		/** @type {BufferSerializableType[]} */
-		const buffers = [];
+		let buffers = [];
 		let buffersTotalLength = 0;
 		const allocate = bytesNeeded => {
 			if (currentBuffer !== null) {
@@ -204,10 +210,7 @@ class BinaryMiddleware extends SerializerMiddleware {
 								SerializerMiddleware.setLazySerializedValue(thing, data);
 								serializedData = data;
 							} else {
-								serializedData = SerializerMiddleware.serializeLazy(
-									thing,
-									data => this._serialize(data, context)
-								);
+								serializedData = this._serializeLazy(thing, context);
 							}
 						}
 						if (typeof serializedData === "function") {
@@ -474,7 +477,13 @@ class BinaryMiddleware extends SerializerMiddleware {
 		};
 		serializeData(data);
 		flush();
-		return buffers;
+
+		// avoid leaking memory
+		currentBuffer = null;
+		leftOverBuffer = null;
+		const _buffers = buffers;
+		buffers = undefined;
+		return _buffers;
 	}
 
 	/**
@@ -486,6 +495,21 @@ class BinaryMiddleware extends SerializerMiddleware {
 		return this._deserialize(data, context);
 	}
 
+	_createLazyDeserialized(content, context) {
+		return SerializerMiddleware.createLazy(
+			memoize(() => this._deserialize(content, context)),
+			this,
+			undefined,
+			content
+		);
+	}
+
+	_deserializeLazy(fn, context) {
+		return SerializerMiddleware.deserializeLazy(fn, data =>
+			this._deserialize(data, context)
+		);
+	}
+
 	/**
 	 * @param {SerializedType} data data
 	 * @param {Object} context context object
@@ -497,6 +521,8 @@ class BinaryMiddleware extends SerializerMiddleware {
 		let currentIsBuffer = Buffer.isBuffer(currentBuffer);
 		let currentPosition = 0;
 
+		const retainedBuffer = context.retainedBuffer || (x => x);
+
 		const checkOverflow = () => {
 			if (currentPosition >= currentBuffer.length) {
 				currentPosition = 0;
@@ -610,23 +636,16 @@ class BinaryMiddleware extends SerializerMiddleware {
 								do {
 									const buf = readUpTo(l);
 									l -= buf.length;
-									content.push(buf);
+									content.push(retainedBuffer(buf));
 								} while (l > 0);
 							}
 						}
-						result.push(
-							SerializerMiddleware.createLazy(
-								memoize(() => this._deserialize(content, context)),
-								this,
-								undefined,
-								content
-							)
-						);
+						result.push(this._createLazyDeserialized(content, context));
 					};
 				case BUFFER_HEADER:
 					return () => {
 						const len = readU32();
-						result.push(read(len));
+						result.push(retainedBuffer(read(len)));
 					};
 				case TRUE_HEADER:
 					return () => result.push(true);
@@ -852,14 +871,10 @@ class BinaryMiddleware extends SerializerMiddleware {
 		});
 
 		/** @type {DeserializedType} */
-		const result = [];
+		let result = [];
 		while (currentBuffer !== null) {
 			if (typeof currentBuffer === "function") {
-				result.push(
-					SerializerMiddleware.deserializeLazy(currentBuffer, data =>
-						this._deserialize(data, context)
-					)
-				);
+				result.push(this._deserializeLazy(currentBuffer, context));
 				currentDataItem++;
 				currentBuffer =
 					currentDataItem < data.length ? data[currentDataItem] : null;
@@ -869,7 +884,11 @@ class BinaryMiddleware extends SerializerMiddleware {
 				dispatchTable[header]();
 			}
 		}
-		return result;
+
+		// avoid leaking memory in context
+		let _result = result;
+		result = undefined;
+		return _result;
 	}
 }
 

diff --git a/lib/util/comparators.js b/lib/util/comparators.js
@@ -34,9 +34,7 @@ const createCachedParameterizedComparator = fn => {
 		 * @param {T} b second item
 		 * @returns {-1|0|1} compare result
 		 */
-		const result = (a, b) => {
-			return fn(arg, a, b);
-		};
+		const result = fn.bind(null, arg);
 		map.set(arg, result);
 		return result;
 	};

diff --git a/schemas/WebpackOptions.check.js b/schemas/WebpackOptions.check.js
diff --git a/schemas/WebpackOptions.json b/schemas/WebpackOptions.json
@@ -924,6 +924,10 @@
       "type": "object",
       "additionalProperties": false,
       "properties": {
+        "allowCollectingMemory": {
+          "description": "Allows to collect unused memory allocated during deserialization. This requires copying data into smaller buffers and has a performance cost.",
+          "type": "boolean"
+        },
         "buildDependencies": {
           "description": "Dependencies the build depends on (in multiple categories, default categories: 'defaultWebpack').",
           "type": "object",

diff --git a/test/Defaults.unittest.js b/test/Defaults.unittest.js
@@ -1475,6 +1475,7 @@ describe("Defaults", () => {
 		@@ ... @@
 		-   "cache": false,
 		+   "cache": Object {
+		+     "allowCollectingMemory": false,
 		+     "buildDependencies": Object {
 		+       "defaultWebpack": Array [
 		+         "<cwd>/lib/",
@@ -1503,6 +1504,69 @@ describe("Defaults", () => {
 		+     "cache": true,
 	`)
 	);
+	test(
+		"cache filesystem development",
+		{ mode: "development", cache: { type: "filesystem" } },
+		e =>
+			e.toMatchInlineSnapshot(`
+			- Expected
+			+ Received
+
+			@@ ... @@
+			-   "cache": false,
+			+   "cache": Object {
+			+     "allowCollectingMemory": true,
+			+     "buildDependencies": Object {
+			+       "defaultWebpack": Array [
+			+         "<cwd>/lib/",
+			+       ],
+			+     },
+			+     "cacheDirectory": "<cwd>/node_modules/.cache/webpack",
+			+     "cacheLocation": "<cwd>/node_modules/.cache/webpack/default-development",
+			+     "hashAlgorithm": "md4",
+			+     "idleTimeout": 60000,
+			+     "idleTimeoutForInitialStore": 0,
+			+     "maxAge": 5184000000,
+			+     "maxMemoryGenerations": 5,
+			+     "name": "default-development",
+			+     "store": "pack",
+			+     "type": "filesystem",
+			+     "version": "",
+			+   },
+			@@ ... @@
+			-   "devtool": false,
+			+   "devtool": "eval",
+			@@ ... @@
+			-   "mode": "none",
+			+   "mode": "development",
+			@@ ... @@
+			-     "unsafeCache": false,
+			+     "unsafeCache": [Function anonymous],
+			@@ ... @@
+			-     "chunkIds": "natural",
+			+     "chunkIds": "named",
+			@@ ... @@
+			-     "moduleIds": "natural",
+			-     "nodeEnv": false,
+			+     "moduleIds": "named",
+			+     "nodeEnv": "development",
+			@@ ... @@
+			-       "minRemainingSize": undefined,
+			+       "minRemainingSize": 0,
+			@@ ... @@
+			-     "pathinfo": false,
+			+     "pathinfo": true,
+			@@ ... @@
+			-     "cache": false,
+			+     "cache": true,
+			@@ ... @@
+			-       "production",
+			+       "development",
+			@@ ... @@
+			-     "cache": false,
+			+     "cache": true,
+		`)
+	);
 
 	test(
 		"disable",
@@ -1691,6 +1755,7 @@ describe("Defaults", () => {
 			-   "cache": false,
 			-   "context": "<cwd>",
 			+   "cache": Object {
+			+     "allowCollectingMemory": false,
 			+     "buildDependencies": Object {
 			+       "defaultWebpack": Array [
 			+         "<cwd>/lib/",

diff --git a/test/__snapshots__/Cli.test.js.snap b/test/__snapshots__/Cli.test.js.snap
@@ -56,6 +56,19 @@ Object {
     "multiple": false,
     "simpleType": "boolean",
   },
+  "cache-allow-collecting-memory": Object {
+    "configs": Array [
+      Object {
+        "description": "Allows to collect unused memory allocated during deserialization. This requires copying data into smaller buffers and has a performance cost.",
+        "multiple": false,
+        "path": "cache.allowCollectingMemory",
+        "type": "boolean",
+      },
+    ],
+    "description": "Allows to collect unused memory allocated during deserialization. This requires copying data into smaller buffers and has a performance cost.",
+    "multiple": false,
+    "simpleType": "boolean",
+  },
   "cache-cache-directory": Object {
     "configs": Array [
       Object {

diff --git a/types.d.ts b/types.d.ts
@@ -3759,6 +3759,11 @@ declare class FetchCompileWasmPlugin {
  * Options object for persistent file-based caching.
  */
 declare interface FileCacheOptions {
+	/**
+	 * Allows to collect unused memory allocated during deserialization. This requires copying data into smaller buffers and has a performance cost.
+	 */
+	allowCollectingMemory?: boolean;
+
 	/**
 	 * Dependencies the build depends on (in multiple categories, default categories: 'defaultWebpack').
 	 */