Merge pull request #8642 from webpack/memory/future-emit-assets

add output.futureEmitAssets

declarations/WebpackOptions.d.ts

@@ -1051,6 +1051,10 @@ export interface OutputOptions {
 	 * Specifies the name of each output file on disk. You must **not** specify an absolute path here! The `output.path` option determines the location on disk the files are written to, filename is used solely for naming the individual files.
 	 */
 	filename?: string | Function;
+	/**
+	 * Use the future version of asset emitting logic, which is allows freeing memory of assets after emitting. It could break plugins which assume that assets are still readable after emitting. Will be the new default in the next major version.
+	 */
+	futureEmitAssets?: boolean;
 	/**
 	 * An expression which is used to address the global object/scope in runtime code
 	 */

lib/Compilation.js

@@ -491,6 +491,8 @@ class Compilation extends Tapable {
 		this._buildingModules = new Map();
 		/** @private @type {Map<Module, Callback[]>} */
 		this._rebuildingModules = new Map();
+		/** @type {Set<string>} */
+		this.emittedAssets = new Set();
 	}
 
 	getStats() {

lib/Compiler.js

@@ -7,6 +7,7 @@
 const parseJson = require("json-parse-better-errors");
 const asyncLib = require("neo-async");
 const path = require("path");
+const { Source } = require("webpack-sources");
 const util = require("util");
 const {
 	Tapable,
@@ -188,6 +189,11 @@ class Compiler extends Tapable {
 
 		/** @type {boolean} */
 		this.watchMode = false;
+
+		/** @private @type {WeakMap<Source, { sizeOnlySource: SizeOnlySource, writtenTo: Map<string, number> }>} */
+		this._assetEmittingSourceCache = new WeakMap();
+		/** @private @type {Map<string, number>} */
+		this._assetEmittingWrittenFiles = new Map();
 	}
 
 	watch(watchOptions, handler) {
@@ -328,19 +334,86 @@ class Compiler extends Tapable {
 							outputPath,
 							targetFile
 						);
-						if (source.existsAt === targetPath) {
-							source.emitted = false;
-							return callback();
-						}
-						let content = source.source();
-
-						if (!Buffer.isBuffer(content)) {
-							content = Buffer.from(content, "utf8");
+						// TODO webpack 5 remove futureEmitAssets option and make it on by default
+						if (this.options.output.futureEmitAssets) {
+							// check if the target file has already been written by this Compiler
+							const targetFileGeneration = this._assetEmittingWrittenFiles.get(
+								targetPath
+							);
+
+							// create an cache entry for this Source if not already existing
+							let cacheEntry = this._assetEmittingSourceCache.get(source);
+							if (cacheEntry === undefined) {
+								cacheEntry = {
+									sizeOnlySource: undefined,
+									writtenTo: new Map()
+								};
+								this._assetEmittingSourceCache.set(source, cacheEntry);
+							}
+
+							// if the target file has already been written
+							if (targetFileGeneration !== undefined) {
+								// check if the Source has been written to this target file
+								const writtenGeneration = cacheEntry.writtenTo.get(targetPath);
+								if (writtenGeneration === targetFileGeneration) {
+									// if yes, we skip writing the file
+									// as it's already there
+									// (we assume one doesn't remove files while the Compiler is running)
+									return callback();
+								}
+							}
+
+							// get the binary (Buffer) content from the Source
+							/** @type {Buffer} */
+							let content;
+							if (typeof source.buffer === "function") {
+								content = source.buffer();
+							} else {
+								const bufferOrString = source.source();
+								if (Buffer.isBuffer(bufferOrString)) {
+									content = bufferOrString;
+								} else {
+									content = Buffer.from(bufferOrString, "utf8");
+								}
+							}
+
+							// Create a replacement resource which only allows to ask for size
+							// This allows to GC all memory allocated by the Source
+							// (expect when the Source is stored in any other cache)
+							cacheEntry.sizeOnlySource = new SizeOnlySource(content.length);
+							compilation.assets[file] = cacheEntry.sizeOnlySource;
+
+							// Write the file to output file system
+							this.outputFileSystem.writeFile(targetPath, content, err => {
+								if (err) return callback(err);
+
+								// information marker that the asset has been emitted
+								compilation.emittedAssets.add(file);
+
+								// cache the information that the Source has been written to that location
+								const newGeneration =
+									targetFileGeneration === undefined
+										? 1
+										: targetFileGeneration + 1;
+								cacheEntry.writtenTo.set(targetPath, newGeneration);
+								this._assetEmittingWrittenFiles.set(targetPath, newGeneration);
+								callback();
+							});
+						} else {
+							if (source.existsAt === targetPath) {
+								source.emitted = false;
+								return callback();
+							}
+							let content = source.source();
+
+							if (!Buffer.isBuffer(content)) {
+								content = Buffer.from(content, "utf8");
+							}
+
+							source.existsAt = targetPath;
+							source.emitted = true;
+							this.outputFileSystem.writeFile(targetPath, content, callback);
 						}
-
-						source.existsAt = targetPath;
-						source.emitted = true;
-						this.outputFileSystem.writeFile(targetPath, content, callback);
 					};
 
 					if (targetFile.match(/\/|\\/)) {
@@ -563,3 +636,48 @@ class Compiler extends Tapable {
 }
 
 module.exports = Compiler;
+
+class SizeOnlySource extends Source {
+	constructor(size) {
+		super();
+		this._size = size;
+	}
+
+	_error() {
+		return new Error(
+			"Content and Map of this Source is no longer available (only size() is supported)"
+		);
+	}
+
+	size() {
+		return this._size;
+	}
+
+	/**
+	 * @param {any} options options
+	 * @returns {string} the source
+	 */
+	source(options) {
+		throw this._error();
+	}
+
+	node() {
+		throw this._error();
+	}
+
+	listMap() {
+		throw this._error();
+	}
+
+	map() {
+		throw this._error();
+	}
+
+	listNode() {
+		throw this._error();
+	}
+
+	updateHash() {
+		throw this._error();
+	}
+}

lib/Stats.js

@@ -400,7 +400,10 @@ class Stats {
 						size: compilation.assets[asset].size(),
 						chunks: [],
 						chunkNames: [],
-						emitted: compilation.assets[asset].emitted
+						// TODO webpack 5: remove .emitted
+						emitted:
+							compilation.assets[asset].emitted ||
+							compilation.emittedAssets.has(asset)
 					};
 
 					if (showPerformance) {

schemas/WebpackOptions.json

@@ -867,6 +867,10 @@
             }
           ]
         },
+        "futureEmitAssets": {
+          "description": "Use the future version of asset emitting logic, which is allows freeing memory of assets after emitting. It could break plugins which assume that assets are still readable after emitting. Will be the new default in the next major version.",
+          "type": "boolean"
+        },
         "globalObject": {
           "description": "An expression which is used to address the global object/scope in runtime code",
           "type": "string",