Skip to content

Commit

Permalink
add emit/updateAsset to Compilation
Browse files Browse the repository at this point in the history
add asset info with
* immutable for long term cache-able asset
* size for the asset size in bytes
* development for devtools
* hotModuleReplacement for HMR assets

show asset info in stats
  • Loading branch information
sokra committed Sep 11, 2019
1 parent f5e1e1e commit 758269e
Show file tree
Hide file tree
Showing 16 changed files with 339 additions and 117 deletions.
7 changes: 3 additions & 4 deletions lib/BannerPlugin.js
Expand Up @@ -108,10 +108,9 @@ class BannerPlugin {

const comment = compilation.getPath(banner(data), data);

compilation.assets[file] = new ConcatSource(
comment,
"\n",
compilation.assets[file]
compilation.updateAsset(
file,
old => new ConcatSource(comment, "\n", old)
);
}
}
Expand Down
134 changes: 126 additions & 8 deletions lib/Compilation.js
Expand Up @@ -104,6 +104,21 @@ const buildChunkGraph = require("./buildChunkGraph");
* @property {string[]=} trace
*/

/**
* @typedef {Object} AssetInfo
* @property {boolean=} immutable true, if the asset can be long term cached forever (contains a hash)
* @property {number=} size size in bytes, only set after asset has been emitted
* @property {boolean=} development true, when asset is only used for development and doesn't count towards user-facing assets
* @property {boolean=} hotModuleReplacement true, when asset ships data for updating an existing application (HMR)
*/

/**
* @typedef {Object} Asset
* @property {string} name the filename of the asset
* @property {Source} source source of the asset
* @property {AssetInfo} info info about the asset
*/

/**
* @param {Chunk} a first chunk to sort by id
* @param {Chunk} b second chunk to sort by id
Expand Down Expand Up @@ -446,6 +461,7 @@ class Compilation extends Tapable {
this.entries = [];
/** @private @type {{name: string, request: string, module: Module}[]} */
this._preparedEntrypoints = [];
/** @type {Map<string, Entrypoint>} */
this.entrypoints = new Map();
/** @type {Chunk[]} */
this.chunks = [];
Expand All @@ -465,6 +481,8 @@ class Compilation extends Tapable {
this.additionalChunkAssets = [];
/** @type {CompilationAssets} */
this.assets = {};
/** @type {Map<string, AssetInfo>} */
this.assetsInfo = new Map();
/** @type {WebpackError[]} */
this.errors = [];
/** @type {WebpackError[]} */
Expand Down Expand Up @@ -1233,6 +1251,7 @@ class Compilation extends Tapable {
this.namedChunkGroups.clear();
this.additionalChunkAssets.length = 0;
this.assets = {};
this.assetsInfo.clear();
for (const module of this.modules) {
module.unseal();
}
Expand Down Expand Up @@ -1963,13 +1982,101 @@ class Compilation extends Tapable {
this.hash = this.fullHash.substr(0, hashDigestLength);
}

/**
* @param {string} file file name
* @param {Source} source asset source
* @param {AssetInfo} assetInfo extra asset information
* @returns {void}
*/
emitAsset(file, source, assetInfo = {}) {
if (this.assets[file]) {
if (this.assets[file] !== source) {

This comment has been minimized.

Copy link
@jsg2021

jsg2021 Sep 12, 2019

I'm seeing this error from the same source, but imported in two different stylesheets... for an image.

This comment has been minimized.

Copy link
@LuisRizo

LuisRizo Sep 12, 2019

Same here. We're locking our Webpack version too 4.39.3 to solve this issue that's preventing us from deploying

throw new Error(
`Conflict: Multiple assets emit to the same filename ${file}`
);
}
const oldInfo = this.assetsInfo.get(file);
this.assetsInfo.set(file, Object.assign({}, oldInfo, assetInfo));
return;
}
this.assets[file] = source;
this.assetsInfo.set(file, assetInfo);
}

/**
* @param {string} file file name
* @param {Source | function(Source): Source} newSourceOrFunction new asset source or function converting old to new
* @param {AssetInfo | function(AssetInfo | undefined): AssetInfo} assetInfoUpdateOrFunction new asset info or function converting old to new
*/
updateAsset(
file,
newSourceOrFunction,
assetInfoUpdateOrFunction = undefined
) {
if (!this.assets[file]) {
throw new Error(
`Called Compilation.updateAsset for not existing filename ${file}`
);
}
if (typeof newSourceOrFunction === "function") {
this.assets[file] = newSourceOrFunction(this.assets[file]);
} else {
this.assets[file] = newSourceOrFunction;
}
if (assetInfoUpdateOrFunction !== undefined) {
const oldInfo = this.assetsInfo.get(file);
if (typeof assetInfoUpdateOrFunction === "function") {
this.assetsInfo.set(file, assetInfoUpdateOrFunction(oldInfo || {}));
} else {
this.assetsInfo.set(
file,
Object.assign({}, oldInfo, assetInfoUpdateOrFunction)
);
}
}
}

getAssets() {
/** @type {Asset[]} */
const array = [];
for (const assetName of Object.keys(this.assets)) {
if (Object.prototype.hasOwnProperty.call(this.assets, assetName)) {
array.push({
name: assetName,
source: this.assets[assetName],
info: this.assetsInfo.get(assetName) || {}
});
}
}
return array;
}

/**
* @param {string} name the name of the asset
* @returns {Asset | undefined} the asset or undefined when not found
*/
getAsset(name) {
if (!Object.prototype.hasOwnProperty.call(this.assets, name))
return undefined;
return {
name,
source: this.assets[name],
info: this.assetsInfo.get(name) || {}
};
}

createModuleAssets() {
for (let i = 0; i < this.modules.length; i++) {
const module = this.modules[i];
if (module.buildInfo.assets) {
const assetsInfo = module.buildInfo.assetsInfo;
for (const assetName of Object.keys(module.buildInfo.assets)) {
const fileName = this.getPath(assetName);
this.assets[fileName] = module.buildInfo.assets[assetName];
this.emitAsset(
fileName,
module.buildInfo.assets[assetName],
assetsInfo ? assetsInfo.get(assetName) : undefined
);
this.hooks.moduleAsset.call(module, fileName);
}
}
Expand Down Expand Up @@ -2003,7 +2110,12 @@ class Compilation extends Tapable {
const cacheName = fileManifest.identifier;
const usedHash = fileManifest.hash;
filenameTemplate = fileManifest.filenameTemplate;
file = this.getPath(filenameTemplate, fileManifest.pathOptions);
const pathAndInfo = this.getPathWithInfo(
filenameTemplate,
fileManifest.pathOptions
);
file = pathAndInfo.path;
const assetInfo = pathAndInfo.info;

// check if the same filename was already written by another chunk
const alreadyWritten = alreadyWrittenFiles.get(file);
Expand Down Expand Up @@ -2051,12 +2163,7 @@ class Compilation extends Tapable {
};
}
}
if (this.assets[file] && this.assets[file] !== source) {
throw new Error(
`Conflict: Multiple assets emit to the same filename ${file}`
);
}
this.assets[file] = source;
this.emitAsset(file, source, assetInfo);
chunk.files.push(file);
this.hooks.chunkAsset.call(chunk, file);
alreadyWrittenFiles.set(file, {
Expand Down Expand Up @@ -2084,6 +2191,17 @@ class Compilation extends Tapable {
return this.mainTemplate.getAssetPath(filename, data);
}

/**
* @param {string} filename used to get asset path with hash
* @param {TODO=} data // TODO: figure out this param type
* @returns {{ path: string, info: AssetInfo }} interpolated path and asset info
*/
getPathWithInfo(filename, data) {
data = data || {};
data.hash = data.hash || this.hash;
return this.mainTemplate.getAssetPathWithInfo(filename, data);
}

/**
* This function allows you to run another instance of webpack inside of webpack however as
* a child with different settings and configurations (if desired) applied. It copies all hooks, plugins
Expand Down
20 changes: 15 additions & 5 deletions lib/Compiler.js
Expand Up @@ -329,8 +329,8 @@ class Compiler extends Tapable {
if (err) return callback(err);

this.parentCompilation.children.push(compilation);
for (const name of Object.keys(compilation.assets)) {
this.parentCompilation.assets[name] = compilation.assets[name];
for (const { name, source, info } of compilation.getAssets()) {
this.parentCompilation.emitAsset(name, source, info);
}

const entries = Array.from(
Expand All @@ -356,9 +356,9 @@ class Compiler extends Tapable {
if (err) return callback(err);

asyncLib.forEachLimit(
compilation.assets,
compilation.getAssets(),
15,
(source, file, callback) => {
({ name: file, source }, callback) => {
let targetFile = file;
const queryStringIdx = targetFile.indexOf("?");
if (queryStringIdx >= 0) {
Expand Down Expand Up @@ -396,10 +396,18 @@ class Compiler extends Tapable {
// if yes, we skip writing the file
// as it's already there
// (we assume one doesn't remove files while the Compiler is running)

compilation.updateAsset(file, cacheEntry.sizeOnlySource, {
size: cacheEntry.sizeOnlySource.size()
});

return callback();
}
}

// TODO webpack 5: if info.immutable check if file already exists in output
// skip emitting if it's already there

// get the binary (Buffer) content from the Source
/** @type {Buffer} */
let content;
Expand All @@ -418,7 +426,9 @@ class Compiler extends Tapable {
// This allows to GC all memory allocated by the Source
// (expect when the Source is stored in any other cache)
cacheEntry.sizeOnlySource = new SizeOnlySource(content.length);
compilation.assets[file] = cacheEntry.sizeOnlySource;
compilation.updateAsset(file, cacheEntry.sizeOnlySource, {
size: content.length
});

// Write the file to output file system
this.outputFileSystem.writeFile(targetPath, content, err => {
Expand Down
22 changes: 18 additions & 4 deletions lib/HotModuleReplacementPlugin.js
Expand Up @@ -277,12 +277,19 @@ module.exports = class HotModuleReplacementPlugin {
compilation.moduleTemplates.javascript,
compilation.dependencyTemplates
);
const filename = compilation.getPath(hotUpdateChunkFilename, {
const {
path: filename,
info: assetInfo
} = compilation.getPathWithInfo(hotUpdateChunkFilename, {
hash: records.hash,
chunk: currentChunk
});
compilation.additionalChunkAssets.push(filename);
compilation.assets[filename] = source;
compilation.emitAsset(
filename,
source,
Object.assign({ hotModuleReplacement: true }, assetInfo)
);
hotUpdateMainContent.c[chunkId] = true;
currentChunk.files.push(filename);
compilation.hooks.chunkAsset.call(currentChunk, filename);
Expand All @@ -292,10 +299,17 @@ module.exports = class HotModuleReplacementPlugin {
}
}
const source = new RawSource(JSON.stringify(hotUpdateMainContent));
const filename = compilation.getPath(hotUpdateMainFilename, {
const {
path: filename,
info: assetInfo
} = compilation.getPathWithInfo(hotUpdateMainFilename, {
hash: records.hash
});
compilation.assets[filename] = source;
compilation.emitAsset(
filename,
source,
Object.assign({ hotModuleReplacement: true }, assetInfo)
);
}
);

Expand Down
9 changes: 8 additions & 1 deletion lib/MainTemplate.js
Expand Up @@ -122,7 +122,7 @@ module.exports = class MainTemplate extends Tapable {
"moduleExpression"
]),
currentHash: new SyncWaterfallHook(["source", "requestedLength"]),
assetPath: new SyncWaterfallHook(["path", "options"]),
assetPath: new SyncWaterfallHook(["path", "options", "assetInfo"]),
hash: new SyncHook(["hash"]),
hashForChunk: new SyncHook(["hash", "chunk"]),
globalHashPaths: new SyncWaterfallHook(["paths"]),
Expand Down Expand Up @@ -521,6 +521,13 @@ module.exports = class MainTemplate extends Tapable {
return this.hooks.assetPath.call(path, options);
}

getAssetPathWithInfo(path, options) {
const assetInfo = {};
// TODO webpack 5: refactor assetPath hook to receive { path, info } object
const newPath = this.hooks.assetPath.call(path, options, assetInfo);
return { path: newPath, info: assetInfo };
}

/**
* Updates hash with information from this template
* @param {Hash} hash the hash to update
Expand Down
8 changes: 6 additions & 2 deletions lib/NormalModule.js
Expand Up @@ -210,15 +210,17 @@ class NormalModule extends Module {
}
};
},
emitFile: (name, content, sourceMap) => {
emitFile: (name, content, sourceMap, assetInfo) => {
if (!this.buildInfo.assets) {
this.buildInfo.assets = Object.create(null);
this.buildInfo.assetsInfo = new Map();
}
this.buildInfo.assets[name] = this.createSourceForAsset(
name,
content,
sourceMap
);
this.buildInfo.assetsInfo.set(name, assetInfo);
},
rootContext: options.context,
webpack: true,
Expand Down Expand Up @@ -432,7 +434,9 @@ class NormalModule extends Module {
this.buildInfo = {
cacheable: false,
fileDependencies: new Set(),
contextDependencies: new Set()
contextDependencies: new Set(),
assets: undefined,
assetsInfo: undefined
};

return this.doBuild(options, compilation, resolver, fs, err => {
Expand Down

0 comments on commit 758269e

Please sign in to comment.