From 126fb9912187fc8a09d00f831267e1eae85c4984 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Thu, 25 Jul 2019 15:52:54 +0200 Subject: [PATCH] move chunk graph building into separate file refactor chunk graph generation to include available modules optimization --- lib/Chunk.js | 6 +- lib/ChunkGroup.js | 2 +- lib/Compilation.js | 472 +--------------------------- lib/buildChunkGraph.js | 683 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 694 insertions(+), 469 deletions(-) create mode 100644 lib/buildChunkGraph.js diff --git a/lib/Chunk.js b/lib/Chunk.js index aa69c7fa9da..16e28a62a76 100644 --- a/lib/Chunk.js +++ b/lib/Chunk.js @@ -355,7 +355,11 @@ class Chunk { return this._modules.getFromUnorderedCache(getModulesIdent); } - remove() { + /** + * @param {string=} reason reason why chunk is removed + * @returns {void} + */ + remove(reason) { // cleanup modules // Array.from is used here to create a clone, because removeChunk modifies this._modules for (const module of Array.from(this._modules)) { diff --git a/lib/ChunkGroup.js b/lib/ChunkGroup.js index 58e76603900..336d21a0478 100644 --- a/lib/ChunkGroup.js +++ b/lib/ChunkGroup.js @@ -345,7 +345,7 @@ class ChunkGroup { } /** - * @param {ModuleReason} reason reason for removing ChunkGroup + * @param {string=} reason reason for removing ChunkGroup * @returns {void} */ remove(reason) { diff --git a/lib/Compilation.js b/lib/Compilation.js index 9a77119b253..2ae5090276b 100644 --- a/lib/Compilation.js +++ b/lib/Compilation.js @@ -27,17 +27,16 @@ const HotUpdateChunkTemplate = require("./HotUpdateChunkTemplate"); const ModuleTemplate = require("./ModuleTemplate"); const RuntimeTemplate = require("./RuntimeTemplate"); const ChunkRenderError = require("./ChunkRenderError"); -const AsyncDependencyToInitialChunkError = require("./AsyncDependencyToInitialChunkError"); const Stats = require("./Stats"); const Semaphore = require("./util/Semaphore"); const createHash = require("./util/createHash"); -const Queue = require("./util/Queue"); const SortableSet = require("./util/SortableSet"); const GraphHelpers = require("./GraphHelpers"); const ModuleDependency = require("./dependencies/ModuleDependency"); const compareLocations = require("./compareLocations"); const { Logger, LogType } = require("./logging/Logger"); const ErrorHelpers = require("./ErrorHelpers"); +const buildChunkGraph = require("./buildChunkGraph"); /** @typedef {import("./Module")} Module */ /** @typedef {import("./Compiler")} Compiler */ @@ -165,16 +164,6 @@ const byNameOrHash = (a, b) => { return 0; }; -/** - * @template T - * @param {Set} a first set - * @param {Set} b second set - * @returns {number} cmp - */ -const bySetSize = (a, b) => { - return a.size - b.size; -}; - /** * @param {DependenciesBlockVariable[]} variables DepBlock Variables to iterate over * @param {DepBlockVarDependenciesCallback} fn callback to apply on iterated elements @@ -1295,7 +1284,10 @@ class Compilation extends Tapable { this.assignDepth(module); } - this.processDependenciesBlocksForChunkGroups(this.chunkGroups.slice()); + buildChunkGraph( + this, + /** @type {Entrypoint[]} */ (this.chunkGroups.slice()) + ); this.sortModules(this.modules); this.hooks.afterChunks.call(this.chunks); @@ -1583,460 +1575,6 @@ class Compilation extends Tapable { return this.hooks.dependencyReference.call(ref, dependency, module); } - /** - * This method creates the Chunk graph from the Module graph - * @private - * @param {TODO[]} inputChunkGroups chunk groups which are processed - * @returns {void} - */ - processDependenciesBlocksForChunkGroups(inputChunkGroups) { - // Process is splitting into two parts: - // Part one traverse the module graph and builds a very basic chunks graph - // in chunkDependencies. - // Part two traverse every possible way through the basic chunk graph and - // tracks the available modules. While traversing it connects chunks with - // eachother and Blocks with Chunks. It stops traversing when all modules - // for a chunk are already available. So it doesn't connect unneeded chunks. - - /** @type {Map} */ - const chunkDependencies = new Map(); - const allCreatedChunkGroups = new Set(); - - // PREPARE - /** @type {Map} */ - const blockInfoMap = new Map(); - - /** - * @param {Dependency} d dependency to iterate over - * @returns {void} - */ - const iteratorDependency = d => { - // We skip Dependencies without Reference - const ref = this.getDependencyReference(currentModule, d); - if (!ref) { - return; - } - // We skip Dependencies without Module pointer - const refModule = ref.module; - if (!refModule) { - return; - } - // We skip weak Dependencies - if (ref.weak) { - return; - } - - blockInfoModules.add(refModule); - }; - - /** - * @param {AsyncDependenciesBlock} b blocks to prepare - * @returns {void} - */ - const iteratorBlockPrepare = b => { - blockInfoBlocks.push(b); - blockQueue.push(b); - }; - - /** @type {Module} */ - let currentModule; - /** @type {DependenciesBlock} */ - let block; - /** @type {DependenciesBlock[]} */ - let blockQueue; - /** @type {Set} */ - let blockInfoModules; - /** @type {AsyncDependenciesBlock[]} */ - let blockInfoBlocks; - - for (const module of this.modules) { - blockQueue = [module]; - currentModule = module; - while (blockQueue.length > 0) { - block = blockQueue.pop(); - blockInfoModules = new Set(); - blockInfoBlocks = []; - - if (block.variables) { - iterationBlockVariable(block.variables, iteratorDependency); - } - - if (block.dependencies) { - iterationOfArrayCallback(block.dependencies, iteratorDependency); - } - - if (block.blocks) { - iterationOfArrayCallback(block.blocks, iteratorBlockPrepare); - } - - const blockInfo = { - modules: Array.from(blockInfoModules), - blocks: blockInfoBlocks - }; - blockInfoMap.set(block, blockInfo); - } - } - - // PART ONE - - /** @type {Map} */ - const chunkGroupCounters = new Map(); - for (const chunkGroup of inputChunkGroups) { - chunkGroupCounters.set(chunkGroup, { index: 0, index2: 0 }); - } - - let nextFreeModuleIndex = 0; - let nextFreeModuleIndex2 = 0; - - /** @type {Map} */ - const blockChunkGroups = new Map(); - - /** @type {Set} */ - const blocksWithNestedBlocks = new Set(); - - const ADD_AND_ENTER_MODULE = 0; - const ENTER_MODULE = 1; - const PROCESS_BLOCK = 2; - const LEAVE_MODULE = 3; - - /** - * @typedef {Object} QueueItem - * @property {number} action - * @property {DependenciesBlock} block - * @property {Module} module - * @property {Chunk} chunk - * @property {ChunkGroup} chunkGroup - */ - - /** - * @param {ChunkGroup} chunkGroup chunk group - * @returns {QueueItem} queue item - */ - const chunkGroupToQueueItem = chunkGroup => ({ - action: ENTER_MODULE, - block: chunkGroup.chunks[0].entryModule, - module: chunkGroup.chunks[0].entryModule, - chunk: chunkGroup.chunks[0], - chunkGroup - }); - - // Start with the provided modules/chunks - /** @type {QueueItem[]} */ - let queue = inputChunkGroups.map(chunkGroupToQueueItem).reverse(); - /** @type {QueueItem[]} */ - let queueDelayed = []; - - /** @type {Module} */ - let module; - /** @type {Chunk} */ - let chunk; - /** @type {ChunkGroup} */ - let chunkGroup; - - // For each async Block in graph - /** - * @param {AsyncDependenciesBlock} b iterating over each Async DepBlock - * @returns {void} - */ - const iteratorBlock = b => { - // 1. We create a chunk for this Block - // but only once (blockChunkGroups map) - let c = blockChunkGroups.get(b); - if (c === undefined) { - c = this.namedChunkGroups.get(b.chunkName); - if (c && c.isInitial()) { - this.errors.push( - new AsyncDependencyToInitialChunkError(b.chunkName, module, b.loc) - ); - c = chunkGroup; - } else { - c = this.addChunkInGroup( - b.groupOptions || b.chunkName, - module, - b.loc, - b.request - ); - chunkGroupCounters.set(c, { index: 0, index2: 0 }); - blockChunkGroups.set(b, c); - allCreatedChunkGroups.add(c); - } - } else { - // TODO webpack 5 remove addOptions check - if (c.addOptions) c.addOptions(b.groupOptions); - c.addOrigin(module, b.loc, b.request); - } - - // 2. We store the Block+Chunk mapping as dependency for the chunk - let deps = chunkDependencies.get(chunkGroup); - if (!deps) chunkDependencies.set(chunkGroup, (deps = [])); - deps.push({ - block: b, - chunkGroup: c, - couldBeFiltered: true - }); - - // 3. We enqueue the DependenciesBlock for traversal - queueDelayed.push({ - action: PROCESS_BLOCK, - block: b, - module: module, - chunk: c.chunks[0], - chunkGroup: c - }); - }; - - // Iterative traversal of the Module graph - // Recursive would be simpler to write but could result in Stack Overflows - while (queue.length) { - while (queue.length) { - const queueItem = queue.pop(); - module = queueItem.module; - block = queueItem.block; - chunk = queueItem.chunk; - chunkGroup = queueItem.chunkGroup; - - switch (queueItem.action) { - case ADD_AND_ENTER_MODULE: { - // We connect Module and Chunk when not already done - if (chunk.addModule(module)) { - module.addChunk(chunk); - } else { - // already connected, skip it - break; - } - } - // fallthrough - case ENTER_MODULE: { - if (chunkGroup !== undefined) { - const index = chunkGroup.getModuleIndex(module); - if (index === undefined) { - chunkGroup.setModuleIndex( - module, - chunkGroupCounters.get(chunkGroup).index++ - ); - } - } - - if (module.index === null) { - module.index = nextFreeModuleIndex++; - } - - queue.push({ - action: LEAVE_MODULE, - block, - module, - chunk, - chunkGroup - }); - } - // fallthrough - case PROCESS_BLOCK: { - // get prepared block info - const blockInfo = blockInfoMap.get(block); - - // Traverse all referenced modules - for (let i = blockInfo.modules.length - 1; i >= 0; i--) { - const refModule = blockInfo.modules[i]; - if (chunk.containsModule(refModule)) { - // skip early if already connected - continue; - } - // enqueue the add and enter to enter in the correct order - // this is relevant with circular dependencies - queue.push({ - action: ADD_AND_ENTER_MODULE, - block: refModule, - module: refModule, - chunk, - chunkGroup - }); - } - - // Traverse all Blocks - iterationOfArrayCallback(blockInfo.blocks, iteratorBlock); - - if (blockInfo.blocks.length > 0 && module !== block) { - blocksWithNestedBlocks.add(block); - } - break; - } - case LEAVE_MODULE: { - if (chunkGroup !== undefined) { - const index = chunkGroup.getModuleIndex2(module); - if (index === undefined) { - chunkGroup.setModuleIndex2( - module, - chunkGroupCounters.get(chunkGroup).index2++ - ); - } - } - - if (module.index2 === null) { - module.index2 = nextFreeModuleIndex2++; - } - break; - } - } - } - const tempQueue = queue; - queue = queueDelayed.reverse(); - queueDelayed = tempQueue; - } - - // PART TWO - /** @type {Set} */ - let newAvailableModules; - - /** - * @typedef {Object} ChunkGroupInfo - * @property {Set} minAvailableModules current minimal set of modules available at this point - * @property {Set[]} availableModulesToBeMerged enqueued updates to the minimal set of available modules - */ - - /** @type {Map} */ - const chunkGroupInfoMap = new Map(); - - /** @type {Queue} */ - const queue2 = new Queue(inputChunkGroups); - - for (const chunkGroup of inputChunkGroups) { - chunkGroupInfoMap.set(chunkGroup, { - minAvailableModules: undefined, - availableModulesToBeMerged: [new Set()] - }); - } - - /** - * Helper function to check if all modules of a chunk are available - * - * @param {ChunkGroup} chunkGroup the chunkGroup to scan - * @param {Set} availableModules the comparitor set - * @returns {boolean} return true if all modules of a chunk are available - */ - const areModulesAvailable = (chunkGroup, availableModules) => { - for (const chunk of chunkGroup.chunks) { - for (const module of chunk.modulesIterable) { - if (!availableModules.has(module)) return false; - } - } - return true; - }; - - // For each edge in the basic chunk graph - /** - * @param {TODO} dep the dependency used for filtering - * @returns {boolean} used to filter "edges" (aka Dependencies) that were pointing - * to modules that are already available. Also filters circular dependencies in the chunks graph - */ - const filterFn = dep => { - const depChunkGroup = dep.chunkGroup; - if (!dep.couldBeFiltered) return true; - if (blocksWithNestedBlocks.has(dep.block)) return true; - if (areModulesAvailable(depChunkGroup, newAvailableModules)) { - return false; // break all modules are already available - } - dep.couldBeFiltered = false; - return true; - }; - - // Iterative traversing of the basic chunk graph - while (queue2.length) { - chunkGroup = queue2.dequeue(); - const info = chunkGroupInfoMap.get(chunkGroup); - const availableModulesToBeMerged = info.availableModulesToBeMerged; - let minAvailableModules = info.minAvailableModules; - - // 1. Get minimal available modules - // It doesn't make sense to traverse a chunk again with more available modules. - // This step calculates the minimal available modules and skips traversal when - // the list didn't shrink. - availableModulesToBeMerged.sort(bySetSize); - let changed = false; - for (const availableModules of availableModulesToBeMerged) { - if (minAvailableModules === undefined) { - minAvailableModules = new Set(availableModules); - info.minAvailableModules = minAvailableModules; - changed = true; - } else { - for (const m of minAvailableModules) { - if (!availableModules.has(m)) { - minAvailableModules.delete(m); - changed = true; - } - } - } - } - availableModulesToBeMerged.length = 0; - if (!changed) continue; - - // 2. Get the edges at this point of the graph - const deps = chunkDependencies.get(chunkGroup); - if (!deps) continue; - if (deps.length === 0) continue; - - // 3. Create a new Set of available modules at this points - newAvailableModules = new Set(minAvailableModules); - for (const chunk of chunkGroup.chunks) { - for (const m of chunk.modulesIterable) { - newAvailableModules.add(m); - } - } - - // 4. Foreach remaining edge - const nextChunkGroups = new Set(); - for (let i = 0; i < deps.length; i++) { - const dep = deps[i]; - - // Filter inline, rather than creating a new array from `.filter()` - if (!filterFn(dep)) { - continue; - } - const depChunkGroup = dep.chunkGroup; - const depBlock = dep.block; - - // 5. Connect block with chunk - GraphHelpers.connectDependenciesBlockAndChunkGroup( - depBlock, - depChunkGroup - ); - - // 6. Connect chunk with parent - GraphHelpers.connectChunkGroupParentAndChild(chunkGroup, depChunkGroup); - - nextChunkGroups.add(depChunkGroup); - } - - // 7. Enqueue further traversal - for (const nextChunkGroup of nextChunkGroups) { - let nextInfo = chunkGroupInfoMap.get(nextChunkGroup); - if (nextInfo === undefined) { - nextInfo = { - minAvailableModules: undefined, - availableModulesToBeMerged: [] - }; - chunkGroupInfoMap.set(nextChunkGroup, nextInfo); - } - nextInfo.availableModulesToBeMerged.push(newAvailableModules); - - // As queue deduplicates enqueued items this makes sure that a ChunkGroup - // is not enqueued twice - queue2.enqueue(nextChunkGroup); - } - } - - // Remove all unconnected chunk groups - for (const chunkGroup of allCreatedChunkGroups) { - if (chunkGroup.getNumberOfParents() === 0) { - for (const chunk of chunkGroup.chunks) { - const idx = this.chunks.indexOf(chunk); - if (idx >= 0) this.chunks.splice(idx, 1); - chunk.remove("unconnected"); - } - chunkGroup.remove("unconnected"); - } - } - } - /** * * @param {Module} module module relationship for removal diff --git a/lib/buildChunkGraph.js b/lib/buildChunkGraph.js new file mode 100644 index 00000000000..0cd0ca634cf --- /dev/null +++ b/lib/buildChunkGraph.js @@ -0,0 +1,683 @@ +/* + MIT License http://www.opensource.org/licenses/mit-license.php + Author Tobias Koppers @sokra +*/ + +"use strict"; + +const AsyncDependencyToInitialChunkError = require("./AsyncDependencyToInitialChunkError"); +const GraphHelpers = require("./GraphHelpers"); + +/** @typedef {import("./AsyncDependenciesBlock")} AsyncDependenciesBlock */ +/** @typedef {import("./Chunk")} Chunk */ +/** @typedef {import("./ChunkGroup")} ChunkGroup */ +/** @typedef {import("./Compilation")} Compilation */ +/** @typedef {import("./DependenciesBlock")} DependenciesBlock */ +/** @typedef {import("./Dependency")} Dependency */ +/** @typedef {import("./Entrypoint")} Entrypoint */ +/** @typedef {import("./Module")} Module */ + +/** + * @typedef {Object} QueueItem + * @property {number} action + * @property {DependenciesBlock} block + * @property {Module} module + * @property {Chunk} chunk + * @property {ChunkGroup} chunkGroup + */ + +/** + * @typedef {Object} ChunkGroupInfo + * @property {Set} minAvailableModules current minimal set of modules available at this point + * @property {boolean} minAvailableModulesOwned true, if minAvailableModules is owned and can be modified + * @property {Set[]} availableModulesToBeMerged enqueued updates to the minimal set of available modules + * @property {QueueItem[]} skippedItems queue items that were skipped because module is already available in parent chunks (need to reconsider when minAvailableModules is shrinking) + * @property {Set} resultingAvailableModules set of modules available including modules from this chunk group + */ + +/** + * @typedef {Object} ChunkGroupDep + * @property {AsyncDependenciesBlock} block referencing block + * @property {ChunkGroup} chunkGroup referenced chunk group + */ + +/** + * @template T + * @param {Set} a first set + * @param {Set} b second set + * @returns {number} cmp + */ +const bySetSize = (a, b) => { + return b.size - a.size; +}; + +/** + * Extracts simplified info from the modules and their dependencies + * @param {Compilation} compilation the compilation + * @returns {Map, blocks: AsyncDependenciesBlock[]}>} the mapping block to modules and inner blocks + */ +const extraceBlockInfoMap = compilation => { + /** @type {Map, blocks: AsyncDependenciesBlock[]}>} */ + const blockInfoMap = new Map(); + + /** + * @param {Dependency} d dependency to iterate over + * @returns {void} + */ + const iteratorDependency = d => { + // We skip Dependencies without Reference + const ref = compilation.getDependencyReference(currentModule, d); + if (!ref) { + return; + } + // We skip Dependencies without Module pointer + const refModule = ref.module; + if (!refModule) { + return; + } + // We skip weak Dependencies + if (ref.weak) { + return; + } + + blockInfoModules.add(refModule); + }; + + /** + * @param {AsyncDependenciesBlock} b blocks to prepare + * @returns {void} + */ + const iteratorBlockPrepare = b => { + blockInfoBlocks.push(b); + blockQueue.push(b); + }; + + /** @type {Module} */ + let currentModule; + /** @type {DependenciesBlock} */ + let block; + /** @type {DependenciesBlock[]} */ + let blockQueue; + /** @type {Set} */ + let blockInfoModules; + /** @type {AsyncDependenciesBlock[]} */ + let blockInfoBlocks; + + for (const module of compilation.modules) { + blockQueue = [module]; + currentModule = module; + while (blockQueue.length > 0) { + block = blockQueue.pop(); + blockInfoModules = new Set(); + blockInfoBlocks = []; + + if (block.variables) { + for (const variable of block.variables) { + for (const dep of variable.dependencies) iteratorDependency(dep); + } + } + + if (block.dependencies) { + for (const dep of block.dependencies) iteratorDependency(dep); + } + + if (block.blocks) { + for (const b of block.blocks) iteratorBlockPrepare(b); + } + + const blockInfo = { + modules: blockInfoModules, + blocks: blockInfoBlocks + }; + blockInfoMap.set(block, blockInfo); + } + } + + return blockInfoMap; +}; + +/** + * + * @param {Compilation} compilation the compilation + * @param {Entrypoint[]} inputChunkGroups input groups + * @param {Map} chunkGroupInfoMap mapping from chunk group to available modules + * @param {Map} chunkDependencies dependencies for chunk groups + * @param {Set} blocksWithNestedBlocks flag for blocks that have nested blocks + * @param {Set} allCreatedChunkGroups filled with all chunk groups that are created here + */ +const visitModules = ( + compilation, + inputChunkGroups, + chunkGroupInfoMap, + chunkDependencies, + blocksWithNestedBlocks, + allCreatedChunkGroups +) => { + const logger = compilation.getLogger("webpack.buildChunkGraph.visitModules"); + const { namedChunkGroups } = compilation; + + logger.time("prepare"); + const blockInfoMap = extraceBlockInfoMap(compilation); + + /** @type {Map} */ + const chunkGroupCounters = new Map(); + for (const chunkGroup of inputChunkGroups) { + chunkGroupCounters.set(chunkGroup, { + index: 0, + index2: 0 + }); + } + + let nextFreeModuleIndex = 0; + let nextFreeModuleIndex2 = 0; + + /** @type {Map} */ + const blockChunkGroups = new Map(); + + const ADD_AND_ENTER_MODULE = 0; + const ENTER_MODULE = 1; + const PROCESS_BLOCK = 2; + const LEAVE_MODULE = 3; + + /** + * @param {QueueItem[]} queue the queue array (will be mutated) + * @param {ChunkGroup} chunkGroup chunk group + * @returns {QueueItem[]} the queue array again + */ + const reduceChunkGroupToQueueItem = (queue, chunkGroup) => { + for (const chunk of chunkGroup.chunks) { + const module = chunk.entryModule; + queue.push({ + action: ENTER_MODULE, + block: module, + module, + chunk, + chunkGroup + }); + } + chunkGroupInfoMap.set(chunkGroup, { + minAvailableModules: new Set(), + minAvailableModulesOwned: true, + availableModulesToBeMerged: [], + skippedItems: [], + resultingAvailableModules: undefined + }); + return queue; + }; + + // Start with the provided modules/chunks + /** @type {QueueItem[]} */ + let queue = inputChunkGroups + .reduce(reduceChunkGroupToQueueItem, []) + .reverse(); + /** @type {Map>} */ + const queueConnect = new Map(); + /** @type {QueueItem[]} */ + let queueDelayed = []; + logger.timeEnd("prepare"); + + /** @type {Module} */ + let module; + /** @type {Chunk} */ + let chunk; + /** @type {ChunkGroup} */ + let chunkGroup; + /** @type {DependenciesBlock} */ + let block; + /** @type {Set} */ + let minAvailableModules; + /** @type {QueueItem[]} */ + let skippedItems; + + // For each async Block in graph + /** + * @param {AsyncDependenciesBlock} b iterating over each Async DepBlock + * @returns {void} + */ + const iteratorBlock = b => { + // 1. We create a chunk for this Block + // but only once (blockChunkGroups map) + let c = blockChunkGroups.get(b); + if (c === undefined) { + c = namedChunkGroups.get(b.chunkName); + if (c && c.isInitial()) { + compilation.errors.push( + new AsyncDependencyToInitialChunkError(b.chunkName, module, b.loc) + ); + c = chunkGroup; + } else { + c = compilation.addChunkInGroup( + b.groupOptions || b.chunkName, + module, + b.loc, + b.request + ); + chunkGroupCounters.set(c, { index: 0, index2: 0 }); + blockChunkGroups.set(b, c); + allCreatedChunkGroups.add(c); + } + } else { + // TODO webpack 5 remove addOptions check + if (c.addOptions) c.addOptions(b.groupOptions); + c.addOrigin(module, b.loc, b.request); + } + + // 2. We store the Block+Chunk mapping as dependency for the chunk + let deps = chunkDependencies.get(chunkGroup); + if (!deps) chunkDependencies.set(chunkGroup, (deps = [])); + deps.push({ + block: b, + chunkGroup: c + }); + + // 3. We create/update the chunk group info + let connectList = queueConnect.get(chunkGroup); + if (connectList === undefined) { + connectList = new Set(); + queueConnect.set(chunkGroup, connectList); + } + connectList.add(c); + + // 4. We enqueue the DependenciesBlock for traversal + queueDelayed.push({ + action: PROCESS_BLOCK, + block: b, + module: module, + chunk: c.chunks[0], + chunkGroup: c + }); + }; + + // Iterative traversal of the Module graph + // Recursive would be simpler to write but could result in Stack Overflows + while (queue.length) { + logger.time("visiting"); + while (queue.length) { + const queueItem = queue.pop(); + module = queueItem.module; + block = queueItem.block; + chunk = queueItem.chunk; + if (chunkGroup !== queueItem.chunkGroup) { + chunkGroup = queueItem.chunkGroup; + const chunkGroupInfo = chunkGroupInfoMap.get(chunkGroup); + minAvailableModules = chunkGroupInfo.minAvailableModules; + skippedItems = chunkGroupInfo.skippedItems; + } + + switch (queueItem.action) { + case ADD_AND_ENTER_MODULE: { + if (minAvailableModules.has(module)) { + // already in parent chunks + // skip it for now, but enqueue for rechecking when minAvailableModules shrinks + skippedItems.push(queueItem); + break; + } + // We connect Module and Chunk when not already done + if (chunk.addModule(module)) { + module.addChunk(chunk); + } else { + // already connected, skip it + break; + } + } + // fallthrough + case ENTER_MODULE: { + if (chunkGroup !== undefined) { + const index = chunkGroup.getModuleIndex(module); + if (index === undefined) { + chunkGroup.setModuleIndex( + module, + chunkGroupCounters.get(chunkGroup).index++ + ); + } + } + + if (module.index === null) { + module.index = nextFreeModuleIndex++; + } + + queue.push({ + action: LEAVE_MODULE, + block, + module, + chunk, + chunkGroup + }); + } + // fallthrough + case PROCESS_BLOCK: { + // get prepared block info + const blockInfo = blockInfoMap.get(block); + + // Buffer items because order need to be reverse to get indicies correct + const skipBuffer = []; + const queueBuffer = []; + // Traverse all referenced modules + for (const refModule of blockInfo.modules) { + if (chunk.containsModule(refModule)) { + // skip early if already connected + continue; + } + if (minAvailableModules.has(refModule)) { + // already in parent chunks, skip it for now + skipBuffer.push({ + action: ADD_AND_ENTER_MODULE, + block: refModule, + module: refModule, + chunk, + chunkGroup + }); + continue; + } + // enqueue the add and enter to enter in the correct order + // this is relevant with circular dependencies + queueBuffer.push({ + action: ADD_AND_ENTER_MODULE, + block: refModule, + module: refModule, + chunk, + chunkGroup + }); + } + // Add buffered items in reversed order + for (let i = skipBuffer.length - 1; i >= 0; i--) { + skippedItems.push(skipBuffer[i]); + } + for (let i = queueBuffer.length - 1; i >= 0; i--) { + queue.push(queueBuffer[i]); + } + + // Traverse all Blocks + for (const block of blockInfo.blocks) iteratorBlock(block); + + if (blockInfo.blocks.length > 0 && module !== block) { + blocksWithNestedBlocks.add(block); + } + break; + } + case LEAVE_MODULE: { + if (chunkGroup !== undefined) { + const index = chunkGroup.getModuleIndex2(module); + if (index === undefined) { + chunkGroup.setModuleIndex2( + module, + chunkGroupCounters.get(chunkGroup).index2++ + ); + } + } + + if (module.index2 === null) { + module.index2 = nextFreeModuleIndex2++; + } + break; + } + } + } + logger.timeEnd("visiting"); + logger.time("calculating available modules"); + + /** @type {Set} */ + const outdatedChunkGroupInfo = new Set(); + + // Figure out new parents for chunk groups + // to get new available modules for these children + for (const [chunkGroup, targets] of queueConnect) { + const info = chunkGroupInfoMap.get(chunkGroup); + let minAvailableModules = info.minAvailableModules; + + // 1. Create a new Set of available modules at this points + const resultingAvailableModules = new Set(minAvailableModules); + for (const chunk of chunkGroup.chunks) { + for (const m of chunk.modulesIterable) { + resultingAvailableModules.add(m); + } + } + info.resultingAvailableModules = resultingAvailableModules; + + // 2. Update chunk group info + for (const target of targets) { + let chunkGroupInfo = chunkGroupInfoMap.get(target); + if (chunkGroupInfo === undefined) { + chunkGroupInfo = { + minAvailableModules: undefined, + minAvailableModulesOwned: undefined, + availableModulesToBeMerged: [], + skippedItems: [], + resultingAvailableModules: undefined + }; + chunkGroupInfoMap.set(target, chunkGroupInfo); + } + chunkGroupInfo.availableModulesToBeMerged.push( + resultingAvailableModules + ); + outdatedChunkGroupInfo.add(chunkGroupInfo); + } + } + queueConnect.clear(); + logger.timeEnd("calculating available modules"); + + logger.time("merging available modules"); + // Execute the merge + for (const info of outdatedChunkGroupInfo) { + const availableModulesToBeMerged = info.availableModulesToBeMerged; + let minAvailableModules = info.minAvailableModules; + + // 1. Get minimal available modules + // It doesn't make sense to traverse a chunk again with more available modules. + // This step calculates the minimal available modules and skips traversal when + // the list didn't shrink. + if (availableModulesToBeMerged.length > 1) { + availableModulesToBeMerged.sort(bySetSize); + } + let changed = false; + for (const availableModules of availableModulesToBeMerged) { + if (minAvailableModules === undefined) { + minAvailableModules = availableModules; + info.minAvailableModules = minAvailableModules; + info.minAvailableModulesOwned = false; + changed = true; + } else { + if (info.minAvailableModulesOwned) { + // We own it and can modify it + for (const m of minAvailableModules) { + if (!availableModules.has(m)) { + minAvailableModules.delete(m); + changed = true; + } + } + } else { + for (const m of minAvailableModules) { + if (!availableModules.has(m)) { + // minAvailableModules need to be modified + // but we don't own it + // construct a new Set as intersection of minAvailableModules and availableModules + /** @type {Set} */ + const newSet = new Set(); + const iterator = minAvailableModules[Symbol.iterator](); + /** @type {IteratorResult} */ + let it; + while (!(it = iterator.next()).done) { + const module = it.value; + if (module === m) break; + newSet.add(module); + } + while (!(it = iterator.next()).done) { + const module = it.value; + if (availableModules.has(module)) { + newSet.add(module); + } + } + minAvailableModules = newSet; + info.minAvailableModulesOwned = true; + info.minAvailableModules = newSet; + changed = true; + break; + } + } + } + } + } + availableModulesToBeMerged.length = 0; + if (!changed) continue; + + // 2. Reconsider skipped items + for (const queueItem of info.skippedItems) { + queue.push(queueItem); + } + info.skippedItems.length = 0; + } + logger.timeEnd("merging available modules"); + + // Run queueDelayed when all items of the queue are processed + // This is important to get the global indicing correct + // Async blocks should be processed after all sync blocks are processed + if (queue.length === 0) { + const tempQueue = queue; + queue = queueDelayed.reverse(); + queueDelayed = tempQueue; + } + } +}; + +/** + * + * @param {Set} blocksWithNestedBlocks flag for blocks that have nested blocks + * @param {Map} chunkDependencies dependencies for chunk groups + * @param {Map} chunkGroupInfoMap mapping from chunk group to available modules + */ +const connectChunkGroups = ( + blocksWithNestedBlocks, + chunkDependencies, + chunkGroupInfoMap +) => { + /** @type {Set} */ + let resultingAvailableModules; + + /** + * Helper function to check if all modules of a chunk are available + * + * @param {ChunkGroup} chunkGroup the chunkGroup to scan + * @param {Set} availableModules the comparitor set + * @returns {boolean} return true if all modules of a chunk are available + */ + const areModulesAvailable = (chunkGroup, availableModules) => { + for (const chunk of chunkGroup.chunks) { + for (const module of chunk.modulesIterable) { + if (!availableModules.has(module)) return false; + } + } + return true; + }; + + // For each edge in the basic chunk graph + /** + * @param {ChunkGroupDep} dep the dependency used for filtering + * @returns {boolean} used to filter "edges" (aka Dependencies) that were pointing + * to modules that are already available. Also filters circular dependencies in the chunks graph + */ + const filterFn = dep => { + const depChunkGroup = dep.chunkGroup; + // TODO is this needed? + if (blocksWithNestedBlocks.has(dep.block)) return true; + if (areModulesAvailable(depChunkGroup, resultingAvailableModules)) { + return false; // break all modules are already available + } + return true; + }; + + // For all deps, check if chunk groups need to be connected + for (const [chunkGroup, deps] of chunkDependencies) { + if (deps.length === 0) continue; + + // 1. Get info from chunk group info map + const info = chunkGroupInfoMap.get(chunkGroup); + resultingAvailableModules = info.resultingAvailableModules; + + // 2. Foreach edge + for (let i = 0; i < deps.length; i++) { + const dep = deps[i]; + + // Filter inline, rather than creating a new array from `.filter()` + // TODO check if inlining filterFn makes sense here + if (!filterFn(dep)) { + continue; + } + const depChunkGroup = dep.chunkGroup; + const depBlock = dep.block; + + // 5. Connect block with chunk + GraphHelpers.connectDependenciesBlockAndChunkGroup( + depBlock, + depChunkGroup + ); + + // 6. Connect chunk with parent + GraphHelpers.connectChunkGroupParentAndChild(chunkGroup, depChunkGroup); + } + } +}; + +/** + * Remove all unconnected chunk groups + * @param {Compilation} compilation the compilation + * @param {Iterable} allCreatedChunkGroups all chunk groups that where created before + */ +const cleanupUnconnectedGroups = (compilation, allCreatedChunkGroups) => { + for (const chunkGroup of allCreatedChunkGroups) { + if (chunkGroup.getNumberOfParents() === 0) { + for (const chunk of chunkGroup.chunks) { + const idx = compilation.chunks.indexOf(chunk); + if (idx >= 0) compilation.chunks.splice(idx, 1); + chunk.remove("unconnected"); + } + chunkGroup.remove("unconnected"); + } + } +}; + +/** + * This method creates the Chunk graph from the Module graph + * @param {Compilation} compilation the compilation + * @param {Entrypoint[]} inputChunkGroups chunk groups which are processed + * @returns {void} + */ +const buildChunkGraph = (compilation, inputChunkGroups) => { + // SHARED STATE + + /** @type {Map} */ + const chunkDependencies = new Map(); + + /** @type {Set} */ + const allCreatedChunkGroups = new Set(); + + /** @type {Map} */ + const chunkGroupInfoMap = new Map(); + + /** @type {Set} */ + const blocksWithNestedBlocks = new Set(); + + // PART ONE + + visitModules( + compilation, + inputChunkGroups, + chunkGroupInfoMap, + chunkDependencies, + blocksWithNestedBlocks, + allCreatedChunkGroups + ); + + // PART TWO + + connectChunkGroups( + blocksWithNestedBlocks, + chunkDependencies, + chunkGroupInfoMap + ); + + // Cleaup work + + cleanupUnconnectedGroups(compilation, allCreatedChunkGroups); +}; + +module.exports = buildChunkGraph;