From 2e0ce0d1a9c3f4bd29c16ec9aa52bc068c8178e5 Mon Sep 17 00:00:00 2001 From: Tobias Koppers Date: Fri, 26 Jul 2019 08:02:30 +0200 Subject: [PATCH] less memory, reduce timing verbosity --- lib/buildChunkGraph.js | 202 +++++++++++++++++++++-------------------- 1 file changed, 104 insertions(+), 98 deletions(-) diff --git a/lib/buildChunkGraph.js b/lib/buildChunkGraph.js index 0cd0ca634cf..bbd6128733e 100644 --- a/lib/buildChunkGraph.js +++ b/lib/buildChunkGraph.js @@ -212,8 +212,11 @@ const visitModules = ( .reverse(); /** @type {Map>} */ const queueConnect = new Map(); + /** @type {Set} */ + const outdatedChunkGroupInfo = new Set(); /** @type {QueueItem[]} */ let queueDelayed = []; + logger.timeEnd("prepare"); /** @type {Module} */ @@ -414,119 +417,122 @@ const visitModules = ( } } logger.timeEnd("visiting"); - logger.time("calculating available modules"); - /** @type {Set} */ - const outdatedChunkGroupInfo = new Set(); + if (queueConnect.size > 0) { + logger.time("calculating available modules"); - // Figure out new parents for chunk groups - // to get new available modules for these children - for (const [chunkGroup, targets] of queueConnect) { - const info = chunkGroupInfoMap.get(chunkGroup); - let minAvailableModules = info.minAvailableModules; + // Figure out new parents for chunk groups + // to get new available modules for these children + for (const [chunkGroup, targets] of queueConnect) { + const info = chunkGroupInfoMap.get(chunkGroup); + let minAvailableModules = info.minAvailableModules; - // 1. Create a new Set of available modules at this points - const resultingAvailableModules = new Set(minAvailableModules); - for (const chunk of chunkGroup.chunks) { - for (const m of chunk.modulesIterable) { - resultingAvailableModules.add(m); + // 1. Create a new Set of available modules at this points + const resultingAvailableModules = new Set(minAvailableModules); + for (const chunk of chunkGroup.chunks) { + for (const m of chunk.modulesIterable) { + resultingAvailableModules.add(m); + } } - } - info.resultingAvailableModules = resultingAvailableModules; - - // 2. Update chunk group info - for (const target of targets) { - let chunkGroupInfo = chunkGroupInfoMap.get(target); - if (chunkGroupInfo === undefined) { - chunkGroupInfo = { - minAvailableModules: undefined, - minAvailableModulesOwned: undefined, - availableModulesToBeMerged: [], - skippedItems: [], - resultingAvailableModules: undefined - }; - chunkGroupInfoMap.set(target, chunkGroupInfo); + info.resultingAvailableModules = resultingAvailableModules; + + // 2. Update chunk group info + for (const target of targets) { + let chunkGroupInfo = chunkGroupInfoMap.get(target); + if (chunkGroupInfo === undefined) { + chunkGroupInfo = { + minAvailableModules: undefined, + minAvailableModulesOwned: undefined, + availableModulesToBeMerged: [], + skippedItems: [], + resultingAvailableModules: undefined + }; + chunkGroupInfoMap.set(target, chunkGroupInfo); + } + chunkGroupInfo.availableModulesToBeMerged.push( + resultingAvailableModules + ); + outdatedChunkGroupInfo.add(chunkGroupInfo); } - chunkGroupInfo.availableModulesToBeMerged.push( - resultingAvailableModules - ); - outdatedChunkGroupInfo.add(chunkGroupInfo); - } - } - queueConnect.clear(); - logger.timeEnd("calculating available modules"); - - logger.time("merging available modules"); - // Execute the merge - for (const info of outdatedChunkGroupInfo) { - const availableModulesToBeMerged = info.availableModulesToBeMerged; - let minAvailableModules = info.minAvailableModules; - - // 1. Get minimal available modules - // It doesn't make sense to traverse a chunk again with more available modules. - // This step calculates the minimal available modules and skips traversal when - // the list didn't shrink. - if (availableModulesToBeMerged.length > 1) { - availableModulesToBeMerged.sort(bySetSize); } - let changed = false; - for (const availableModules of availableModulesToBeMerged) { - if (minAvailableModules === undefined) { - minAvailableModules = availableModules; - info.minAvailableModules = minAvailableModules; - info.minAvailableModulesOwned = false; - changed = true; - } else { - if (info.minAvailableModulesOwned) { - // We own it and can modify it - for (const m of minAvailableModules) { - if (!availableModules.has(m)) { - minAvailableModules.delete(m); - changed = true; - } - } - } else { - for (const m of minAvailableModules) { - if (!availableModules.has(m)) { - // minAvailableModules need to be modified - // but we don't own it - // construct a new Set as intersection of minAvailableModules and availableModules - /** @type {Set} */ - const newSet = new Set(); - const iterator = minAvailableModules[Symbol.iterator](); - /** @type {IteratorResult} */ - let it; - while (!(it = iterator.next()).done) { - const module = it.value; - if (module === m) break; - newSet.add(module); + queueConnect.clear(); + logger.timeEnd("calculating available modules"); + + if (outdatedChunkGroupInfo.size > 0) { + logger.time("merging available modules"); + // Execute the merge + for (const info of outdatedChunkGroupInfo) { + const availableModulesToBeMerged = info.availableModulesToBeMerged; + let minAvailableModules = info.minAvailableModules; + + // 1. Get minimal available modules + // It doesn't make sense to traverse a chunk again with more available modules. + // This step calculates the minimal available modules and skips traversal when + // the list didn't shrink. + if (availableModulesToBeMerged.length > 1) { + availableModulesToBeMerged.sort(bySetSize); + } + let changed = false; + for (const availableModules of availableModulesToBeMerged) { + if (minAvailableModules === undefined) { + minAvailableModules = availableModules; + info.minAvailableModules = minAvailableModules; + info.minAvailableModulesOwned = false; + changed = true; + } else { + if (info.minAvailableModulesOwned) { + // We own it and can modify it + for (const m of minAvailableModules) { + if (!availableModules.has(m)) { + minAvailableModules.delete(m); + changed = true; + } } - while (!(it = iterator.next()).done) { - const module = it.value; - if (availableModules.has(module)) { - newSet.add(module); + } else { + for (const m of minAvailableModules) { + if (!availableModules.has(m)) { + // minAvailableModules need to be modified + // but we don't own it + // construct a new Set as intersection of minAvailableModules and availableModules + /** @type {Set} */ + const newSet = new Set(); + const iterator = minAvailableModules[Symbol.iterator](); + /** @type {IteratorResult} */ + let it; + while (!(it = iterator.next()).done) { + const module = it.value; + if (module === m) break; + newSet.add(module); + } + while (!(it = iterator.next()).done) { + const module = it.value; + if (availableModules.has(module)) { + newSet.add(module); + } + } + minAvailableModules = newSet; + info.minAvailableModulesOwned = true; + info.minAvailableModules = newSet; + changed = true; + break; } } - minAvailableModules = newSet; - info.minAvailableModulesOwned = true; - info.minAvailableModules = newSet; - changed = true; - break; } } } - } - } - availableModulesToBeMerged.length = 0; - if (!changed) continue; + availableModulesToBeMerged.length = 0; + if (!changed) continue; - // 2. Reconsider skipped items - for (const queueItem of info.skippedItems) { - queue.push(queueItem); + // 2. Reconsider skipped items + for (const queueItem of info.skippedItems) { + queue.push(queueItem); + } + info.skippedItems.length = 0; + } + outdatedChunkGroupInfo.clear(); + logger.timeEnd("merging available modules"); } - info.skippedItems.length = 0; } - logger.timeEnd("merging available modules"); // Run queueDelayed when all items of the queue are processed // This is important to get the global indicing correct