diff --git a/lib/Chunk.js b/lib/Chunk.js index 16e28a62a76..db644ee0026 100644 --- a/lib/Chunk.js +++ b/lib/Chunk.js @@ -385,7 +385,7 @@ class Chunk { /** * * @param {Chunk} otherChunk the chunk to integrate with - * @param {ModuleReason} reason reason why the module is being integrated + * @param {string} reason reason why the module is being integrated * @returns {boolean} returns true or false if integration succeeds or fails */ integrate(otherChunk, reason) { @@ -462,6 +462,11 @@ class Chunk { return false; } + /** + * @param {Chunk} a chunk + * @param {Chunk} b chunk + * @returns {boolean} true, if a is always available when b is reached + */ const isAvailable = (a, b) => { const queue = new Set(b.groupsIterable); for (const chunkGroup of queue) { diff --git a/lib/optimize/LimitChunkCountPlugin.js b/lib/optimize/LimitChunkCountPlugin.js index 0f8bd6dcbe5..87f28491f4e 100644 --- a/lib/optimize/LimitChunkCountPlugin.js +++ b/lib/optimize/LimitChunkCountPlugin.js @@ -6,8 +6,33 @@ const validateOptions = require("schema-utils"); const schema = require("../../schemas/plugins/optimize/LimitChunkCountPlugin.json"); +const LazyBucketSortedSet = require("../util/LazyBucketSortedSet"); /** @typedef {import("../../declarations/plugins/optimize/LimitChunkCountPlugin").LimitChunkCountPluginOptions} LimitChunkCountPluginOptions */ +/** @typedef {import("../Chunk")} Chunk */ +/** @typedef {import("../Compiler")} Compiler */ + +/** + * @typedef {Object} ChunkCombination + * @property {boolean} deleted this is set to true when combination was removed + * @property {number} sizeDiff + * @property {number} integratedSize + * @property {Chunk} a + * @property {Chunk} b + * @property {number} aIdx + * @property {number} bIdx + * @property {number} aSize + * @property {number} bSize + */ + +const addToSetMap = (map, key, value) => { + const set = map.get(key); + if (set === undefined) { + map.set(key, new Set([value])); + } else { + set.add(value); + } +}; class LimitChunkCountPlugin { /** @@ -19,6 +44,11 @@ class LimitChunkCountPlugin { validateOptions(schema, options, "Limit Chunk Count Plugin"); this.options = options; } + + /** + * @param {Compiler} compiler the webpack compiler + * @returns {void} + */ apply(compiler) { const options = this.options; compiler.hooks.compilation.tap("LimitChunkCountPlugin", compilation => { @@ -30,47 +60,169 @@ class LimitChunkCountPlugin { if (maxChunks < 1) return; if (chunks.length <= maxChunks) return; + let remainingChunksToMerge = chunks.length - maxChunks; + + // order chunks in a deterministic way const orderedChunks = chunks.slice().sort((a, b) => a.compareTo(b)); - const sortedExtendedPairCombinations = orderedChunks - .reduce((combinations, a, idx) => { - // create combination pairs - for (let i = 0; i < idx; i++) { - const b = orderedChunks[i]; - combinations.push([b, a]); - } - return combinations; - }, []) - .map(pair => { - // extend combination pairs with size and integrated size - const a = pair[0].size(options); - const b = pair[1].size(options); - const ab = pair[0].integratedSize(pair[1], options); - return [a + b - ab, ab, pair[0], pair[1], a, b]; - }) - .filter(extendedPair => { + // create a lazy sorted data structure to keep all combinations + // this is large. Size = chunks * (chunks - 1) / 2 + // It uses a multi layer bucket sort plus normal sort in the last layer + // It's also lazy so only accessed buckets are sorted + const combinations = new LazyBucketSortedSet( + // Layer 1: ordered by largest size benefit + c => c.sizeDiff, + (a, b) => b - a, + // Layer 2: ordered by smallest combined size + c => c.integratedSize, + (a, b) => a - b, + // Layer 3: ordered by position difference in orderedChunk (-> to be deterministic) + c => c.bIdx - c.aIdx, + (a, b) => a - b, + // Layer 4: ordered by position in orderedChunk (-> to be deterministic) + (a, b) => a.bIdx - b.bIdx + ); + + // we keep a mappng from chunk to all combinations + // but this mapping is not kept up-to-date with deletions + // so `deleted` flag need to be considered when iterating this + /** @type {Map>} */ + const combinationsByChunk = new Map(); + + orderedChunks.forEach((b, bIdx) => { + // create combination pairs with size and integrated size + for (let aIdx = 0; aIdx < bIdx; aIdx++) { + const a = orderedChunks[aIdx]; + const integratedSize = a.integratedSize(b, options); + // filter pairs that do not have an integratedSize // meaning they can NOT be integrated! - return extendedPair[1] !== false; - }) - .sort((a, b) => { - // sadly javascript does an inplace sort here - // sort them by size - const diff1 = b[0] - a[0]; - if (diff1 !== 0) return diff1; - const diff2 = a[1] - b[1]; - if (diff2 !== 0) return diff2; - const diff3 = a[2].compareTo(b[2]); - if (diff3 !== 0) return diff3; - return a[3].compareTo(b[3]); - }); - - const pair = sortedExtendedPairCombinations[0]; - - if (pair && pair[2].integrate(pair[3], "limit")) { - chunks.splice(chunks.indexOf(pair[3]), 1); - return true; + if (integratedSize === false) continue; + + const aSize = a.size(options); + const bSize = b.size(options); + const c = { + deleted: false, + sizeDiff: aSize + bSize - integratedSize, + integratedSize, + a, + b, + aIdx, + bIdx, + aSize, + bSize + }; + combinations.add(c); + addToSetMap(combinationsByChunk, a, c); + addToSetMap(combinationsByChunk, b, c); + } + return combinations; + }); + + // list of modified chunks during this run + // combinations affected by this change are skipped to allow + // futher optimizations + /** @type {Set} */ + const modifiedChunks = new Set(); + + let changed = false; + // eslint-disable-next-line no-constant-condition + loop: while (true) { + const combination = combinations.popFirst(); + if (combination === undefined) break; + + combination.deleted = true; + const { a, b, integratedSize } = combination; + + // skip over pair when + // one of the already merged chunks is a parent of one of the chunks + if (modifiedChunks.size > 0) { + const queue = new Set(a.groupsIterable); + for (const group of b.groupsIterable) { + queue.add(group); + } + for (const group of queue) { + for (const mChunk of modifiedChunks) { + if (mChunk !== a && mChunk !== b && mChunk.isInGroup(group)) { + // This is a potential pair which needs recalculation + // We can't do that now, but it merge before following pairs + // so we leave space for it, and consider chunks as modified + // just for the worse case + remainingChunksToMerge--; + if (remainingChunksToMerge <= 0) break loop; + modifiedChunks.add(a); + modifiedChunks.add(b); + continue loop; + } + } + for (const parent of group.parentsIterable) { + queue.add(parent); + } + } + } + + // merge the chunks + if (a.integrate(b, "limit")) { + chunks.splice(chunks.indexOf(b), 1); + + // flag chunk a as modified as further optimization are possible for all children here + modifiedChunks.add(a); + + changed = true; + remainingChunksToMerge--; + if (remainingChunksToMerge <= 0) break; + + // Update all affected combinations + // delete all combination with the removed chunk + // we will use combinations with the kept chunk instead + for (const combination of combinationsByChunk.get(b)) { + if (combination.deleted) continue; + combination.deleted = true; + combinations.delete(combination); + } + + // Update combinations with the kept chunk with new sizes + for (const combination of combinationsByChunk.get(a)) { + if (combination.deleted) continue; + if (combination.a === a) { + // Update size + const newIntegratedSize = a.integratedSize( + combination.b, + options + ); + if (newIntegratedSize === false) { + combination.deleted = true; + combinations.delete(combination); + continue; + } + const finishUpdate = combinations.startUpdate(combination); + combination.integratedSize = newIntegratedSize; + combination.aSize = integratedSize; + combination.sizeDiff = + combination.bSize + integratedSize - newIntegratedSize; + finishUpdate(); + } else if (combination.b === a) { + // Update size + const newIntegratedSize = combination.a.integratedSize( + a, + options + ); + if (newIntegratedSize === false) { + combination.deleted = true; + combinations.delete(combination); + continue; + } + const finishUpdate = combinations.startUpdate(combination); + combination.integratedSize = newIntegratedSize; + combination.bSize = integratedSize; + combination.sizeDiff = + integratedSize + combination.aSize - newIntegratedSize; + finishUpdate(); + } + } + } } + if (changed) return true; } ); }); diff --git a/lib/util/LazyBucketSortedSet.js b/lib/util/LazyBucketSortedSet.js new file mode 100644 index 00000000000..61d3d3f2edc --- /dev/null +++ b/lib/util/LazyBucketSortedSet.js @@ -0,0 +1,235 @@ +/* + MIT License http://www.opensource.org/licenses/mit-license.php + Author Tobias Koppers @sokra +*/ + +"use strict"; + +const SortableSet = require("./SortableSet"); + +/** + * @template T + * @template K + * Multi layer bucket sorted set + * Supports adding non-existing items (DO NOT ADD ITEM TWICE) + * Supports removing exiting items (DO NOT REMOVE ITEM NOT IN SET) + * Supports popping the first items according to defined order + * Supports iterating all items without order + * Supports updating an item in an efficient way + * Supports size property, which is the number of items + * Items are lazy partially sorted when needed + */ +class LazyBucketSortedSet { + /** + * @param {function(T): K} getKey function to get key from item + * @param {function(K, K): number} comparator comparator to sort keys + * @param {...((function(T): any) | (function(any, any): number))} args more pairs of getKey and comparator plus optional final comparator for the last layer + */ + constructor(getKey, comparator, ...args) { + this._getKey = getKey; + this._innerArgs = args; + this._leaf = args.length <= 1; + this._keys = new SortableSet(undefined, comparator); + /** @type {Map | SortableSet>} */ + this._map = new Map(); + this._unsortedItems = new Set(); + this.size = 0; + } + + /** + * @param {T} item an item + * @returns {void} + */ + add(item) { + this.size++; + this._unsortedItems.add(item); + } + + /** + * @param {K} key key of item + * @param {T} item the item + * @returns {void} + */ + _addInternal(key, item) { + let entry = this._map.get(key); + if (entry === undefined) { + entry = this._leaf + ? new SortableSet(undefined, this._innerArgs[0]) + : new /** @type {any} */ (LazyBucketSortedSet)(...this._innerArgs); + this._keys.add(key); + this._map.set(key, entry); + } + entry.add(item); + } + + /** + * @param {T} item an item + * @returns {void} + */ + delete(item) { + this.size--; + if (this._unsortedItems.has(item)) { + this._unsortedItems.delete(item); + return; + } + const key = this._getKey(item); + const entry = this._map.get(key); + entry.delete(item); + if (entry.size === 0) { + this._deleteKey(key); + } + } + + /** + * @param {K} key key to be removed + * @returns {void} + */ + _deleteKey(key) { + this._keys.delete(key); + this._map.delete(key); + } + + /** + * @returns {T | undefined} an item + */ + popFirst() { + if (this.size === 0) return undefined; + this.size--; + if (this._unsortedItems.size > 0) { + for (const item of this._unsortedItems) { + const key = this._getKey(item); + this._addInternal(key, item); + } + this._unsortedItems.clear(); + } + this._keys.sort(); + const key = this._keys.values().next().value; + const entry = this._map.get(key); + if (this._leaf) { + const leafEntry = /** @type {SortableSet} */ (entry); + leafEntry.sort(); + const item = leafEntry.values().next().value; + leafEntry.delete(item); + if (leafEntry.size === 0) { + this._deleteKey(key); + } + return item; + } else { + const nodeEntry = /** @type {LazyBucketSortedSet} */ (entry); + const item = nodeEntry.popFirst(); + if (nodeEntry.size === 0) { + this._deleteKey(key); + } + return item; + } + } + + /** + * @param {T} item to be updated item + * @returns {function(true=): void} finish update + */ + startUpdate(item) { + if (this._unsortedItems.has(item)) { + return remove => { + if (remove) { + this._unsortedItems.delete(item); + this.size--; + return; + } + }; + } + const key = this._getKey(item); + if (this._leaf) { + const oldEntry = /** @type {SortableSet} */ (this._map.get(key)); + return remove => { + if (remove) { + this.size--; + oldEntry.delete(item); + if (oldEntry.size === 0) { + this._deleteKey(key); + } + return; + } + const newKey = this._getKey(item); + if (key === newKey) { + // This flags the sortable set as unordered + oldEntry.add(item); + } else { + oldEntry.delete(item); + if (oldEntry.size === 0) { + this._deleteKey(key); + } + this._addInternal(newKey, item); + } + }; + } else { + const oldEntry = /** @type {LazyBucketSortedSet} */ (this._map.get( + key + )); + const finishUpdate = oldEntry.startUpdate(item); + return remove => { + if (remove) { + this.size--; + finishUpdate(true); + if (oldEntry.size === 0) { + this._deleteKey(key); + } + return; + } + const newKey = this._getKey(item); + if (key === newKey) { + finishUpdate(); + } else { + finishUpdate(true); + if (oldEntry.size === 0) { + this._deleteKey(key); + } + this._addInternal(newKey, item); + } + }; + } + } + + /** + * @param {Iterator[]} iterators list of iterators to append to + * @returns {void} + */ + _appendIterators(iterators) { + if (this._unsortedItems.size > 0) + iterators.push(this._unsortedItems[Symbol.iterator]()); + for (const key of this._keys) { + const entry = this._map.get(key); + if (this._leaf) { + const leafEntry = /** @type {SortableSet} */ (entry); + const iterator = leafEntry[Symbol.iterator](); + iterators.push(iterator); + } else { + const nodeEntry = /** @type {LazyBucketSortedSet} */ (entry); + nodeEntry._appendIterators(iterators); + } + } + } + + /** + * @returns {Iterator} the iterator + */ + [Symbol.iterator]() { + const iterators = []; + this._appendIterators(iterators); + iterators.reverse(); + let currentIterator = iterators.pop(); + return { + next: () => { + const res = currentIterator.next(); + if (res.done) { + if (iterators.length === 0) return res; + currentIterator = iterators.pop(); + return currentIterator.next(); + } + return res; + } + }; + } +} + +module.exports = LazyBucketSortedSet;