diff --git a/lib/Compilation.js b/lib/Compilation.js index 2a2d0adc6c6..0873eb254c2 100644 --- a/lib/Compilation.js +++ b/lib/Compilation.js @@ -22,6 +22,7 @@ const Dependency = require("./Dependency"); const ChunkRenderError = require("./ChunkRenderError"); const CachedSource = require("webpack-sources").CachedSource; const Stats = require("./Stats"); +const Semaphore = require("./util/Semaphore"); function byId(a, b) { if(a.id < b.id) return -1; @@ -62,6 +63,8 @@ class Compilation extends Tapable { this.hotUpdateChunkTemplate = new HotUpdateChunkTemplate(this.outputOptions); this.moduleTemplate = new ModuleTemplate(this.outputOptions); + this.semaphore = new Semaphore(options.parallelism || 100); + this.entries = []; this.preparedChunks = []; this.entrypoints = {}; @@ -229,120 +232,128 @@ class Compilation extends Tapable { callback(); }; - const factory = item[0]; - factory.create({ - contextInfo: { - issuer: module.nameForCondition && module.nameForCondition(), - compiler: _this.compiler.name - }, - context: module.context, - dependencies: dependencies - }, function factoryCallback(err, dependentModule) { - let afterFactory; - - function isOptional() { - return dependencies.filter(d => !d.optional).length === 0; - } + _this.semaphore.acquire(() => { + const factory = item[0]; + factory.create({ + contextInfo: { + issuer: module.nameForCondition && module.nameForCondition(), + compiler: _this.compiler.name + }, + context: module.context, + dependencies: dependencies + }, function factoryCallback(err, dependentModule) { + let afterFactory; + + function isOptional() { + return dependencies.filter(d => !d.optional).length === 0; + } - function errorOrWarningAndCallback(err) { - if(isOptional()) { - return warningAndCallback(err); - } else { - return errorAndCallback(err); + function errorOrWarningAndCallback(err) { + if(isOptional()) { + return warningAndCallback(err); + } else { + return errorAndCallback(err); + } } - } - function iterationDependencies(depend) { - for(let index = 0; index < depend.length; index++) { - const dep = depend[index]; - dep.module = dependentModule; - dependentModule.addReason(module, dep); + function iterationDependencies(depend) { + for(let index = 0; index < depend.length; index++) { + const dep = depend[index]; + dep.module = dependentModule; + dependentModule.addReason(module, dep); + } } - } - if(err) { - return errorOrWarningAndCallback(new ModuleNotFoundError(module, err, dependencies)); - } - if(!dependentModule) { - return process.nextTick(callback); - } - if(_this.profile) { - if(!dependentModule.profile) { - dependentModule.profile = {}; + if(err) { + _this.semaphore.release(); + return errorOrWarningAndCallback(new ModuleNotFoundError(module, err, dependencies)); + } + if(!dependentModule) { + _this.semaphore.release(); + return process.nextTick(callback); + } + if(_this.profile) { + if(!dependentModule.profile) { + dependentModule.profile = {}; + } + afterFactory = Date.now(); + dependentModule.profile.factory = afterFactory - start; } - afterFactory = Date.now(); - dependentModule.profile.factory = afterFactory - start; - } - dependentModule.issuer = module; - const newModule = _this.addModule(dependentModule, cacheGroup); + dependentModule.issuer = module; + const newModule = _this.addModule(dependentModule, cacheGroup); - if(!newModule) { // from cache - dependentModule = _this.getModule(dependentModule); + if(!newModule) { // from cache + dependentModule = _this.getModule(dependentModule); - if(dependentModule.optional) { - dependentModule.optional = isOptional(); - } + if(dependentModule.optional) { + dependentModule.optional = isOptional(); + } - iterationDependencies(dependencies); + iterationDependencies(dependencies); - if(_this.profile) { - if(!module.profile) { - module.profile = {}; - } - const time = Date.now() - start; - if(!module.profile.dependencies || time > module.profile.dependencies) { - module.profile.dependencies = time; + if(_this.profile) { + if(!module.profile) { + module.profile = {}; + } + const time = Date.now() - start; + if(!module.profile.dependencies || time > module.profile.dependencies) { + module.profile.dependencies = time; + } } + + _this.semaphore.release(); + return process.nextTick(callback); } - return process.nextTick(callback); - } + if(newModule instanceof Module) { + if(_this.profile) { + newModule.profile = dependentModule.profile; + } - if(newModule instanceof Module) { - if(_this.profile) { - newModule.profile = dependentModule.profile; - } + newModule.optional = isOptional(); + newModule.issuer = dependentModule.issuer; + dependentModule = newModule; - newModule.optional = isOptional(); - newModule.issuer = dependentModule.issuer; - dependentModule = newModule; + iterationDependencies(dependencies); - iterationDependencies(dependencies); + if(_this.profile) { + const afterBuilding = Date.now(); + module.profile.building = afterBuilding - afterFactory; + } - if(_this.profile) { - const afterBuilding = Date.now(); - module.profile.building = afterBuilding - afterFactory; + _this.semaphore.release(); + if(recursive) { + return process.nextTick(_this.processModuleDependencies.bind(_this, dependentModule, callback)); + } else { + return process.nextTick(callback); + } } - if(recursive) { - return process.nextTick(_this.processModuleDependencies.bind(_this, dependentModule, callback)); - } else { - return process.nextTick(callback); - } - } + dependentModule.optional = isOptional(); - dependentModule.optional = isOptional(); + iterationDependencies(dependencies); - iterationDependencies(dependencies); + _this.buildModule(dependentModule, isOptional(), module, dependencies, err => { + if(err) { + _this.semaphore.release(); + return errorOrWarningAndCallback(err); + } - _this.buildModule(dependentModule, isOptional(), module, dependencies, err => { - if(err) { - return errorOrWarningAndCallback(err); - } + if(_this.profile) { + const afterBuilding = Date.now(); + dependentModule.profile.building = afterBuilding - afterFactory; + } - if(_this.profile) { - const afterBuilding = Date.now(); - dependentModule.profile.building = afterBuilding - afterFactory; - } + _this.semaphore.release(); + if(recursive) { + _this.processModuleDependencies(dependentModule, callback); + } else { + return callback(); + } + }); - if(recursive) { - _this.processModuleDependencies(dependentModule, callback); - } else { - return callback(); - } }); - }); }, function finalCallbackAddModuleDependencies(err) { // In V8, the Error objects keep a reference to the functions on the stack. These warnings & @@ -379,79 +390,85 @@ class Compilation extends Tapable { throw new Error(`No dependency factory available for this dependency type: ${dependency.constructor.name}`); } - moduleFactory.create({ - contextInfo: { - issuer: "", - compiler: this.compiler.name - }, - context: context, - dependencies: [dependency] - }, (err, module) => { - if(err) { - return errorAndCallback(new EntryModuleNotFoundError(err)); - } - - let afterFactory; - - if(this.profile) { - if(!module.profile) { - module.profile = {}; + this.semaphore.acquire(() => { + moduleFactory.create({ + contextInfo: { + issuer: "", + compiler: this.compiler.name + }, + context: context, + dependencies: [dependency] + }, (err, module) => { + if(err) { + this.semaphore.release(); + return errorAndCallback(new EntryModuleNotFoundError(err)); } - afterFactory = Date.now(); - module.profile.factory = afterFactory - start; - } - const result = this.addModule(module); - if(!result) { - module = this.getModule(module); - - onModule(module); + let afterFactory; if(this.profile) { - const afterBuilding = Date.now(); - module.profile.building = afterBuilding - afterFactory; + if(!module.profile) { + module.profile = {}; + } + afterFactory = Date.now(); + module.profile.factory = afterFactory - start; } - return callback(null, module); - } + const result = this.addModule(module); + if(!result) { + module = this.getModule(module); - if(result instanceof Module) { - if(this.profile) { - result.profile = module.profile; - } + onModule(module); - module = result; + if(this.profile) { + const afterBuilding = Date.now(); + module.profile.building = afterBuilding - afterFactory; + } - onModule(module); + this.semaphore.release(); + return callback(null, module); + } - moduleReady.call(this); - return; - } + if(result instanceof Module) { + if(this.profile) { + result.profile = module.profile; + } - onModule(module); + module = result; - this.buildModule(module, false, null, null, (err) => { - if(err) { - return errorAndCallback(err); - } + onModule(module); - if(this.profile) { - const afterBuilding = Date.now(); - module.profile.building = afterBuilding - afterFactory; + moduleReady.call(this); + return; } - moduleReady.call(this); - }); + onModule(module); - function moduleReady() { - this.processModuleDependencies(module, err => { + this.buildModule(module, false, null, null, (err) => { if(err) { - return callback(err); + this.semaphore.release(); + return errorAndCallback(err); } - return callback(null, module); + if(this.profile) { + const afterBuilding = Date.now(); + module.profile.building = afterBuilding - afterFactory; + } + + moduleReady.call(this); }); - } + + function moduleReady() { + this.semaphore.release(); + this.processModuleDependencies(module, err => { + if(err) { + return callback(err); + } + + return callback(null, module); + }); + } + }); }); } diff --git a/lib/util/Semaphore.js b/lib/util/Semaphore.js new file mode 100644 index 00000000000..c8cab0a4516 --- /dev/null +++ b/lib/util/Semaphore.js @@ -0,0 +1,32 @@ +/* + MIT License http://www.opensource.org/licenses/mit-license.php + Author Tobias Koppers @sokra +*/ +"use strict"; + +class Semaphore { + constructor(available) { + this.available = available; + this.waiters = []; + } + + acquire(callback) { + if(this.available > 0) { + this.available--; + callback(); + } else { + this.waiters.push(callback); + } + } + + release() { + if(this.waiters.length > 0) { + const callback = this.waiters.pop(); + process.nextTick(callback); + } else { + this.available++; + } + } +} + +module.exports = Semaphore; diff --git a/schemas/webpackOptionsSchema.json b/schemas/webpackOptionsSchema.json index 35c30e1fa5b..ada46dd6ffb 100644 --- a/schemas/webpackOptionsSchema.json +++ b/schemas/webpackOptionsSchema.json @@ -900,6 +900,11 @@ "output": { "$ref": "#/definitions/output" }, + "parallelism": { + "description": "The number of parallel processed modules in the compilation.", + "minimum": 1, + "type": "number" + }, "performance": { "description": "Configuration for web performance recommendations.", "anyOf": [