Skip to content

Commit

Permalink
Limit the number of parallel processed modules
Browse files Browse the repository at this point in the history 
fixes #3164
fixes #3166
  • Loading branch information
@sokra
sokra committed Aug 11, 2017
1 parent f7bcba7 commit f9bf8a9
Show file tree
Hide file tree
Showing 3 changed files with 195 additions and 141 deletions.
299 changes: 158 additions & 141 deletions lib/Compilation.js
View file
Expand Up @@ -22,6 +22,7 @@ const Dependency = require("./Dependency");
const ChunkRenderError = require("./ChunkRenderError");
const CachedSource = require("webpack-sources").CachedSource;
const Stats = require("./Stats");
const Semaphore = require("./util/Semaphore");

function byId(a, b) {
if(a.id < b.id) return -1;
Expand Down Expand Up @@ -62,6 +63,8 @@ class Compilation extends Tapable {
this.hotUpdateChunkTemplate = new HotUpdateChunkTemplate(this.outputOptions);
this.moduleTemplate = new ModuleTemplate(this.outputOptions);

this.semaphore = new Semaphore(options.parallelism || 100);

this.entries = [];
this.preparedChunks = [];
this.entrypoints = {};
Expand Down Expand Up @@ -229,120 +232,128 @@ class Compilation extends Tapable {
callback();
};

const factory = item[0];
factory.create({
contextInfo: {
issuer: module.nameForCondition && module.nameForCondition(),
compiler: _this.compiler.name
},
context: module.context,
dependencies: dependencies
}, function factoryCallback(err, dependentModule) {
let afterFactory;

function isOptional() {
return dependencies.filter(d => !d.optional).length === 0;
}
_this.semaphore.acquire(() => {
const factory = item[0];
factory.create({
contextInfo: {
issuer: module.nameForCondition && module.nameForCondition(),
compiler: _this.compiler.name
},
context: module.context,
dependencies: dependencies
}, function factoryCallback(err, dependentModule) {
let afterFactory;

function isOptional() {
return dependencies.filter(d => !d.optional).length === 0;
}

function errorOrWarningAndCallback(err) {
if(isOptional()) {
return warningAndCallback(err);
} else {
return errorAndCallback(err);
function errorOrWarningAndCallback(err) {
if(isOptional()) {
return warningAndCallback(err);
} else {
return errorAndCallback(err);
}
}
}

function iterationDependencies(depend) {
for(let index = 0; index < depend.length; index++) {
const dep = depend[index];
dep.module = dependentModule;
dependentModule.addReason(module, dep);
function iterationDependencies(depend) {
for(let index = 0; index < depend.length; index++) {
const dep = depend[index];
dep.module = dependentModule;
dependentModule.addReason(module, dep);
}
}
}

if(err) {
return errorOrWarningAndCallback(new ModuleNotFoundError(module, err, dependencies));
}
if(!dependentModule) {
return process.nextTick(callback);
}
if(_this.profile) {
if(!dependentModule.profile) {
dependentModule.profile = {};
if(err) {
_this.semaphore.release();
return errorOrWarningAndCallback(new ModuleNotFoundError(module, err, dependencies));
}
if(!dependentModule) {
_this.semaphore.release();
return process.nextTick(callback);
}
if(_this.profile) {
if(!dependentModule.profile) {
dependentModule.profile = {};
}
afterFactory = Date.now();
dependentModule.profile.factory = afterFactory - start;
}
afterFactory = Date.now();
dependentModule.profile.factory = afterFactory - start;
}

dependentModule.issuer = module;
const newModule = _this.addModule(dependentModule, cacheGroup);
dependentModule.issuer = module;
const newModule = _this.addModule(dependentModule, cacheGroup);

if(!newModule) { // from cache
dependentModule = _this.getModule(dependentModule);
if(!newModule) { // from cache
dependentModule = _this.getModule(dependentModule);

if(dependentModule.optional) {
dependentModule.optional = isOptional();
}
if(dependentModule.optional) {
dependentModule.optional = isOptional();
}

iterationDependencies(dependencies);
iterationDependencies(dependencies);

if(_this.profile) {
if(!module.profile) {
module.profile = {};
}
const time = Date.now() - start;
if(!module.profile.dependencies || time > module.profile.dependencies) {
module.profile.dependencies = time;
if(_this.profile) {
if(!module.profile) {
module.profile = {};
}
const time = Date.now() - start;
if(!module.profile.dependencies || time > module.profile.dependencies) {
module.profile.dependencies = time;
}
}

_this.semaphore.release();
return process.nextTick(callback);
}

return process.nextTick(callback);
}
if(newModule instanceof Module) {
if(_this.profile) {
newModule.profile = dependentModule.profile;
}

if(newModule instanceof Module) {
if(_this.profile) {
newModule.profile = dependentModule.profile;
}
newModule.optional = isOptional();
newModule.issuer = dependentModule.issuer;
dependentModule = newModule;

newModule.optional = isOptional();
newModule.issuer = dependentModule.issuer;
dependentModule = newModule;
iterationDependencies(dependencies);

iterationDependencies(dependencies);
if(_this.profile) {
const afterBuilding = Date.now();
module.profile.building = afterBuilding - afterFactory;
}

if(_this.profile) {
const afterBuilding = Date.now();
module.profile.building = afterBuilding - afterFactory;
_this.semaphore.release();
if(recursive) {
return process.nextTick(_this.processModuleDependencies.bind(_this, dependentModule, callback));
} else {
return process.nextTick(callback);
}
}

if(recursive) {
return process.nextTick(_this.processModuleDependencies.bind(_this, dependentModule, callback));
} else {
return process.nextTick(callback);
}
}
dependentModule.optional = isOptional();

dependentModule.optional = isOptional();
iterationDependencies(dependencies);

iterationDependencies(dependencies);
_this.buildModule(dependentModule, isOptional(), module, dependencies, err => {
if(err) {
_this.semaphore.release();
return errorOrWarningAndCallback(err);
}

_this.buildModule(dependentModule, isOptional(), module, dependencies, err => {
if(err) {
return errorOrWarningAndCallback(err);
}
if(_this.profile) {
const afterBuilding = Date.now();
dependentModule.profile.building = afterBuilding - afterFactory;
}

if(_this.profile) {
const afterBuilding = Date.now();
dependentModule.profile.building = afterBuilding - afterFactory;
}
_this.semaphore.release();
if(recursive) {
_this.processModuleDependencies(dependentModule, callback);
} else {
return callback();
}
});

if(recursive) {
_this.processModuleDependencies(dependentModule, callback);
} else {
return callback();
}
});

});
}, function finalCallbackAddModuleDependencies(err) {
// In V8, the Error objects keep a reference to the functions on the stack. These warnings &
Expand Down Expand Up @@ -379,79 +390,85 @@ class Compilation extends Tapable {
throw new Error(`No dependency factory available for this dependency type: ${dependency.constructor.name}`);
}

moduleFactory.create({
contextInfo: {
issuer: "",
compiler: this.compiler.name
},
context: context,
dependencies: [dependency]
}, (err, module) => {
if(err) {
return errorAndCallback(new EntryModuleNotFoundError(err));
}

let afterFactory;

if(this.profile) {
if(!module.profile) {
module.profile = {};
this.semaphore.acquire(() => {
moduleFactory.create({
contextInfo: {
issuer: "",
compiler: this.compiler.name
},
context: context,
dependencies: [dependency]
}, (err, module) => {
if(err) {
this.semaphore.release();
return errorAndCallback(new EntryModuleNotFoundError(err));
}
afterFactory = Date.now();
module.profile.factory = afterFactory - start;
}

const result = this.addModule(module);
if(!result) {
module = this.getModule(module);

onModule(module);
let afterFactory;

if(this.profile) {
const afterBuilding = Date.now();
module.profile.building = afterBuilding - afterFactory;
if(!module.profile) {
module.profile = {};
}
afterFactory = Date.now();
module.profile.factory = afterFactory - start;
}

return callback(null, module);
}
const result = this.addModule(module);
if(!result) {
module = this.getModule(module);

if(result instanceof Module) {
if(this.profile) {
result.profile = module.profile;
}
onModule(module);

module = result;
if(this.profile) {
const afterBuilding = Date.now();
module.profile.building = afterBuilding - afterFactory;
}

onModule(module);
this.semaphore.release();
return callback(null, module);
}

moduleReady.call(this);
return;
}
if(result instanceof Module) {
if(this.profile) {
result.profile = module.profile;
}

onModule(module);
module = result;

this.buildModule(module, false, null, null, (err) => {
if(err) {
return errorAndCallback(err);
}
onModule(module);

if(this.profile) {
const afterBuilding = Date.now();
module.profile.building = afterBuilding - afterFactory;
moduleReady.call(this);
return;
}

moduleReady.call(this);
});
onModule(module);

function moduleReady() {
this.processModuleDependencies(module, err => {
this.buildModule(module, false, null, null, (err) => {
if(err) {
return callback(err);
this.semaphore.release();
return errorAndCallback(err);
}

return callback(null, module);
if(this.profile) {
const afterBuilding = Date.now();
module.profile.building = afterBuilding - afterFactory;
}

moduleReady.call(this);
});
}

function moduleReady() {
this.semaphore.release();
this.processModuleDependencies(module, err => {
if(err) {
return callback(err);
}

return callback(null, module);
});
}
});
});
}

Expand Down

0 comments on commit f9bf8a9

Please sign in to comment.