Extend manualChunks API (#3542)

* Extend manualChunks API

* rename iterators

* Add inverse dependency information to getModuleInfo

* Replace Array.from with spread operator, sort ids

* Asyncify ModuleLoader

* Inline dynamic import argument extraction

* Get rid of getEntryModules

* Improve coverage

* Add this.getModuleIds and deprecate this.moduleIds

* Document this.getModuleIds

* Document manualChunks

docs/05-plugin-development.md

@@ -539,6 +539,16 @@ Get the combined source maps of all previous plugins. This context function can
 
 Get the file name of a chunk or asset that has been emitted via [`this.emitFile`](guide/en/#thisemitfileemittedfile-emittedchunk--emittedasset--string). The file name will be relative to `outputOptions.dir`.
 
+#### `this.getModuleIds() => IterableIterator<string>`
+
+Returns an `Iterator` that gives access to all module ids in the current graph. It can be iterated via
+
+```js
+for (const moduleId of this.getModuleIds()) { /* ... */ }
+```
+
+or converted into an Array via `Array.from(this.getModuleIds())`.
+
 #### `this.getModuleInfo(moduleId: string) => ModuleInfo`
 
 Returns additional information about the module in question in the form
@@ -549,7 +559,9 @@ Returns additional information about the module in question in the form
   isEntry: boolean, // is this a user- or plugin-defined entry point
   isExternal: boolean, // for external modules that are not included in the graph
   importedIds: string[], // the module ids statically imported by this module
+  importers: string[], // the ids of all modules that statically import this module
   dynamicallyImportedIds: string[], // the module ids imported by this module via dynamic import()
+  dynamicImporters: string[], // the ids of all modules that import this module via dynamic import()
   hasModuleSideEffects: boolean // are imports of this module included if nothing is imported from it
 }
 ```
@@ -560,16 +572,6 @@ If the module id cannot be found, an error is thrown.
 
 An `Object` containing potentially useful Rollup metadata. `meta` is the only context property accessible from the [`options`](guide/en/#options) hook.
 
-#### `this.moduleIds: IterableIterator<string>`
-
-An `Iterator` that gives access to all module ids in the current graph. It can be iterated via
-
-```js
-for (const moduleId of this.moduleIds) { /* ... */ }
-```
-
-or converted into an Array via `Array.from(this.moduleIds)`.
-
 #### `this.parse(code: string, acornOptions?: AcornOptions) => ESTree.Program`
 
 Use Rollup's internal acorn instance to parse code to an AST.
@@ -621,6 +623,15 @@ The `position` argument is a character index where the warning was raised. If pr
 
 - `this.isExternal(id: string, importer: string | undefined, isResolved: boolean) => boolean` - _**Use [`this.resolve`](guide/en/#thisresolvesource-string-importer-string-options-skipself-boolean--promiseid-string-external-boolean--null)**_ - Determine if a given module ID is external when imported by `importer`. When `isResolved` is false, Rollup will try to resolve the id before testing if it is external.
 
+- `this.moduleIds: IterableIterator<string>` - _**Use [`this.getModuleIds`](guide/en/#thisgetmoduleids--iterableiteratorstring)**_ - An `Iterator` that gives access to all module ids in the current graph. It can be iterated via
+
+  ```js
+  for (const moduleId of this.moduleIds) { /* ... */ }
+  ```
+
+  or converted into an Array via `Array.from(this.moduleIds)`.
+
+
 - `this.resolveId(source: string, importer?: string) => Promise<string | null>` - _**Use [`this.resolve`](guide/en/#thisresolvesource-string-importer-string-options-skipself-boolean--promiseid-string-external-boolean--null)**_ - Resolve imports to module ids (i.e. file names) using the same plugins that Rollup uses. Returns `null` if an id cannot be resolved.
 
 ### File URLs

docs/999-big-list-of-options.md

@@ -324,7 +324,7 @@ Default: `false`
 This will inline dynamic imports instead of creating new chunks to create a single bundle. Only possible if a single input is provided.
 
 #### manualChunks
-Type: `{ [chunkAlias: string]: string[] } | ((id: string) => string | void)`
+Type: `{ [chunkAlias: string]: string[] } | ((id: string, {getModuleInfo, getModuleIds}) => string | void)`
 
 Allows the creation of custom shared common chunks. When using the object form, each property represents a chunk that contains the listed modules and all their dependencies if they are part of the module graph unless they are already in another manual chunk. The name of the chunk will be determined by the property key.
 
@@ -350,6 +350,58 @@ manualChunks(id) {
 
 Be aware that manual chunks can change the behaviour of the application if side-effects are triggered before the corresponding modules are actually used.
 
+When using the function form, `manualChunks` will be passed an object as second parameter containing the functions `getModuleInfo` and `getModuleIds` that work the same way as [`this.getModuleInfo`](guide/en/#thisgetmoduleinfomoduleid-string--moduleinfo) and [`this.getModuleIds`](guide/en/#thisgetmoduleids--iterableiteratorstring) on the plugin context.
+
+This can be used to dynamically determine into which manual chunk a module should be placed depending on its position in the module graph. For instance consider a scenario where you have a set of components, each of which dynamically imports a set of translated strings, i.e.
+
+```js
+// Inside the "foo" component
+
+function getTranslatedStrings(currentLanguage) {
+  switch (currentLanguage) {
+    case 'en': return import('./foo.strings.en.js');
+    case 'de': return import('./foo.strings.de.js');
+    // ...
+  }
+}
+```
+
+If a lot of such components are used together, this will result in a lot of dynamic imports of very small chunks: Even though we known that all language files of the same language that are imported by the same chunk will always be used together, Rollup does not have this information.
+
+The following code will merge all files of the same language that are only used by a single entry point:
+
+```js
+manualChunks(id, { getModuleInfo }) {
+  const match = /.*\.strings\.(\w+)\.js/.exec(id);
+  if (match) {
+    const language = match[1]; // e.g. "en"
+    const dependentEntryPoints = [];
+
+    // we use a Set here so we handle each module at most once. This
+    // prevents infinite loops in case of circular dependencies
+    const idsToHandle = new Set(getModuleInfo(id).dynamicImporters);
+
+    for (const moduleId of idsToHandle) {
+      const { isEntry, isDynamicEntry, importers } = getModuleInfo(moduleId);
+      if (isEntry || isDynamicEntry) dependentEntryPoints.push(moduleId);
+
+      // The Set iterator is intelligent enough to iterate over elements that
+      // are added during iteration
+      for (const importerId of importers) idsToHandle.add(importerId);
+    }
+
+    // If there is a unique entry, we put it into into a chunk based on the entry name
+    if (dependentEntryPoints.length === 1) {
+      return `${dependentEntryPoints[0].split('/').slice(-1)[0]}.strings.${language}`;
+    }
+    // For multiple entries, we put it into a "shared" chunk
+    if (dependentEntryPoints.length > 1) {
+      return `shared.strings.${language}`;
+    }
+  }
+}
+```
+
 #### onwarn
 Type: `(warning: RollupWarning, defaultHandler: (warning: string | RollupWarning) => void) => void;`
 

src/Chunk.ts

@@ -185,7 +185,7 @@ export default class Chunk {
 			if (module.isEntryPoint) {
 				this.entryModules.push(module);
 			}
-			if (module.dynamicallyImportedBy.length > 0) {
+			if (module.includedDynamicImporters.length > 0) {
 				this.dynamicEntryModules.push(module);
 			}
 		}
@@ -267,18 +267,18 @@ export default class Chunk {
 	generateFacades(): Chunk[] {
 		const facades: Chunk[] = [];
 		const dynamicEntryModules = this.dynamicEntryModules.filter(module =>
-			module.dynamicallyImportedBy.some(importingModule => importingModule.chunk !== this)
+			module.includedDynamicImporters.some(importingModule => importingModule.chunk !== this)
 		);
 		this.isDynamicEntry = dynamicEntryModules.length > 0;
 		const exposedNamespaces = dynamicEntryModules.map(module => module.namespace);
 		for (const module of this.entryModules) {
-			const requiredFacades: FacadeName[] = Array.from(module.userChunkNames).map(name => ({
+			const requiredFacades: FacadeName[] = [...module.userChunkNames].map(name => ({
 				name
 			}));
 			if (requiredFacades.length === 0 && module.isUserDefinedEntryPoint) {
 				requiredFacades.push({});
 			}
-			requiredFacades.push(...Array.from(module.chunkFileNames).map(fileName => ({ fileName })));
+			requiredFacades.push(...[...module.chunkFileNames].map(fileName => ({ fileName })));
 			if (requiredFacades.length === 0) {
 				requiredFacades.push({});
 			}
@@ -1060,7 +1060,7 @@ export default class Chunk {
 		}
 		if (
 			(module.isEntryPoint && module.preserveSignature !== false) ||
-			module.dynamicallyImportedBy.some(importer => importer.chunk !== this)
+			module.includedDynamicImporters.some(importer => importer.chunk !== this)
 		) {
 			const map = module.getExportNamesByVariable();
 			for (const exportedVariable of map.keys()) {

src/ExternalModule.ts

@@ -7,11 +7,13 @@ import { isAbsolute, normalize, relative } from './utils/path';
 export default class ExternalModule {
 	chunk: void;
 	declarations: { [name: string]: ExternalVariable };
+	dynamicImporters: string[] = [];
 	execIndex: number;
 	exportedVariables: Map<ExternalVariable, string>;
 	exportsNames = false;
 	exportsNamespace = false;
 	id: string;
+	importers: string[] = [];
 	moduleSideEffects: boolean;
 	mostCommonSuggestion = 0;
 	nameSuggestions: { [name: string]: number };

src/Graph.ts

@@ -9,10 +9,10 @@ import ExternalModule from './ExternalModule';
 import Module, { defaultAcornOptions } from './Module';
 import { ModuleLoader, UnresolvedModule } from './ModuleLoader';
 import {
-	GetManualChunk,
 	InputOptions,
 	IsExternal,
 	ManualChunksOption,
+	ModuleInfo,
 	ModuleJSON,
 	PreserveEntrySignaturesOption,
 	RollupCache,
@@ -181,7 +181,6 @@ export default class Graph {
 			this.pluginDriver,
 			options.preserveSymlinks === true,
 			options.external as (string | RegExp)[] | IsExternal,
-			(typeof options.manualChunks === 'function' && options.manualChunks) as GetManualChunk | null,
 			(this.treeshakingOptions ? this.treeshakingOptions.moduleSideEffects : null)!,
 			(this.treeshakingOptions ? this.treeshakingOptions.pureExternalModules : false)!
 		);
@@ -246,6 +245,35 @@ export default class Graph {
 		};
 	}
 
+	getModuleInfo = (moduleId: string): ModuleInfo => {
+		const foundModule = this.moduleById.get(moduleId);
+		if (foundModule == null) {
+			throw new Error(`Unable to find module ${moduleId}`);
+		}
+		const importedIds: string[] = [];
+		const dynamicallyImportedIds: string[] = [];
+		if (foundModule instanceof Module) {
+			for (const source of foundModule.sources) {
+				importedIds.push(foundModule.resolvedIds[source].id);
+			}
+			for (const { resolution } of foundModule.dynamicImports) {
+				if (resolution instanceof Module || resolution instanceof ExternalModule) {
+					dynamicallyImportedIds.push(resolution.id);
+				}
+			}
+		}
+		return {
+			dynamicallyImportedIds,
+			dynamicImporters: foundModule.dynamicImporters,
+			hasModuleSideEffects: foundModule.moduleSideEffects,
+			id: foundModule.id,
+			importedIds,
+			importers: foundModule.importers,
+			isEntry: foundModule instanceof Module && foundModule.isEntryPoint,
+			isExternal: foundModule instanceof ExternalModule
+		};
+	};
+
 	warn(warning: RollupWarning) {
 		warning.toString = () => {
 			let str = '';
@@ -279,7 +307,11 @@ export default class Graph {
 		const chunks: Chunk[] = [];
 		if (this.preserveModules) {
 			for (const module of this.modules) {
-				if (module.isIncluded() || module.isEntryPoint || module.dynamicallyImportedBy.length > 0) {
+				if (
+					module.isIncluded() ||
+					module.isEntryPoint ||
+					module.includedDynamicImporters.length > 0
+				) {
 					const chunk = new Chunk(this, [module]);
 					chunk.entryModules = [module];
 					chunks.push(chunk);
@@ -360,6 +392,9 @@ export default class Graph {
 				typeof manualChunks === 'object' &&
 				this.moduleLoader.addManualChunks(manualChunks)
 		]);
+		if (typeof manualChunks === 'function') {
+			this.moduleLoader.assignManualChunks(manualChunks);
+		}
 		if (entryModules.length === 0) {
 			throw new Error('You must supply options.input to rollup');
 		}

src/Module.ts

@@ -15,7 +15,7 @@ import Literal from './ast/nodes/Literal';
 import MetaProperty from './ast/nodes/MetaProperty';
 import * as NodeType from './ast/nodes/NodeType';
 import Program from './ast/nodes/Program';
-import { Node, NodeBase } from './ast/nodes/shared/Node';
+import { ExpressionNode, NodeBase } from './ast/nodes/shared/Node';
 import TemplateLiteral from './ast/nodes/TemplateLiteral';
 import VariableDeclaration from './ast/nodes/VariableDeclaration';
 import ModuleScope from './ast/scopes/ModuleScope';
@@ -195,9 +195,10 @@ export default class Module {
 	code!: string;
 	comments: CommentDescription[] = [];
 	dependencies = new Set<Module | ExternalModule>();
-	dynamicallyImportedBy: Module[] = [];
 	dynamicDependencies = new Set<Module | ExternalModule>();
+	dynamicImporters: string[] = [];
 	dynamicImports: {
+		argument: string | ExpressionNode;
 		node: ImportExpression;
 		resolution: Module | ExternalModule | string | null;
 	}[] = [];
@@ -208,8 +209,10 @@ export default class Module {
 	exportsAll: { [name: string]: string } = Object.create(null);
 	facadeChunk: Chunk | null = null;
 	importDescriptions: { [name: string]: ImportDescription } = Object.create(null);
+	importers: string[] = [];
 	importMetas: MetaProperty[] = [];
 	imports = new Set<Variable>();
+	includedDynamicImporters: Module[] = [];
 	isExecuted = false;
 	isUserDefinedEntryPoint = false;
 	manualChunkAlias: string = null as any;
@@ -348,7 +351,11 @@ export default class Module {
 				relevantDependencies.add(variable.module);
 			}
 		}
-		if (this.isEntryPoint || this.dynamicallyImportedBy.length > 0 || this.graph.preserveModules) {
+		if (
+			this.isEntryPoint ||
+			this.includedDynamicImporters.length > 0 ||
+			this.graph.preserveModules
+		) {
 			for (const exportName of [...this.getReexports(), ...this.getExports()]) {
 				let variable = this.getVariableForExportName(exportName);
 				if (variable instanceof SyntheticNamedExportVariable) {
@@ -384,23 +391,6 @@ export default class Module {
 		return (this.relevantDependencies = relevantDependencies);
 	}
 
-	getDynamicImportExpressions(): (string | Node)[] {
-		return this.dynamicImports.map(({ node }) => {
-			const importArgument = node.source;
-			if (
-				importArgument instanceof TemplateLiteral &&
-				importArgument.quasis.length === 1 &&
-				importArgument.quasis[0].value.cooked
-			) {
-				return importArgument.quasis[0].value.cooked;
-			}
-			if (importArgument instanceof Literal && typeof importArgument.value === 'string') {
-				return importArgument.value;
-			}
-			return importArgument;
-		});
-	}
-
 	getExportNamesByVariable(): Map<Variable, string[]> {
 		if (this.exportNamesByVariable) {
 			return this.exportNamesByVariable;
@@ -451,7 +441,7 @@ export default class Module {
 				}
 			}
 		}
-		return (this.transitiveReexports = Array.from(reexports));
+		return (this.transitiveReexports = [...reexports]);
 	}
 
 	getRenderedExports() {
@@ -741,7 +731,7 @@ export default class Module {
 			ast: this.esTreeAst,
 			code: this.code,
 			customTransformCache: this.customTransformCache,
-			dependencies: Array.from(this.dependencies).map(module => module.id),
+			dependencies: [...this.dependencies].map(module => module.id),
 			id: this.id,
 			moduleSideEffects: this.moduleSideEffects,
 			originalCode: this.originalCode,
@@ -800,7 +790,15 @@ export default class Module {
 	}
 
 	private addDynamicImport(node: ImportExpression) {
-		this.dynamicImports.push({ node, resolution: null });
+		let argument: ExpressionNode | string = node.source;
+		if (argument instanceof TemplateLiteral) {
+			if (argument.quasis.length === 1 && argument.quasis[0].value.cooked) {
+				argument = argument.quasis[0].value.cooked;
+			}
+		} else if (argument instanceof Literal && typeof argument.value === 'string') {
+			argument = argument.value;
+		}
+		this.dynamicImports.push({ node, resolution: null, argument });
 	}
 
 	private addExport(
@@ -931,7 +929,7 @@ export default class Module {
 			resolution: string | Module | ExternalModule | undefined;
 		}).resolution;
 		if (resolution instanceof Module) {
-			resolution.dynamicallyImportedBy.push(this);
+			resolution.includedDynamicImporters.push(this);
 			resolution.includeAllExports();
 		}
 	}