Skip to content

Commit

Permalink
refactor(@angular-devkit/build-angular): cache downlevel bundles
Browse files Browse the repository at this point in the history
  • Loading branch information
@clydin @kyliau
clydin authored and kyliau committed Aug 20, 2019
1 parent af02bf4 commit 1c59aa5
Show file tree
Hide file tree
Showing 6 changed files with 347 additions and 50 deletions.
1 change: 1 addition & 0 deletions package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@
"@types/clean-css": "^4.2.1",
"@types/copy-webpack-plugin": "^4.4.1",
"@types/express": "^4.16.0",
"@types/find-cache-dir": "^2.0.0",
"@types/glob": "^7.0.0",
"@types/inquirer": "^0.0.44",
"@types/jasmine": "^3.3.8",
Expand Down
2 changes: 2 additions & 0 deletions packages/angular_devkit/build_angular/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,14 @@
"ajv": "6.10.2",
"autoprefixer": "9.6.1",
"browserslist": "4.6.6",
"cacache": "12.0.2",
"caniuse-lite": "1.0.30000989",
"circular-dependency-plugin": "5.2.0",
"clean-css": "4.2.1",
"copy-webpack-plugin": "5.0.4",
"core-js": "3.2.1",
"file-loader": "4.2.0",
"find-cache-dir": "3.0.0",
"glob": "7.1.4",
"istanbul-instrumenter-loader": "3.0.1",
"karma-source-map-support": "1.4.0",
Expand Down
160 changes: 137 additions & 23 deletions packages/angular_devkit/build_angular/src/browser/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,10 @@ import {
virtualFs,
} from '@angular-devkit/core';
import { NodeJsSyncHost } from '@angular-devkit/core/node';
import { createHash } from 'crypto';
import * as findCacheDirectory from 'find-cache-dir';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import { from, of } from 'rxjs';
import { bufferCount, catchError, concatMap, map, mergeScan, switchMap } from 'rxjs/operators';
Expand Down Expand Up @@ -62,6 +65,7 @@ import {
normalizeOptimization,
normalizeSourceMaps,
} from '../utils';
import { CacheKey, ProcessBundleOptions } from '../utils/process-bundle';
import { assertCompatibleAngularVersion } from '../utils/version';
import {
generateBrowserWebpackConfigFromContext,
Expand All @@ -70,6 +74,10 @@ import {
} from '../utils/webpack-browser-config';
import { Schema as BrowserBuilderSchema } from './schema';

const cacache = require('cacache');
const cacheDownlevelPath = findCacheDirectory({ name: 'angular-build-dl' });
const packageVersion = require('../../package.json').version;

export type BrowserBuilderOutput = json.JsonObject &
BuilderOutput & {
outputPath: string;
Expand Down Expand Up @@ -240,6 +248,7 @@ export function buildWebpackBrowser(
1,
),
bufferCount(configs.length),
// tslint:disable-next-line: no-big-function
switchMap(async buildEvents => {
configs.length = 0;
const success = buildEvents.every(r => r.success);
Expand Down Expand Up @@ -274,9 +283,10 @@ export function buildWebpackBrowser(
optimize: normalizeOptimization(options.optimization).scripts,
sourceMaps: sourceMapOptions.scripts,
hiddenSourceMaps: sourceMapOptions.hidden,
vendorSourceMaps: sourceMapOptions.vendor,
};

const actions: {}[] = [];
const actions: ProcessBundleOptions[] = [];
const seen = new Set<string>();
for (const file of emittedFiles) {
// Scripts and non-javascript files are not processed
Expand Down Expand Up @@ -348,6 +358,7 @@ export function buildWebpackBrowser(
code,
map,
runtime: file.file.startsWith('runtime'),
ignoreOriginal: es5Polyfills,
});

// Add the newly created ES5 bundles to the index as nomodule scripts
Expand All @@ -359,30 +370,133 @@ export function buildWebpackBrowser(

// Execute the bundle processing actions
context.logger.info('Generating ES5 bundles for differential loading...');
await new Promise<void>((resolve, reject) => {
const workerFile = require.resolve('../utils/process-bundle');
const workers = workerFarm(
{
maxRetries: 1,
},
path.extname(workerFile) !== '.ts'
? workerFile
: require.resolve('../utils/process-bundle-bootstrap'),
['process'],
);
let completed = 0;
const workCallback = (error: Error | null) => {
if (error) {
workerFarm.end(workers);
reject(error);
} else if (++completed === actions.length) {
workerFarm.end(workers);
resolve();

const processActions: typeof actions = [];
const cacheActions: { src: string; dest: string }[] = [];
for (const action of actions) {
// Create base cache key with elements:
// * package version - different build-angular versions cause different final outputs
// * code length/hash - ensure cached version matches the same input code
const codeHash = createHash('sha1')
.update(action.code)
.digest('hex');
const baseCacheKey = `${packageVersion}|${action.code.length}|${codeHash}`;

// Postfix added to sourcemap cache keys when vendor sourcemaps are present
// Allows non-destructive caching of both variants
const SourceMapVendorPostfix =
!!action.sourceMaps && action.vendorSourceMaps ? '|vendor' : '';

// Determine cache entries required based on build settings
const cacheKeys = [];

// If optimizing and the original is not ignored, add original as required
if ((action.optimize || action.optimizeOnly) && !action.ignoreOriginal) {
cacheKeys[CacheKey.OriginalCode] = baseCacheKey + '|orig';

// If sourcemaps are enabled, add original sourcemap as required
if (action.sourceMaps) {
cacheKeys[CacheKey.OriginalMap] =
baseCacheKey + SourceMapVendorPostfix + '|orig-map';
}
}
// If not only optimizing, add downlevel as required
if (!action.optimizeOnly) {
cacheKeys[CacheKey.DownlevelCode] = baseCacheKey + '|dl';

// If sourcemaps are enabled, add downlevel sourcemap as required
if (action.sourceMaps) {
cacheKeys[CacheKey.DownlevelMap] =
baseCacheKey + SourceMapVendorPostfix + '|dl-map';
}
}

// Attempt to get required cache entries
const cacheEntries = [];
for (const key of cacheKeys) {
if (key) {
cacheEntries.push(await cacache.get.info(cacheDownlevelPath, key));
} else {
cacheEntries.push(null);
}
}

// Check if required cache entries are present
let cached = cacheKeys.length > 0;
for (let i = 0; i < cacheKeys.length; ++i) {
if (cacheKeys[i] && !cacheEntries[i]) {
cached = false;
break;
}
}

// If all required cached entries are present, use the cached entries
// Otherwise process the files
if (cached) {
if (cacheEntries[CacheKey.OriginalCode]) {
cacheActions.push({
src: cacheEntries[CacheKey.OriginalCode].path,
dest: action.filename,
});
}
};
if (cacheEntries[CacheKey.OriginalMap]) {
cacheActions.push({
src: cacheEntries[CacheKey.OriginalMap].path,
dest: action.filename + '.map',
});
}
if (cacheEntries[CacheKey.DownlevelCode]) {
cacheActions.push({
src: cacheEntries[CacheKey.DownlevelCode].path,
dest: action.filename.replace('es2015', 'es5'),
});
}
if (cacheEntries[CacheKey.DownlevelMap]) {
cacheActions.push({
src: cacheEntries[CacheKey.DownlevelMap].path,
dest: action.filename.replace('es2015', 'es5') + '.map',
});
}
} else {
processActions.push({
...action,
cacheKeys,
cachePath: cacheDownlevelPath || undefined,
});
}
}

for (const action of cacheActions) {
fs.copyFileSync(action.src, action.dest, fs.constants.COPYFILE_FICLONE);
}

if (processActions.length > 0) {
await new Promise<void>((resolve, reject) => {
const workerFile = require.resolve('../utils/process-bundle');
const workers = workerFarm(
{
maxRetries: 1,
},
path.extname(workerFile) !== '.ts'
? workerFile
: require.resolve('../utils/process-bundle-bootstrap'),
['process'],
);
let completed = 0;
const workCallback = (error: Error | null) => {
if (error) {
workerFarm.end(workers);
reject(error);
} else if (++completed === processActions.length) {
workerFarm.end(workers);
resolve();
}
};

processActions.forEach(action => workers['process'](action, workCallback));
});
}

actions.forEach(action => workers['process'](action, workCallback));
});
context.logger.info('ES5 bundle generation complete.');
} else {
const { emittedFiles = [] } = firstBuild;
Expand Down
54 changes: 48 additions & 6 deletions packages/angular_devkit/build_angular/src/utils/process-bundle.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,16 +11,28 @@ import { SourceMapConsumer, SourceMapGenerator } from 'source-map';
import { minify } from 'terser';

const { transformAsync } = require('@babel/core');
const cacache = require('cacache');

interface ProcessBundleOptions {
export interface ProcessBundleOptions {
filename: string;
code: string;
map?: string;
sourceMaps: boolean;
hiddenSourceMaps: boolean;
runtime: boolean;
sourceMaps?: boolean;
hiddenSourceMaps?: boolean;
vendorSourceMaps?: boolean;
runtime?: boolean;
optimize: boolean;
optimizeOnly?: boolean;
ignoreOriginal?: boolean;
cacheKeys?: (string | null)[];
cachePath?: string;
}

export const enum CacheKey {
OriginalCode = 0,
OriginalMap = 1,
DownlevelCode = 2,
DownlevelMap = 3,
}

export function process(
Expand All @@ -31,6 +43,10 @@ export function process(
}

async function processWorker(options: ProcessBundleOptions): Promise<void> {
if (!options.cacheKeys) {
options.cacheKeys = [];
}

// If no downlevelling required than just mangle code and return
if (options.optimizeOnly) {
return mangleOriginal(options);
Expand Down Expand Up @@ -139,7 +155,9 @@ async function processWorker(options: ProcessBundleOptions): Promise<void> {
map = result.map;

// Mangle original code
mangleOriginal(options);
if (!options.ignoreOriginal) {
await mangleOriginal(options);
}
} else if (map) {
map = JSON.stringify(map);
}
Expand All @@ -149,13 +167,20 @@ async function processWorker(options: ProcessBundleOptions): Promise<void> {
code += `\n//# sourceMappingURL=${path.basename(newFilePath)}.map`;
}

if (options.cachePath && options.cacheKeys[CacheKey.DownlevelMap]) {
await cacache.put(options.cachePath, options.cacheKeys[CacheKey.DownlevelMap], map);
}

fs.writeFileSync(newFilePath + '.map', map);
}

if (options.cachePath && options.cacheKeys[CacheKey.DownlevelCode]) {
await cacache.put(options.cachePath, options.cacheKeys[CacheKey.DownlevelCode], code);
}
fs.writeFileSync(newFilePath, code);
}

function mangleOriginal(options: ProcessBundleOptions): void {
async function mangleOriginal(options: ProcessBundleOptions): Promise<void> {
const resultOriginal = minify(options.code, {
compress: false,
ecma: 6,
Expand All @@ -176,8 +201,25 @@ function mangleOriginal(options: ProcessBundleOptions): void {
throw resultOriginal.error;
}

if (options.cachePath && options.cacheKeys && options.cacheKeys[CacheKey.OriginalCode]) {
await cacache.put(
options.cachePath,
options.cacheKeys[CacheKey.OriginalCode],
resultOriginal.code,
);
}

fs.writeFileSync(options.filename, resultOriginal.code);

if (resultOriginal.map) {
if (options.cachePath && options.cacheKeys && options.cacheKeys[CacheKey.OriginalMap]) {
await cacache.put(
options.cachePath,
options.cacheKeys[CacheKey.OriginalMap],
resultOriginal.map,
);
}

fs.writeFileSync(options.filename + '.map', resultOriginal.map);
}
}

0 comments on commit 1c59aa5

Please sign in to comment.