Skip to content

Commit

Permalink
Merge branch 'perf_tuning_asset_install'
Browse files Browse the repository at this point in the history
  • Loading branch information
chriseppstein committed Mar 23, 2019
2 parents fc08edf + 15501b4 commit 8388b72
Show file tree
Hide file tree
Showing 11 changed files with 521 additions and 156 deletions.
2 changes: 1 addition & 1 deletion packages/broccoli-eyeglass/README.md
Expand Up @@ -196,7 +196,7 @@ environment variable `BROCCOLI_EYEGLASS=forceInvalidateCache`.

The caches will only be invalidated correctly if this broccoli plugin
knows what files are depended on and output. Sass files and eyeglass
assets are already tracked. But other files migh be involved in your
assets are already tracked. But other files might be involved in your
build, if that is the case, `eyeglassCompiler.events.emit("dependency", absolutePath)`
must be called during the build. Similarly, if there are
other files output during compilation, then you must call
Expand Down
221 changes: 167 additions & 54 deletions packages/broccoli-eyeglass/src/broccoli_sass_compiler.ts

Large diffs are not rendered by default.

57 changes: 28 additions & 29 deletions packages/broccoli-eyeglass/src/index.ts
Expand Up @@ -15,7 +15,6 @@ import hashForDep = require("hash-for-dep");
import { EyeglassOptions } from "eyeglass/lib/util/Options";

type SassImplementation = typeof sass;
const persistentCacheDebug = debugGenerator("broccoli-eyeglass:persistent-cache");
const assetImportCacheDebug = debugGenerator("broccoli-eyeglass:asset-import-cache");
const CURRENT_VERSION: string = require(path.join(__dirname, "..", "package.json")).version;

Expand Down Expand Up @@ -63,7 +62,6 @@ class EyeglassCompiler extends BroccoliSassCompiler {
private relativeAssets: boolean | undefined;
private assetDirectories: Array<string> | undefined;
private assetsHttpPrefix: string | undefined;
private _assetImportCache: Record<string, string>;
private _assetImportCacheStats: { hits: number; misses: number };
private _dependenciesHash: string | undefined;
constructor(inputTrees: BroccoliPlugin.BroccoliNode | Array<BroccoliPlugin.BroccoliNode>, options: BroccoliEyeglassOptions) {
Expand Down Expand Up @@ -101,7 +99,6 @@ class EyeglassCompiler extends BroccoliSassCompiler {
this.assetsHttpPrefix = assetsHttpPrefix;
this.events.on("compiling", this.handleNewFile.bind(this));

this._assetImportCache = Object.create(null);
this._assetImportCacheStats = {
hits: 0,
misses: 0,
Expand Down Expand Up @@ -133,31 +130,10 @@ class EyeglassCompiler extends BroccoliSassCompiler {
options.eyeglass.engines = options.eyeglass.engines || {};
options.eyeglass.engines.sass = options.eyeglass.engines.sass || sass;
options.eyeglass.installWithSymlinks = true;
options.eyeglass.buildCache = this.buildCache;

let eyeglass = new Eyeglass(options);

// set up asset dependency tracking
let self = this;
let realResolve = eyeglass.assets.resolve;

eyeglass.assets.resolve = function(filepath, fullUri, cb) {
self.events.emit("dependency", filepath).then(() => {
realResolve.call(eyeglass.assets, filepath, fullUri, cb);
}, cb);
};

let realInstall = eyeglass.assets.install;
eyeglass.assets.install = function(file, uri, cb) {
realInstall.call(eyeglass.assets, file, uri, (error: unknown, file?: string) => {
if (error) {
cb(error, file);
} else {
self.events.emit("additional-output", file).then(() => {
cb(null, file);
}, cb);
}
});
};

if (this.assetDirectories) {
for (var i = 0; i < this.assetDirectories.length; i++) {
Expand All @@ -175,6 +151,26 @@ class EyeglassCompiler extends BroccoliSassCompiler {
if (this.configureEyeglass) {
this.configureEyeglass(eyeglass, options.eyeglass.engines.sass, details);
}

// set up asset dependency tracking
eyeglass.assets.resolver((filepath, fullUri, realResolve, cb) => {
this.events.emit("dependency", filepath).then(() => {
realResolve(filepath, fullUri, cb);
}, cb);
});

eyeglass.assets.installer((file, uri, realInstall, cb) => {
realInstall(file, uri, (error: unknown, destFile?: string) => {
if (error) {
cb(error, file);
} else {
this.events.emit("additional-output", destFile, uri, file).then(() => {
cb(null, file);
}, cb);
}
});
});

details.options = eyeglass.options;
details.options.eyeglass.engines.eyeglass = eyeglass;
}
Expand All @@ -200,7 +196,6 @@ class EyeglassCompiler extends BroccoliSassCompiler {
let hash = crypto.createHash("sha1");
let cachableOptions = stringify(this.cachableOptions(options));

persistentCacheDebug("cachableOptions are %s", cachableOptions);
hash.update(cachableOptions);
hash.update("broccoli-eyeglass@" + EyeglassCompiler.currentVersion());

Expand Down Expand Up @@ -233,14 +228,18 @@ class EyeglassCompiler extends BroccoliSassCompiler {
// Cache the asset import code that is generated in eyeglass
cacheAssetImports(key: string, getValue: () => string): string {
// if this has already been generated, return it from cache
if (this._assetImportCache[key] !== undefined) {
let assetImportKey = `assetImport(${key})`;
let assetImport = this.buildCache.get(assetImportKey) as string | undefined;
if (assetImport !== undefined) {
assetImportCacheDebug("cache hit for key '%s'", key);
this._assetImportCacheStats.hits += 1;
return this._assetImportCache[key];
return assetImport;
}
assetImportCacheDebug("cache miss for key '%s'", key);
this._assetImportCacheStats.misses += 1;
return (this._assetImportCache[key] = getValue());
assetImport = getValue();
this.buildCache.set(assetImportKey, assetImport);
return assetImport;
}
}

Expand Down
34 changes: 25 additions & 9 deletions packages/broccoli-eyeglass/src/types/sync-disk-cache.d.ts
@@ -1,16 +1,32 @@
export = SyncDiskCache;
interface CacheHit {
isCached: true;
key: string;
value: string;
}
interface CacheMiss {
isCached: false;
key: undefined;
value: undefined;
}

interface SyncDiskCacheOptions {
location?: string;
compression?: 'deflate' | 'deflateRaw' | 'gzip';
}

declare class SyncDiskCache {
constructor(key: any, _?: any);
constructor(key?: string, options?: SyncDiskCacheOptions);
tmpdir: any;
compression: any;
key: any;
root: any;
clear(...args: any[]): any;
compress(...args: any[]): any;
decompress(...args: any[]): any;
get(...args: any[]): any;
has(...args: any[]): any;
pathFor(...args: any[]): any;
remove(...args: any[]): any;
set(...args: any[]): any;
clear(): void;
compress(value: string): string;
decompress(value: string): string;
get(key: string): CacheHit | CacheMiss;
has(key: string): boolean;
pathFor(key: string): string;
remove(key: string): void;
set(key: string, value: string): string;
}
23 changes: 17 additions & 6 deletions packages/broccoli-eyeglass/test/test_eyeglass_plugin.js
Expand Up @@ -223,7 +223,7 @@ describe("EyeglassCompiler", function() {
let path = modules.path();
modules.copy(FIXTURES.path("manualModule"));
input.copy(FIXTURES.path("usesManualModule/input"));

let optimizer = new EyeglassCompiler(input.path(), {
cssDir: ".",
fullException: true,
Expand Down Expand Up @@ -1361,14 +1361,19 @@ describe("EyeglassCompiler", function() {
output = createBuilder(compiler);

// cache should start empty
assert.strictEqual(Object.keys(compiler._assetImportCache).length, 0);
assert.strictEqual(compiler.buildCache.size, 0);

yield output.build();

assertEqualDirs(output.path(), expectedOutput);

// cache should have one entry
assert.strictEqual(Object.keys(compiler._assetImportCache).length, 1);
let assetsCached = 0;
for (let k of compiler.buildCache.keys()) {
if (k.startsWith("assetImport(")) {
assetsCached += 1;
}
}
assert.strictEqual(assetsCached, 1);
// first file should be a miss, 2nd should return from cache
assert.strictEqual(compiler._assetImportCacheStats.misses, 1);
assert.strictEqual(compiler._assetImportCacheStats.hits, 1);
Expand Down Expand Up @@ -1422,13 +1427,19 @@ describe("EyeglassCompiler", function() {
output = createBuilder(compiler);

// cache should start empty
assert.strictEqual(Object.keys(compiler._assetImportCache).length, 0);
assert.strictEqual(compiler.buildCache.size, 0);

yield output.build();

assertEqualDirs(output.path(), expectedOutput);
// cache should have one entry
assert.strictEqual(Object.keys(compiler._assetImportCache).length, 1);
let assetsCached = 0;
for (let k of compiler.buildCache.keys()) {
if (k.startsWith("assetImport(")) {
assetsCached += 1;
}
}
assert.strictEqual(assetsCached, 1);
// first file should be a miss, 2nd should return from cache
assert.strictEqual(compiler._assetImportCacheStats.misses, 1);
assert.strictEqual(compiler._assetImportCacheStats.hits, 1);
Expand Down
3 changes: 2 additions & 1 deletion packages/ember-cli-eyeglass/package.json
Expand Up @@ -60,7 +60,6 @@
"eslint-plugin-ember": "^6.2.0",
"eslint-plugin-node": "^8.0.1",
"eyeglass": "^2.2.0",
"fs-extra": "^7.0.0",
"lazy": "file:./tests/dummy/lib/lazy/",
"loader.js": "^4.0.1",
"mocha": "^5.2.0",
Expand All @@ -79,6 +78,8 @@
"broccoli-eyeglass": "^5.0.2",
"broccoli-funnel": "^2.0.1",
"broccoli-merge-trees": "^3.0.0",
"broccoli-plugin": "^1.3.1",
"fs-extra": "^7.0.0",
"lodash.clonedeep": "^4.5.0",
"lodash.defaultsdeep": "^4.6.0"
},
Expand Down
82 changes: 82 additions & 0 deletions packages/ember-cli-eyeglass/src/broccoli-ln-s.ts
@@ -0,0 +1,82 @@
import BroccoliPlugin = require("broccoli-plugin");
import path = require("path");
import * as fs from "fs-extra";

type EnsureSymlinkSync = (srcFile: string, destLink: string) => void;
/* eslint-disable-next-line @typescript-eslint/no-var-requires */
const ensureSymlink: EnsureSymlinkSync = require("ensure-symlink");

/**
* An object where the keys are the files that will be created in the tree and
* the values are the source files.
*
* @interface FileMap
*/
interface FileMap {
[relativePath: string]: string;
}

type BroccoliSymbolicLinkerOptions =
Pick<BroccoliPlugin.BroccoliPluginOptions, "annotation" | "persistentOutput">;

/**
* Creates symlinks to the source files specified.
*
* BroccoliSymbolicLinker
*/
export class BroccoliSymbolicLinker extends BroccoliPlugin {
files: FileMap;
constructor(fileMap?: FileMap | undefined, options: BroccoliSymbolicLinkerOptions = {}) {
let pluginOpts: BroccoliPlugin.BroccoliPluginOptions = {needsCache: false};
Object.assign(pluginOpts, options);
super([], pluginOpts);
this.files = Object.assign({}, fileMap);
}
reset(fileMap?: FileMap | undefined): void {
this.files = Object.assign({}, fileMap);
}
/**
* Record that a symlink should be created from src to dest.
*
* This can be called many times before the build method is invoked.
* Calling it after will not have an effect until the next time build() is
* invoked.
*
* @param src The file that should be symlinked into the tree.
* @param dest the relative path from the tree's root to the location of the
* symlink. the filename does not have to be the same.
* @returns the absolute path to the location where the symlink will be created.
*/
// eslint-disable-next-line @typescript-eslint/camelcase
ln_s(src: string, dest: string): string {
this.files[dest] = src;
return path.join(this.outputPath, dest);
}
/**
* Returns the number of symlinks that will be created.
*/
numberOfFiles(): number {
return Object.keys(this.files).length;
}
/**
* Create the symlinks. Directories to them will be created as necessary.
*/
build(): void {
// eslint-disable-next-line no-console
// console.log(`Building ${this.numberOfFiles()} symlinks for ${this["_annotation"]}.`);
for (let dest of Object.keys(this.files)) {
let src = this.files[dest];
// console.log(`linking ${src} to ${dest}`);
dest = path.join(this.outputPath, dest)
let dir = path.dirname(dest);
fs.mkdirpSync(dir);
ensureSymlink(src, dest)
}
}
/**
* Output the symlinks that will be created for debugging.
*/
debug(): string {
return Object.keys(this.files).join("\n");
}
}

0 comments on commit 8388b72

Please sign in to comment.