Skip to content

Commit

Permalink
Additional outputs outside the broccoli tree.
Browse files Browse the repository at this point in the history 
If additional output from a sass compilation is written to a location
that is outside of the broccoli output tree we can't just ignore it
because the persistent cache hit would not create the same outputs. But
because the output locatio is outside of the tree, it's not this
plugin's responsbility to restore that output.

To manage this, the "additional-output" event can now provide a source
location and target url for the asset. When there's a persistent cache
hit, the "cached-asset" event is fired with the source location and
target url that was provided by "additional-output". In this way the
build integration that is using this plugin must decide how to restore
the cache for those files correctly.
  • Loading branch information
@chriseppstein
chriseppstein committed Mar 21, 2019
1 parent 204a6fc commit eb40f9b
Show file tree
Hide file tree
Showing 4 changed files with 167 additions and 77 deletions.
2 changes: 1 addition & 1 deletion packages/broccoli-eyeglass/README.md
View file
Expand Up @@ -196,7 +196,7 @@ environment variable `BROCCOLI_EYEGLASS=forceInvalidateCache`.

The caches will only be invalidated correctly if this broccoli plugin
knows what files are depended on and output. Sass files and eyeglass
assets are already tracked. But other files migh be involved in your
assets are already tracked. But other files might be involved in your
build, if that is the case, `eyeglassCompiler.events.emit("dependency", absolutePath)`
must be called during the build. Similarly, if there are
other files output during compilation, then you must call
Expand Down
163 changes: 121 additions & 42 deletions packages/broccoli-eyeglass/src/broccoli_sass_compiler.ts
View file
Expand Up @@ -22,7 +22,6 @@ import heimdall = require("heimdalljs");
const FSTreeFromEntries = FSTree.fromEntries;
const debug = debugGenerator("broccoli-eyeglass");
const hotCacheDebug = debugGenerator("broccoli-eyeglass:hot-cache");
const persistentCacheDebug = debugGenerator("broccoli-eyeglass:persistent-cache");

let sass: typeof nodeSass;
let renderSass: (options: nodeSass.Options) => Promise<nodeSass.Result>;
Expand All @@ -31,6 +30,13 @@ type SassPromiseRenderer =
((options: nodeSass.Options) => Promise<nodeSass.Result>)
| ((options: nodeSass.SyncOptions) => Promise<nodeSass.Result>);

interface CachedContents {
contents: Record<string, string>;
urls: Record<string, string>;
}

type CachedDependencies = Array<[string, string]>;

function initSass(): void {
if (!sass) {
sass = nodeSass;
Expand Down Expand Up @@ -285,21 +291,33 @@ function writeDataToFile(cachedFile: string, outputFile: string, data: Buffer):
// * compiler.events.emit("dependency", absolutePath);
// marks the file as a dependency for the Sass file being compiled so that future
// compiles will invalidate the cache if that file changes.
// * compiler.events.emit("additional-output", absolutePath);
// * compiler.events.emit("additional-output", absolutePathToOutput, httpPathToOutput, absolutePathToSource);
// marks the file as an additional output for the Sass file being compiled so that future
// cached compiles will be able to install or remove them as needed in conjunction with the
// sass file.
// sass file. Note: the source will not be considered a dependency unless the "dependency" event
// is also emitted.
//
// You can subscribe to the following events:
//
// * compiler.events.on("compiling", function(details) { });
// prepare for a compilation to occur with these options. The options
// are a unique copy for this compilation. E.g. This can be used to
// prime a cache in the options for the compilation.
// * compiler.events.on("compiled", function(details, result) { });
// * compiler.events.on("compiled", (details, result) => { });
// receive notification of a successful compilation.
// * compiler.events.on("failed", function(details, error) { });
// * compiler.events.on("failed", (details, error) => { });
// receive notification of a compilation failure.
// * compiler.events.on("stale-external-output", (outputFile) => {})
// receive a notification that a file outside of this broccoli output
// tree might need to be removed because the only known sass file
// in this tree to output it has been deleted.
// * compiler.events.on("cached-asset", (absolutePathToSource, httpPathToOutput) => {})
// receive a notification that an additional asset that was created
// when the caller fired "additional-output" (see above) needs to be
// restored because the sass file that produced it was retrieved from
// cache. This is only invoked when the asset was created outside of the
// broccoli tree for this addon. If the asset was in the tree, it will
// be automatically recreated from the cache.
//
// For all these events, a compilation details object is passed of the
// following form:
Expand All @@ -320,11 +338,11 @@ export default class BroccoliSassCompiler extends BroccoliPlugin {
private colors: any;
private currentTree: null | FSTree;
private dependencies: Record<string, Set<string>>;
private outputURLs: Record<string, Map<string, string>>;
private outputs: Record<string, Set<string>>;

protected cssDir: string;
protected discover: boolean | undefined;
protected events: EventEmitter;
protected fullException: boolean;
protected maxListeners: number;
protected options: nodeSass.Options;
Expand All @@ -335,6 +353,9 @@ export default class BroccoliSassCompiler extends BroccoliPlugin {
protected sourceFiles: Array<string>;
protected treeName: string | undefined;
protected verbose: boolean;
protected persistentCacheDebug: debugGenerator.Debugger;

public events: EventEmitter;

constructor(inputTree: BroccoliPlugin.BroccoliNode | Array<BroccoliPlugin.BroccoliNode>, options: BroccoliSassOptions & nodeSass.Options) {
if (Array.isArray(inputTree)) {
Expand Down Expand Up @@ -362,10 +383,12 @@ export default class BroccoliSassCompiler extends BroccoliPlugin {
this.currentTree = null;
this.dependencies = {};
this.outputs = {};
this.outputURLs = {}

if (shouldPersist(process.env, !!options.persistentCache)) {
this.persistentCache = new DiskCache(options.persistentCache);
}
this.persistentCacheDebug = debugGenerator(`broccoli-eyeglass:persistent-cache:${options.persistentCache || 'disabled'}`);

this.treeName = options.annotation;
delete options.annotation;
Expand Down Expand Up @@ -553,7 +576,7 @@ export default class BroccoliSassCompiler extends BroccoliPlugin {
return this.handleCacheMiss(details, reason, key, compilationTimer);
}

let dependencies: Array<[string, string]> = JSON.parse(cachedDependencies.value);
let dependencies: CachedDependencies = JSON.parse(cachedDependencies.value);

// check dependency caches
if (dependencies.some(dep => this.dependencyChanged(details.srcPath, dep))) {
Expand All @@ -568,9 +591,9 @@ export default class BroccoliSassCompiler extends BroccoliPlugin {
}

let depFiles = dependencies.map(depAndHash => depAndHash[0]);
let value: [Array<string>, Record<string, string>] = [depFiles, JSON.parse(cachedOutput.value)];
let value: [Array<string>, CachedContents] = [depFiles, JSON.parse(cachedOutput.value)];
compilationTimer.stats.cacheHitCount++;
return RSVP.resolve(this.handleCacheHit(details, value));
return RSVP.resolve(this.handleCacheHit(details, value).then(() => {}));
} catch (error) {
return this.handleCacheMiss(details, error, key, compilationTimer);
}
Expand Down Expand Up @@ -616,17 +639,16 @@ export default class BroccoliSassCompiler extends BroccoliPlugin {
* @return String
*/
outputKey(key: string): string {
return "[[[output of " + key + "]]]";
return "[[[output of " + key + "] v2]]";
}

/* retrieve the files from cache, write them, and populate the hot cache information
* for rebuilds.
*/
handleCacheHit(details: CompilationDetails, inputAndOutputFiles: [Array<string>, Record<string, string>]): void {
let inputFiles = inputAndOutputFiles[0];
let outputFiles = inputAndOutputFiles[1];
handleCacheHit(details: CompilationDetails, inputAndOutputFiles: [Array<string>, CachedContents]): Promise<Array<void>> {
let [inputFiles, outputFiles] = inputAndOutputFiles;

persistentCacheDebug(
this.persistentCacheDebug(
"Persistent cache hit for %s. Writing to: %s",
details.sassFilename,
details.fullCssFilename
Expand All @@ -644,23 +666,38 @@ export default class BroccoliSassCompiler extends BroccoliPlugin {
this.addDependency(details.fullSassFilename, path.resolve(details.srcPath, dep));
});

let files = Object.keys(outputFiles);
let {contents, urls} = outputFiles;
let files = Object.keys(contents);

persistentCacheDebug(
this.persistentCacheDebug(
"cached output files for %s are: %s",
details.sassFilename,
files.join(", ")
);

files.map(file => {
let data = outputFiles[file];
for (let file of files) {
let data = contents[file];
let cachedFile = path.join(this.cachePath!, file);
let outputFile = path.join(this.outputPath, file);
// populate the output cache for rebuilds
this.addOutput(details.fullSassFilename, outputFile);

writeDataToFile(cachedFile, outputFile, Buffer.from(data, "base64"));
});
}
let eventPromises: Array<Promise<any>> = [];
let allUrls = Object.keys(urls);
if (allUrls.length > 0) {
this.persistentCacheDebug(
"firing 'cached-asset' for each asset url for %s: %s",
details.sassFilename,
allUrls.join(", ")
);
}
for (let url of allUrls) {
let sourceFile = urls[url];
eventPromises.push(this.events.emit("cached-asset", sourceFile, url))
}
return RSVP.all(eventPromises);
}

scopedFileName(file: string): string {
Expand All @@ -675,6 +712,16 @@ export default class BroccoliSassCompiler extends BroccoliPlugin {
relativize(file: string): string {
return removePathPrefix(this.inputPaths[0], [file])[0];
}
isOutputInTree(file: string): boolean {
if (path.isAbsolute(file)) {
return file.startsWith(this.outputPath);
} else {
return true;
}
}
relativizeOutput(file: string): string {
return removePathPrefix(this.outputPath, [file])[0];
}

relativizeAll(files: Array<string>): Array<string> {
return removePathPrefix(this.inputPaths[0], files);
Expand All @@ -692,6 +739,10 @@ export default class BroccoliSassCompiler extends BroccoliPlugin {
return this.outputs[this.relativize(file)] || new Set();
}

outputURLsFrom(file: string): Map<string, string> {
return this.outputURLs[this.relativize(file)] || new Map();
}

/**
* Some filenames returned from importers are not really files
* on disk. These three prefixes are used in eyeglass.
Expand All @@ -708,13 +759,13 @@ export default class BroccoliSassCompiler extends BroccoliPlugin {
/* hash all dependencies synchronously and return the files that exist
* as an array of pairs (filename, hash).
*/
hashDependencies(details: CompilationDetails): Array<[string, string]> {
hashDependencies(details: CompilationDetails): CachedDependencies {
let depsWithHashes = new Array<[string, string]>();

this.dependenciesOf(details.fullSassFilename).forEach(f => {
try {
if (this.isNotFile(f)) {
persistentCacheDebug("Ignoring non-file dependency: %s", f);
this.persistentCacheDebug("Ignoring non-file dependency: %s", f);
} else {
let h = this.hashForFile(f);

Expand All @@ -725,7 +776,7 @@ export default class BroccoliSassCompiler extends BroccoliPlugin {
}
} catch (e) {
if (typeof e === "object" && e !== null && e.code === "ENOENT") {
persistentCacheDebug("Ignoring non-existent file: %s", f);
this.persistentCacheDebug("Ignoring non-existent file: %s", f);
} else {
throw e;
}
Expand All @@ -739,31 +790,33 @@ export default class BroccoliSassCompiler extends BroccoliPlugin {
/* read all output files asynchronously and return the contents
* as a hash of relative filenames to base64 encoded strings.
*/
readOutputs(details: CompilationDetails): Record<string, string> {
let reads = new Array<[string, string]>();
readOutputs(details: CompilationDetails): CachedContents {
let contents: Record<string, string> = {};
let urls: Record<string, string> = {};
let outputs = this.outputsFrom(details.fullSassFilename);

outputs.forEach(output => reads.push([output, fs.readFileSync(output, "base64")]));

return reads.reduce((content: Record<string, string>, output) => {
let fileName = output[0];
let contents = output[1];

if (fileName.startsWith(details.destDir)) {
content[fileName.substring(details.destDir.length + 1)] = contents;
for (let output of outputs) {
if (this.isOutputInTree(output)) {
contents[this.relativizeOutput(output)] = fs.readFileSync(output, "base64");
} else {
persistentCacheDebug(
this.persistentCacheDebug(
"refusing to cache output file found outside the output tree: %s",
fileName
output
);
}
return content;
}, {});
}

let outputURLs = this.outputURLsFrom(details.fullSassFilename);
for (let url of outputURLs.keys()) {
urls[url] = outputURLs.get(url)!;
}

return {contents, urls};
}

/* Writes the dependencies and output contents to the persistent cache */
populateCache(key: string, details: CompilationDetails, _result: nodeSass.Result): void {
persistentCacheDebug("Populating cache for " + key);
this.persistentCacheDebug("Populating cache for " + key);

let cache = this.persistentCache!;

Expand All @@ -777,14 +830,14 @@ export default class BroccoliSassCompiler extends BroccoliPlugin {
/* When the cache misses, we need to compile the file and then populate the cache */
handleCacheMiss(details: CompilationDetails, reason: Error | {message: string; stack?: Array<string>}, key: string, compilationTimer: heimdall.Cookie<SassRenderSchema>): Promise<void> {
compilationTimer.stats.cacheMissCount++;
persistentCacheDebug(
this.persistentCacheDebug(
"Persistent cache miss for %s. Reason: %s",
details.sassFilename,
reason.message
);
// for errors
if (reason.stack) {
persistentCacheDebug("Stacktrace:", reason.stack);
this.persistentCacheDebug("Stacktrace:", reason.stack);
}
return this.compileCssFile(details, compilationTimer).then(result => {
return this.populateCache(key, details, result);
Expand Down Expand Up @@ -818,8 +871,21 @@ export default class BroccoliSassCompiler extends BroccoliPlugin {
this.addDependency(details.fullSassFilename, absolutePath);
};

let additionalOutputListener = (filename: string): void => {
this.addOutput(details.fullSassFilename, filename);
let additionalOutputListener = (absolutePathToOutput: string, httpPathToOutput: string | undefined, absolutePathToSource: string | undefined): void => {
this.persistentCacheDebug("additional-output %s -> %s -> %s", absolutePathToSource, httpPathToOutput, absolutePathToOutput);
if (!this.isOutputInTree(absolutePathToOutput)) {
// it's outside this tree, don't cache the output.
if (absolutePathToSource && httpPathToOutput) {
this.persistentCacheDebug("additional-output is outside tree will cache source & url");
// something outside this tree is putting it there, so we need to
// let that same thing deal with it again when the warm cache is accessed.
// we will track this file from its source location and target url
this.addSource(details.fullSassFilename, absolutePathToSource, httpPathToOutput);
}
} else {
this.persistentCacheDebug("additional-output is in tree will cache contents");
this.addOutput(details.fullSassFilename, absolutePathToOutput);
}
};

this.events.addListener("additional-output", additionalOutputListener);
Expand Down Expand Up @@ -881,6 +947,13 @@ export default class BroccoliSassCompiler extends BroccoliPlugin {
return unique(files);
}

addSource(sassFilename: string, sourceFilename: string, httpPathToOutput: string): void {
sassFilename = this.relativize(sassFilename);
this.outputURLs[sassFilename] = this.outputURLs[sassFilename] || new Map<string, string>();
let urlMap = this.outputURLs[sassFilename];
urlMap.set(httpPathToOutput, sourceFilename);
}

addOutput(sassFilename: string, outputFilename: string): void {
sassFilename = this.relativize(sassFilename);

Expand All @@ -893,6 +966,9 @@ export default class BroccoliSassCompiler extends BroccoliPlugin {
if (this.outputs[f]) {
delete this.outputs[f];
}
if (this.outputURLs[f]) {
delete this.outputURLs[f];
}
});
}

Expand Down Expand Up @@ -978,6 +1054,7 @@ export default class BroccoliSassCompiler extends BroccoliPlugin {
this.currentTree = null;
this.dependencies = {};
this.outputs = {};
this.outputURLs = {};
}

_build(): Promise<void | Array<void | nodeSass.Result>> {
Expand Down Expand Up @@ -1046,11 +1123,13 @@ export default class BroccoliSassCompiler extends BroccoliPlugin {

if (removed.length > 0) {
let outputs = this.outputsFromOnly(removed);
// TODO: outputURLsFromOnly(removed)
outputs.forEach(output => {
if (output.indexOf(outputPath) === 0) {
fs.unlinkSync(output);
} else {
hotCacheDebug("not removing because outside the outputTree", output);
this.events.emit("stale-external-output", output)
}
});
this.clearOutputs(removed);
Expand Down Expand Up @@ -1089,7 +1168,7 @@ export default class BroccoliSassCompiler extends BroccoliPlugin {
this.buildCount++;

if (this.buildCount === 1 && process.env.BROCCOLI_EYEGLASS === "forceInvalidateCache") {
persistentCacheDebug("clearing cache because forceInvalidateCache was set.");
this.persistentCacheDebug("clearing cache because forceInvalidateCache was set.");
this.persistentCache && this.persistentCache.clear();
}

Expand Down

0 comments on commit eb40f9b

Please sign in to comment.