Be smarter about cache keys for inDevelopment eyeglass addons.

We were using hash-for-dep to invalidate the cache if any javascript
in an eyeglass module changed. But many eyeglass addons are hybrid
packages that have a lot of javascript that isn't relevant to the sass
files.

This patch switches to creating a list of dependencies of js files
starting with the eyeglass exports file and only considers those files
as relevant to the sass file cache. This approach is slower because it
parses the js files for import and require statements, but because
it is a more narrow cache, that extra cost pays for itself.

This only affects eyeglass modules marked as inDevelopment which
use an eyeglass exports file.

packages/broccoli-eyeglass/package.json

@@ -42,12 +42,12 @@
     "chained-emitter": "^0.1.2",
     "colors": "^1.3.1",
     "debug": "^3.1.0",
+    "dependency-tree": "^7.0.2",
     "ensure-symlink": "^1.0.2",
     "eyeglass": "^2.2.2",
     "fs-extra": "^7.0.0",
     "fs-tree-diff": "^1.0.0",
     "glob": "^7.1.2",
-    "hash-for-dep": "^1.5.0",
     "heimdalljs": "^0.2.6",
     "json-stable-stringify": "^1.0.1",
     "lodash.clonedeep": "^4.5.0",
@@ -66,6 +66,7 @@
   "devDependencies": {
     "@types/broccoli-plugin": "^1.3.0",
     "@types/debug": "^4.1.0",
+    "@types/dependency-tree": "^6.1.0",
     "@types/fs-extra": "^5.0.5",
     "@types/glob": "^7.1.1",
     "@types/json-stable-stringify": "^1.0.32",

packages/broccoli-eyeglass/src/broccoli_sass_compiler.ts

@@ -1,7 +1,6 @@
 "use strict";
 import debugGenerator = require("debug");
 import * as path from "path";
-import crypto = require("crypto");
 import fs = require("fs-extra");
 import RSVP = require("rsvp");
 import mkdirp = require("mkdirp");
@@ -18,6 +17,7 @@ import MergeTrees = require("broccoli-merge-trees");
 import { EventEmitter } from "chained-emitter";
 import DiskCache = require("sync-disk-cache");
 import heimdall = require("heimdalljs");
+import {statSync, realpathSync} from "fs";
 
 const FSTreeFromEntries = FSTree.fromEntries;
 const debug = debugGenerator("broccoli-eyeglass");
@@ -627,26 +627,33 @@ export default class BroccoliSassCompiler extends BroccoliPlugin {
    * @return hash object of the file data
    **/
   hashForFile(absolutePath: string): string {
-    let cacheKey = `hashForFile(${absolutePath})`;
-    let cachedHash = this.buildCache.get(cacheKey);
-    if (cachedHash) {
-      return cachedHash as string;
+    return this.fileKey(absolutePath);
+  }
+
+  /* compute a key for a file that will change if the file has changed. */
+  fileKey(file: string, isRealPath = false): string {
+    let cachedKeyKey = `fileKey(${file})`;
+    let cachedKey = this.buildCache.get(cachedKeyKey) as string;
+    if (cachedKey) { return cachedKey; }
+    let stat = statSync(file);
+    let key;
+    if (!isRealPath && stat.isSymbolicLink) {
+      key = this.fileKey(realpathSync(file), true);
     } else {
-      let data = fs.readFileSync(absolutePath, "UTF8");
-      let hash = crypto.createHash("md5").update(data).digest("hex");
-      this.buildCache.set(cacheKey, hash);
-      return hash;
+      key = `${stat.mtimeMs}:${stat.size}:${stat.mode}`;
     }
+    this.buildCache.set(cachedKeyKey, key);
+    return key;
   }
 
+
   /* construct a base cache key for a file to be compiled.
    *
    * @argument srcDir The directory in which to resolve relative paths against.
    * @argument relativeFilename The filename relative to the srcDir that is being compiled.
    * @argument options The compilation options.
    *
-   * @return Promise that resolves to the cache key for the file or rejects if
-   *         it can't read the file.
+   * @return The cache key for the file
    **/
   keyForSourceFile(srcDir: string, relativeFilename: string, _options: nodeSass.Options): string {
     let absolutePath = path.join(srcDir, relativeFilename);

packages/broccoli-eyeglass/src/index.ts

@@ -11,11 +11,13 @@ import stringify = require("json-stable-stringify");
 import debugGenerator = require("debug");
 import Eyeglass = require("eyeglass");
 import * as sass from "node-sass";
-import hashForDep = require("hash-for-dep");
+import dependencyTree = require("dependency-tree");
 import { EyeglassOptions } from "eyeglass/lib/util/Options";
+import EyeglassModule from "eyeglass/lib/modules/EyeglassModule";
 
 type SassImplementation = typeof sass;
 const assetImportCacheDebug = debugGenerator("broccoli-eyeglass:asset-import-cache");
+const dependencyDebug = debugGenerator("broccoli-eyeglass:file-dependencies")
 const CURRENT_VERSION: string = require(path.join(__dirname, "..", "package.json")).version;
 
 function httpJoin(...args: Array<string>): string {
@@ -191,20 +193,16 @@ class EyeglassCompiler extends BroccoliSassCompiler {
 
   dependenciesHash(_srcDir: string, _relativeFilename: string, options: Eyeglass.EyeglassOptions): string {
     if (!this._dependenciesHash) {
-      let eyeglass = new Eyeglass(options); // options
-      // let eyeglass: Eyeglass = options.eyeglass!.engines!.eyeglass
+      let eyeglass = new Eyeglass(options);
       let hash = crypto.createHash("sha1");
-      let cachableOptions = stringify(this.cachableOptions(options));
-
-      hash.update(cachableOptions);
       hash.update("broccoli-eyeglass@" + EyeglassCompiler.currentVersion());
 
       let egModules = sortby(eyeglass.modules.list, m => m.name);
 
       egModules.forEach(mod => {
         let name: string = mod.name;
         if (mod.inDevelopment || mod.eyeglass.inDevelopment) {
-          let depHash: string = hashForDep(mod.path);
+          let depHash: string = this.hashForJs(mod);
           hash.update(name + "@" + depHash);
         } else {
           let version: string = mod.version || "<unversioned>";
@@ -220,9 +218,10 @@ class EyeglassCompiler extends BroccoliSassCompiler {
 
   keyForSourceFile(srcDir: string, relativeFilename: string, options: Eyeglass.EyeglassOptions): string {
     let key = super.keyForSourceFile(srcDir, relativeFilename, options);
+    let optsString = stringify(this.cachableOptions(options));
     let dependencies = this.dependenciesHash(srcDir, relativeFilename, options);
 
-    return key + "+" + dependencies;
+    return key + "+" + optsString + "+" + dependencies;
   }
 
   // Cache the asset import code that is generated in eyeglass
@@ -241,6 +240,28 @@ class EyeglassCompiler extends BroccoliSassCompiler {
     this.buildCache.set(assetImportKey, assetImport);
     return assetImport;
   }
+
+  hashForJs(mod: EyeglassModule): string {
+    let jsPath = mod.mainPath;
+    if (!jsPath) {
+      return ""
+    }
+    let cacheKey = `hashForJs(${jsPath})`;
+    let cachedHash = this.buildCache.get(cacheKey) as string;
+    if (cachedHash) {
+      return cachedHash;
+    }
+    let files = dependencyTree.toList({filename: jsPath, directory: path.dirname(jsPath)});
+    dependencyDebug(`${jsPath} => \n\t${files.join("\n\t")}`);
+    let hash = crypto.createHash("sha1");
+    for (let file of files) {
+      hash.update(file);
+      hash.update(this.fileKey(file));
+    }
+    let result = hash.digest("base64");
+    this.buildCache.set(cacheKey, result);
+    return result;
+  }
 }
 
 export = EyeglassCompiler;

packages/broccoli-eyeglass/test/test_eyeglass_plugin.js

@@ -1235,7 +1235,6 @@ describe("EyeglassCompiler", function() {
           "project.css": ".foo {\n  content: changed-foo; }\n",
         };
 
-        require("hash-for-dep")._resetCache();
         delete require.cache[fs.realpathSync(eyeglassModule.path("eyeglass-exports.js"))];
         delete require.cache[fs.realpathSync(eyeglassModule.path("lib/foo.js"))];
 
@@ -1328,7 +1327,6 @@ describe("EyeglassCompiler", function() {
           "project.css": ".foo {\n  content: changed-foo; }\n",
         };
 
-        require("hash-for-dep")._resetCache();
         delete require.cache[fs.realpathSync(eyeglassModule.path("eyeglass-exports.js"))];
         delete require.cache[fs.realpathSync(eyeglassModule.path("lib/foo.js"))];