add module discovery caching per #187 (#189)

* (perf) add module discovery caching
linkedin · Oct 5, 2018 · 3c75d33 · 3c75d33
1 parent 6ee9709
commit 3c75d33
Show file tree

Hide file tree

Showing 12 changed files with 19,360 additions and 60 deletions.
diff --git a/README.md b/README.md
@@ -84,6 +84,24 @@ Manually added eyeglass modules will only be able to be imported by the
 main application's sass files. Dependencies between such manual modules
 are not currently supported.
 
+## Module Caching
+
+By default, eyeglass uses a global module cache to help improve the performance of module discovery. This should be safe for almost all use cases, but if you modify your `node_modules` directory and/or `package.json` dependencies during build time, this may cause issues. You can opt-out of the by passing the eyeglass option `useGlobalModuleCache: false`.
+
+```js
+eyeglass({
+  // sass options
+  // ...
+  eyeglass: {
+    // eyeglass options
+    // ...
+    useGlobalModuleCache: false
+  }
+});
+```
+
+Alternatively, you can programmatically purge the global cache using `eyeglass.modules.cache.modules.purge()`.
+
 # Working with assets
 
 It's quite common to need to refer to assets from within your

diff --git a/lib/assets/Assets.js b/lib/assets/Assets.js
@@ -124,7 +124,8 @@ Assets.prototype.resolveAsset = function($assetsMap, $uri, cb) {
             cb(error);
           } else {
             if (file) {
-              debug.assets(
+              /* istanbul ignore next - don't test debug */
+              debug.assets && debug.assets(
                 "%s resolved to %s with URI %s",
                 originalUri,
                 path.relative(options.root, file),

diff --git a/lib/importers/FSImporter.js b/lib/importers/FSImporter.js
@@ -20,6 +20,7 @@ function FSImporter(eyeglass, sass, options, fallbackImporter) {
       } else {
         absolutePath = path.resolve(path.dirname(prev));
       }
+      /* istanbul ignore else - TODO: revisit this */
       if (absolutePath) {
         var sassContents = '@import "eyeglass/fs"; @include fs-register-path('
                          + identifier + ', "' + absolutePath + '");';

diff --git a/lib/importers/ImportUtilities.js b/lib/importers/ImportUtilities.js
@@ -28,7 +28,8 @@ ImportUtilities.createImporter = function(importer) {
 ImportUtilities.prototype.importOnce = function(data, done) {
   if (this.options.eyeglass.enableImportOnce && this.context.eyeglass.imported[data.file]) {
     // log that we've already imported this file
-    debug.import("%s was already imported", data.file);
+    /* istanbul ignore next - don't test debug */
+    debug.import && debug.import("%s was already imported", data.file);
     done({contents: "", file: "already-imported:" + data.file});
   } else {
     this.context.eyeglass.imported[data.file] = true;

diff --git a/lib/index.js b/lib/index.js
@@ -23,7 +23,11 @@ function Eyeglass(options, deprecatedNodeSassArg) {
 
   this.options = new Options(options, this.deprecate, deprecatedNodeSassArg);
   this.assets = new Assets(this, this.options.eyeglass.engines.sass);
-  this.modules = new EyeglassModules(this.options.eyeglass.root, this.options.eyeglass.modules);
+  this.modules = new EyeglassModules(
+    this.options.eyeglass.root,
+    this.options.eyeglass.modules,
+    this.options.eyeglass.useGlobalModuleCache
+  );
 
   fs.mkdirpSync(this.options.eyeglass.cacheDir);
 

diff --git a/lib/modules/EyeglassModules.js b/lib/modules/EyeglassModules.js
@@ -3,6 +3,7 @@
 var resolve = require("../util/resolve");
 var packageUtils = require("../util/package");
 var EyeglassModule = require("./EyeglassModule");
+var SimpleCache = require("../util/SimpleCache");
 var debug = require("../util/debug");
 var path = require("path");
 var merge = require("lodash.merge");
@@ -11,14 +12,17 @@ var archy = require("archy");
 var fs = require("fs");
 var URI = require("../util/URI");
 
+var globalModuleCache = new SimpleCache();
+
 /**
   * Discovers all of the modules for a given directory
   *
   * @constructor
   * @param   {String} dir - the directory to discover modules in
   * @param   {Array} modules - the explicit modules to include
+  * @param   {Boolean} useGlobalModuleCache - whether or not to use the global module cache
   */
-function EyeglassModules(dir, modules) {
+function EyeglassModules(dir, modules, useGlobalModuleCache) {
   this.issues = {
     dependencies: {
       versions: [],
@@ -31,7 +35,8 @@ function EyeglassModules(dir, modules) {
   };
 
   this.cache = {
-    access: {}
+    access: new SimpleCache(),
+    modules: useGlobalModuleCache ? globalModuleCache : new SimpleCache()
   };
 
   // find the nearest package.json for the given directory
@@ -70,7 +75,8 @@ function EyeglassModules(dir, modules) {
   // check for any issues we may have encountered
   checkForIssues.call(this);
 
-  debug.modules("discovered modules\n\t" + this.getGraph().replace(/\n/g, "\n\t"));
+  /* istanbul ignore next - don't test debug */
+  debug.modules && debug.modules("discovered modules\n\t" + this.getGraph().replace(/\n/g, "\n\t"));
 }
 
 /**
@@ -257,39 +263,30 @@ function canAccessModule(name, origin) {
   var canAccessFrom = function canAccessFrom(origin) {
     // find the nearest package for the origin
     var pkg = packageUtils.findNearestPackage(origin);
-
-    var cache = this.cache.access;
-    cache[name] = cache[name] || {};
-
-    // check for cached result
-    if (cache[name][pkg] !== undefined) {
-      return cache[name][pkg];
-    }
-
-    // find all the branches that match the origin
-    var branches = findBranchesByPath({
-      dependencies: this.tree
-    }, pkg);
-
-    var canAccess = branches.some(function(branch) {
-      // if the reference is to itself (branch.name)
-      // OR it's an immediate dependency (branch.dependencies[name])
-      if (branch.name === name || branch.dependencies && branch.dependencies[name]) {
-        return true;
-      }
-    });
-
-    debug.import(
-      "%s can%s be imported from %s",
-      name,
-      (canAccess ? "" : "not"),
-      origin
-    );
-
-    // cache it for future lookups
-    cache[name][pkg] = canAccess;
-
-    return canAccess;
+    var cacheKey = pkg + "!" + origin;
+    return this.cache.access.getOrElse(cacheKey, function() {
+      // find all the branches that match the origin
+      var branches = findBranchesByPath({
+        dependencies: this.tree
+      }, pkg);
+
+      var canAccess = branches.some(function(branch) {
+        // if the reference is to itself (branch.name)
+        // OR it's an immediate dependency (branch.dependencies[name])
+        if (branch.name === name || branch.dependencies && branch.dependencies[name]) {
+          return true;
+        }
+      });
+
+      /* istanbul ignore next - don't test debug */
+      debug.import && debug.import(
+        "%s can%s be imported from %s",
+        name,
+        (canAccess ? "" : "not"),
+        origin
+      );
+      return canAccess;
+    }.bind(this));
   }.bind(this);
 
   // check if we can access from the origin...
@@ -346,7 +343,8 @@ function getDependencyVersionIssues(modules, finalModule) {
   return modules.map(function(mod) {
     // if the versions are not identical, log it
     if (mod.version !== finalModule.version) {
-      debug.modules(
+      /* istanbul ignore next - don't test debug */
+      debug.modules && debug.modules(
         "asked for %s@%s but using %s",
         mod.name,
         mod.version,
@@ -411,16 +409,19 @@ function getFinalModule(name) {
   * @returns {Object} the resolved module definition
   */
 function resolveModule(pkgPath, isRoot) {
-  var pkg = packageUtils.getPackage(pkgPath);
-  var isEyeglassMod = EyeglassModule.isEyeglassModule(pkg.data);
-  // if the module is an eyeglass module OR it's the root project
-  if (isEyeglassMod || (pkg.data && isRoot)) {
-    // return a module reference
-    return new EyeglassModule({
-      isEyeglassModule: isEyeglassMod,
-      path: path.dirname(pkg.path)
-    }, discoverModules.bind(this), isRoot);
-  }
+  var cacheKey = "resolveModule~" + pkgPath + "!" + !!isRoot;
+  return this.cache.modules.getOrElse(cacheKey, function() {
+    var pkg = packageUtils.getPackage(pkgPath);
+    var isEyeglassMod = EyeglassModule.isEyeglassModule(pkg.data);
+    // if the module is an eyeglass module OR it's the root project
+    if (isEyeglassMod || (pkg.data && isRoot)) {
+      // return a module reference
+      return new EyeglassModule({
+        isEyeglassModule: isEyeglassMod,
+        path: path.dirname(pkg.path)
+      }, discoverModules.bind(this), isRoot);
+    }
+  }.bind(this));
 }
 
 /**
@@ -442,10 +443,13 @@ function getEyeglassSelf() {
   *
   * @see resolve()
   */
-function resolveModulePackage() {
-  try {
-    return resolve.apply(null, arguments);
-  } catch (e) {}
+function resolveModulePackage(id, parent, parentDir) {
+  var cacheKey = "resolveModulePackage~" + id + "!" + parent + "!" + parentDir;
+  return this.cache.modules.getOrElse(cacheKey, function() {
+    try {
+      return resolve(id, parent, parentDir);
+    } catch (e) {}
+  });
 }
 
 /**
@@ -456,8 +460,6 @@ function resolveModulePackage() {
   */
 function discoverModules(options) {
   var pkg = options.pkg || packageUtils.getPackage(options.dir);
-
-  // get the collection of dependencies
   var dependencies = {};
 
   // if there's package.json contents...
@@ -476,7 +478,8 @@ function discoverModules(options) {
   // for each dependency...
   dependencies = Object.keys(dependencies).reduce(function(modules, dependency) {
     // resolve the package.json
-    var resolvedPkg = resolveModulePackage(
+    var resolvedPkg = resolveModulePackage.call(
+      this,
       packageUtils.getPackagePath(dependency),
       pkg.path,
       URI.system(options.dir)

diff --git a/lib/modules/ModuleFunctions.js b/lib/modules/ModuleFunctions.js
@@ -44,7 +44,8 @@ function ModuleFunctions(eyeglass, sass, options, existingFunctions) {
     }
 
     // log any functions found in this module
-    debug.functions(
+    /* istanbul ignore next - don't test debug */
+    debug.functions && debug.functions(
       "functions discovered in module %s:%s%s",
       mod.name,
       DELIM,
@@ -60,7 +61,8 @@ function ModuleFunctions(eyeglass, sass, options, existingFunctions) {
   functions = syncFn.all(functions);
 
   // log all the functions we discovered
-  debug.functions(
+  /* istanbul ignore next - don't test debug */
+  debug.functions && debug.functions(
     "all discovered functions:%s%s",
     DELIM,
     Object.keys(functions).join(DELIM)

diff --git a/lib/util/Options.js b/lib/util/Options.js
@@ -124,6 +124,9 @@ function defaultEyeglassOptions(options) {
   // default enableImportOnce
   options.enableImportOnce = defaultValue(options.enableImportOnce, true);
 
+  // use global module caching by default
+  options.useGlobalModuleCache = defaultValue(options.useGlobalModuleCache, true);
+
   return options;
 }
 

diff --git a/lib/util/SimpleCache.js b/lib/util/SimpleCache.js
@@ -0,0 +1,64 @@
+"use strict";
+
+/**
+  * A simple caching implementation
+  */
+function SimpleCache() {
+  this.cache = {};
+}
+
+/**
+  * Returns the current cached value
+  *
+  * @param   {String} key - they cache key to lookup
+  * @returns {*} the cached value
+  */
+SimpleCache.prototype.get = function(key) {
+  return this.cache[key];
+};
+
+/**
+  * Sets the cached value
+  *
+  * @param    {String} key - they cache key to update
+  * @param    {*} value - they value to store
+  */
+SimpleCache.prototype.set = function(key, value) {
+  this.cache[key] = value;
+};
+
+/**
+  * Whether or not the cache has a value for a given key
+  *
+  * @param    {String} key - they cache key to lookup
+  * @returns  {Boolean} whether or not the key is set
+  */
+SimpleCache.prototype.has = function(key) {
+  return this.cache.hasOwnProperty(key);
+};
+
+/**
+  * Gets the current value from the cache (if it exists), otherwise invokes the callback
+  *
+  * @param    {String} key - they cache key lookup
+  * @param    {Function} callback - the callback to be invoked when the key is not in the cache
+  * @returns
+  */
+SimpleCache.prototype.getOrElse = function(key, callback) {
+  // if we do not yet have a result, generate it and store it in the cache
+  if (!this.has(key)) {
+    this.set(key, callback());
+  }
+
+  // return the result from the cache
+  return this.get(key);
+};
+
+/**
+  * Purges the cache
+  */
+SimpleCache.prototype.purge = function(key) {
+  this.cache = {};
+};
+
+module.exports = SimpleCache;
diff --git a/lib/util/debug.js b/lib/util/debug.js
@@ -4,6 +4,10 @@ var debug = require("debug");
 var PREFIX = "eyeglass:";
 
 module.exports = ["import", "modules", "functions", "assets"].reduce(function(obj, item) {
-  obj[item] = debug(PREFIX + item);
+  var namespace = PREFIX + item;
+  /* istanbul ignore if - don't test debug */
+  if (debug.enabled(namespace)) {
+    obj[item] = debug(namespace);
+  }
   return obj;
 }, {});