feat(bundler): build.options.cache turn on tracing cache and transpil…

…e cache

Cache is centralized in OS temp dir.
Tracing cache can be shared among apps.
Transpile cache is only for esnext. Scoped to appName-env, to respect env based babelrc.
No transpile cache for TS project, gulp-typescript does incremental build, not working with gulp-cache.
New au command "au clear-cache" clear both caches.
Reduce consecutive build time to about 1/5 for esnext app, based on preliminary benchmark.
TODO benchmark on TS app.

lib/build/bundle.js

@@ -307,7 +307,7 @@ exports.Bundle = class {
         if (buildOptions.isApplicable('rev')) {
           //Generate a unique hash based off of the bundle contents
           //Must generate hash after we write the loader config so that any other bundle changes (hash changes) can cause a new hash for the vendor file
-          this.hash = generateHash(concat.content);
+          this.hash = generateHash(concat.content).slice(0, 10);
           bundleFileName = generateHashedPath(this.config.name, this.hash);
         }
 
@@ -319,7 +319,7 @@ exports.Bundle = class {
       } else if (buildOptions.isApplicable('rev')) {
         //Generate a unique hash based off of the bundle contents
         //Must generate hash after we write the loader config so that any other bundle changes (hash changes) can cause a new hash for the vendor file
-        this.hash = generateHash(concat.content);
+        this.hash = generateHash(concat.content).slice(0, 10);
         bundleFileName = generateHashedPath(this.config.name, this.hash);
       }
 

lib/build/bundled-source.js

@@ -4,7 +4,7 @@ const findDeps = require('./find-deps').findDeps;
 const cjsTransform = require('./amodro-trace/read/cjs');
 const esTransform = require('./amodro-trace/read/es');
 const allWriteTransforms = require('./amodro-trace/write/all');
-const {moduleIdWithPlugin} = require('./utils');
+const Utils = require('./utils');
 const logger = require('aurelia-logging').getLogger('BundledSource');
 
 exports.BundledSource = class {
@@ -58,6 +58,10 @@ exports.BundledSource = class {
     return this.bundler.loaderConfig;
   }
 
+  _getUseCache() {
+    return this.bundler.buildOptions.isApplicable('cache');
+  }
+
   get moduleId() {
     if (this._moduleId) return this._moduleId;
 
@@ -123,27 +127,9 @@ exports.BundledSource = class {
       // support Node.js's json module
       let contents = `define(\'${moduleId}\',[],function(){return JSON.parse(${JSON.stringify(this.contents)});});\n`;
       // be nice to requirejs json plugin users, add json! prefix
-      contents += `define(\'${moduleIdWithPlugin(moduleId, 'json', loaderType)}\',[\'${moduleId}\'],function(m){return m;});\n`;
+      contents += `define(\'${Utils.moduleIdWithPlugin(moduleId, 'json', loaderType)}\',[\'${moduleId}\'],function(m){return m;});\n`;
       this.contents = contents;
     } else {
-      // forceCjsWrap bypasses a r.js parse bug.
-      // See lib/amodro-trace/read/cjs.js for more info.
-      let forceCjsWrap = !!modulePath.match(/\/(cjs|commonjs)\//i);
-      let contents;
-
-      try {
-        contents = cjsTransform(modulePath, this.contents, forceCjsWrap);
-      } catch (ignore) {
-        // file is not in amd/cjs format, try native es module
-        try {
-          contents = esTransform(modulePath, this.contents);
-        } catch (e) {
-          logger.error('Could not convert to AMD module, skipping ' + modulePath);
-          logger.error('Error was: ' + e);
-          contents = this.contents;
-        }
-      }
-
       deps = [];
 
       let context = {pkgsMainMap: {}, config: {shim: {}}};
@@ -197,24 +183,72 @@ exports.BundledSource = class {
         }
       }
 
-      const writeTransform = allWriteTransforms({
+      const opts = {
         stubModules: loaderConfig.stubModules,
         wrapShim: wrapShim || loaderConfig.wrapShim,
         replacement: replacement
-      });
+      };
+
+      // Use cache for js files to avoid expensive parsing and transform.
+      let cache;
+      let hash;
+      const useCache = this._getUseCache();
+      if (useCache) {
+        // Only hash on moduleId, opts and contents.
+        // This ensures cache on npm packages can be shared
+        // among different apps.
+        const key = [
+          moduleId,
+          JSON.stringify(context),
+          JSON.stringify(opts),
+          this.contents // contents here is after gulp transpile task
+        ].join('|');
+        hash = Utils.generateHash(key);
+        cache = Utils.getCache(hash);
+      }
 
-      contents = writeTransform(context, moduleId, modulePath, contents);
+      if (cache) {
+        this.contents = cache.contents;
+        deps = cache.deps;
+      } else {
+        let contents;
+        // forceCjsWrap bypasses a r.js parse bug.
+        // See lib/amodro-trace/read/cjs.js for more info.
+        let forceCjsWrap = !!modulePath.match(/\/(cjs|commonjs)\//i);
 
-      const tracedDeps = findDeps(modulePath, contents);
-      if (tracedDeps && tracedDeps.length) {
-        deps.push.apply(deps, tracedDeps);
+        try {
+          contents = cjsTransform(modulePath, this.contents, forceCjsWrap);
+        } catch (ignore) {
+          // file is not in amd/cjs format, try native es module
+          try {
+            contents = esTransform(modulePath, this.contents);
+          } catch (e) {
+            logger.error('Could not convert to AMD module, skipping ' + modulePath);
+            logger.error('Error was: ' + e);
+            contents = this.contents;
+          }
+        }
+
+        const writeTransform = allWriteTransforms(opts);
+        contents = writeTransform(context, moduleId, modulePath, contents);
+
+        const tracedDeps = findDeps(modulePath, contents);
+        if (tracedDeps && tracedDeps.length) {
+          deps.push.apply(deps, tracedDeps);
+        }
+        this.contents = contents;
+
+        // write cache
+        if (useCache && hash) {
+          Utils.setCache(hash, {
+            deps: deps,
+            contents: this.contents
+          });
+        }
       }
-      this.contents = contents;
     }
 
     this.requiresTransform = false;
-
-    // return deps
     if (!deps || deps.length === 0) return;
 
     let moduleIds = Array.from(new Set(deps)) // unique

lib/build/index.js

@@ -2,6 +2,8 @@
 const Bundler = require('./bundler').Bundler;
 const PackageAnalyzer = require('./package-analyzer').PackageAnalyzer;
 const PackageInstaller = require('./package-installer').PackageInstaller;
+const cacheDir = require('./utils').cacheDir;
+const del = require('del');
 
 let bundler;
 let project;
@@ -55,6 +57,11 @@ exports.dest = function(opts) {
     .then(() => bundler.write());
 };
 
+exports.clearCache = function() {
+  // delete cache folder outside of cwd
+  return del(cacheDir, {force: true});
+};
+
 function buildLoaderConfig(p) {
   project = p || project;
   through = require('through2'); //dep of vinyl-fs

lib/build/utils.js

@@ -2,6 +2,7 @@
 const path = require('path');
 const crypto = require('crypto');
 const fs = require('../file-system');
+const tmpDir = require('os').tmpdir();
 
 exports.knownExtensions = ['.js', '.json', '.css', '.svg', '.html'];
 
@@ -38,14 +39,28 @@ exports.moduleIdWithPlugin = function(moduleId, pluginName, type) {
   }
 };
 
-exports.generateBundleName = function(contents, fileName, rev) {
-  let hash;
-  if (rev === true) {
-    hash = exports.generateHash(new Buffer(contents, 'utf-8'));
-  } else {
-    hash = rev;
+const CACHE_DIR = path.resolve(tmpDir, 'aurelia-cli-cache');
+exports.cacheDir = CACHE_DIR;
+
+function cachedFilePath(hash) {
+  const folder = hash.substr(0, 2);
+  const fileName = hash.substr(2);
+  return path.resolve(CACHE_DIR, folder, fileName);
+}
+
+exports.getCache = function(hash) {
+  const filePath = cachedFilePath(hash);
+  try {
+    return JSON.parse(fs.readFileSync(filePath));
+  } catch (e) {
+    // ignore
   }
-  return rev ? exports.generateHashedPath(fileName, hash) : fileName;
+};
+
+exports.setCache = function(hash, object) {
+  const filePath = cachedFilePath(hash);
+  // async write
+  fs.writeFile(filePath, JSON.stringify(object));
 };
 
 exports.runSequentially = function(tasks, cb) {
@@ -85,11 +100,8 @@ exports.revertHashedPath = function(pth, hash) {
   });
 };
 
-exports.generateHash = function(buf) {
-  if (!Buffer.isBuffer(buf)) {
-    throw new TypeError('Expected a buffer');
-  }
-  return crypto.createHash('md5').update(buf).digest('hex').slice(0, 10);
+exports.generateHash = function(bufOrStr) {
+  return crypto.createHash('md5').update(bufOrStr).digest('hex');
 };
 
 exports.escapeForRegex = function(str) {

lib/commands/new/buildsystems/cli/index.js

@@ -40,7 +40,9 @@ module.exports = function(project, options) {
     ],
     options: {
       minify: 'stage & prod',
-      sourcemaps: 'dev & stage'
+      sourcemaps: 'dev & stage',
+      rev: false,
+      cache: 'dev & stage'
     },
     bundles: [
       {
@@ -106,6 +108,8 @@ module.exports = function(project, options) {
   ).addToTasks(
     ProjectItem.resource('build.ext', 'tasks/build.ext', model.transpiler),
     ProjectItem.resource('build.json', 'tasks/build.json'),
+    ProjectItem.resource('clear-cache.ext', 'tasks/clear-cache.ext', model.transpiler),
+    ProjectItem.resource('clear-cache.json', 'tasks/clear-cache.json'),
     ProjectItem.resource('copy-files.ext', 'tasks/copy-files.ext', model.transpiler),
     ProjectItem.resource('run.ext', 'tasks/run.ext', model.transpiler),
     ProjectItem.resource('run.json', 'tasks/run.json'),

lib/commands/new/buildsystems/cli/transpilers/babel.js

@@ -25,6 +25,7 @@ module.exports = function(project) {
     'babel-polyfill',
     'babel-register',
     'gulp-babel',
+    'gulp-cache',
     'gulp-eslint'
   );
 };

lib/dependencies.json

@@ -44,6 +44,7 @@
   "file-loader": "^1.1.11",
   "gulp": "^4.0.0",
   "gulp-babel": "^7.0.1",
+  "gulp-cache": "^1.0.2",
   "gulp-changed-in-place": "^2.3.0",
   "gulp-eslint": "^4.0.2",
   "gulp-htmlmin": "^4.0.0",

lib/resources/tasks/clear-cache.js

@@ -0,0 +1,16 @@
+import gulp from 'gulp';
+import {build} from 'aurelia-cli';
+import cache from 'gulp-cache';
+
+function clearTraceCache() {
+  return build.clearCache();
+}
+
+function clearTranspileCache() {
+  return cache.clearAll();
+}
+
+export default gulp.series(
+  clearTraceCache,
+  clearTranspileCache
+);

lib/resources/tasks/clear-cache.json

@@ -0,0 +1,4 @@
+{
+  "name": "clear-cache",
+  "description": "Clear both transpile cache (only for esnext), and tracing-cache (for CLI built-in tracer)."
+}

lib/resources/tasks/clear-cache.ts

@@ -0,0 +1,6 @@
+import * as gulp from 'gulp';
+import {build} from 'aurelia-cli';
+
+export default function clearCache() {
+  return build.clearCache();
+}

lib/resources/tasks/transpile.js

@@ -5,24 +5,34 @@ import babel from 'gulp-babel';
 import sourcemaps from 'gulp-sourcemaps';
 import notify from 'gulp-notify';
 import rename from 'gulp-rename';
+import cache from 'gulp-cache';
 import project from '../aurelia.json';
-import {CLIOptions, build} from 'aurelia-cli';
+import {CLIOptions, build, Configuration} from 'aurelia-cli';
 
-function configureEnvironment() {
-  let env = CLIOptions.getEnvironment();
+let env = CLIOptions.getEnvironment();
+const buildOptions = new Configuration(project.build.options);
+const useCache = buildOptions.isApplicable('cache');
 
+function configureEnvironment() {
   return gulp.src(`aurelia_project/environments/${env}.js`)
     .pipe(changedInPlace({firstPass: true}))
     .pipe(rename('environment.js'))
     .pipe(gulp.dest(project.paths.root));
 }
 
 function buildJavaScript() {
+  let transpile = babel(project.transpiler.options);
+  if (useCache) {
+    // the cache directory is "gulp-cache/projName-env" inside folder require('os').tmpdir()
+    // use command 'au clear-cache' to purge all caches
+    transpile = cache(transpile, {name: project.name + '-' + env});
+  }
+
   return gulp.src(project.transpiler.source)
     .pipe(plumber({errorHandler: notify.onError('Error: <%= error.message %>')}))
     .pipe(changedInPlace({firstPass: true}))
     .pipe(sourcemaps.init())
-    .pipe(babel(project.transpiler.options))
+    .pipe(transpile)
     .pipe(build.bundle());
 }