feat(compiler): Use timestamps to verify cache validity

.gitignore

@@ -1,3 +1,4 @@
 /node_modules/
 /dist/
+/coverage/
 npm-debug.*.log

index.js

@@ -97,8 +97,22 @@ class HtmlWebpackPlugin {
     // Clear the cache once a new HtmlWebpackPlugin is added
     childCompiler.clearCache(compiler);
 
-    compiler.hooks.compile.tap('HtmlWebpackPlugin', () => {
+    // Register all HtmlWebpackPlugins instances at the child compiler
+    compiler.hooks.thisCompilation.tap('HtmlWebpackPlugin', (compilation) => {
+      // Clear the cache if the child compiler is outdated
+      if (childCompiler.hasOutDatedTemplateCache(compilation)) {
+        childCompiler.clearCache(compiler);
+      }
+      // Add this instances template to the child compiler
       childCompiler.addTemplateToCompiler(compiler, this.options.template);
+      // Add file dependencies of child compiler to parent compiler
+      // to keep them watched even if we get the result from the cache
+      compilation.hooks.additionalChunkAssets.tap('HtmlWebpackPlugin', () => {
+        const childCompilerDependencies = childCompiler.getFileDependencies(compiler);
+        childCompilerDependencies.forEach(fileDependency => {
+          compilation.compilationDependencies.add(fileDependency);
+        });
+      });
     });
 
     // setup hooks for third party plugins
@@ -111,12 +125,13 @@ class HtmlWebpackPlugin {
           compilation.errors.push(prettyError(err, compiler.context).toString());
           return {
             content: self.options.showErrors ? prettyError(err, compiler.context).toJsonHtml() : 'ERROR',
-            outputName: self.options.filename
+            outputName: self.options.filename,
+            hash: ''
           };
         })
         .then(compilationResult => {
           // If the compilation change didnt change the cache is valid
-          isCompilationCached = compilationResult.hash && self.childCompilerHash === compilationResult.hash;
+          isCompilationCached = Boolean(compilationResult.hash) && self.childCompilerHash === compilationResult.hash;
           self.childCompilerHash = compilationResult.hash;
           self.childCompilationOutputName = compilationResult.outputName;
           callback();
@@ -131,9 +146,6 @@ class HtmlWebpackPlugin {
        * @param {() => void} callback
       */
       (compilation, callback) => {
-        // Clear the childCompilerCache
-        childCompiler.clearCache(compiler);
-
         // Get all entry point names for this html file
         const entryNames = Array.from(compilation.entrypoints.keys());
         const filteredEntryNames = self.filterChunks(entryNames, self.options.chunks, self.options.excludeChunks);

lib/compiler.js

@@ -33,36 +33,61 @@ class HtmlWebpackChildCompiler {
      */
     this.compilationPromise;
     /**
-     * @type {Date}
+     * @type {number}
      */
-    this.compilationStarted;
+    this.compilationStartedTimestamp;
+    /**
+     * @type {number}
+     */
+    this.compilationEndedTimestamp;
     /**
      * All file dependencies of the child compiler
      * @type {string[]}
      */
     this.fileDependencies = [];
+    /**
+     * Store if the cache was already verified for the given compilation
+     * @type {WeakMap<WebpackCompilation, boolean>}}
+     */
+    this.cacheVerifiedForCompilation = new WeakMap();
   }
 
   /**
    * Add a templatePath to the child compiler
    * The given template will be compiled by `compileTemplates`
    * @param {string} template - The webpack path to the template e.g. `'!!html-loader!index.html'`
+   * @returns {boolean} true if the template is new
    */
   addTemplate (template) {
     const templateId = this.templates.indexOf(template);
     // Don't add the template to the compiler if a similar template was already added
     if (templateId !== -1) {
-      return templateId;
+      return false;
     }
     // A child compiler can compile only once
     // throw an error if a new template is added after the compilation started
-    if (this.compilationPromise) {
+    if (this.isCompiling()) {
       throw new Error('New templates can only be added before `compileTemplates` was called.');
     }
     // Add the template to the childCompiler
-    const newTemplateId = this.templates.length;
     this.templates.push(template);
-    return newTemplateId;
+    // Mark the cache invalid
+    return true;
+  }
+
+  /**
+   * Returns true if the childCompiler is currently compiling
+   * @retuns {boolean}
+   */
+  isCompiling () {
+    return !this.didCompile() && this.compilationStartedTimestamp !== undefined;
+  }
+
+  /**
+   * Returns true if the childCOmpiler is done compiling
+   */
+  didCompile () {
+    return this.compilationEndedTimestamp !== undefined;
   }
 
   /**
@@ -116,7 +141,7 @@ class HtmlWebpackChildCompiler {
       new SingleEntryPlugin(childCompiler.context, template, `HtmlWebpackPlugin_${index}`).apply(childCompiler);
     });
 
-    this.compilationStarted = new Date();
+    this.compilationStartedTimestamp = new Date().getTime();
     this.compilationPromise = new Promise((resolve, reject) => {
       childCompiler.runAsChild((err, entries, childCompilation) => {
         // Extract templates
@@ -152,12 +177,44 @@ class HtmlWebpackChildCompiler {
             entry: entries[entryIndex]
           };
         });
+        this.compilationEndedTimestamp = new Date().getTime();
         resolve(result);
       });
     });
 
     return this.compilationPromise;
   }
+
+  /**
+   * Returns `false` if any template file depenendencies has changed
+   * for the given main compilation
+   *
+   * @param {WebpackCompilation} mainCompilation
+   * @returns {boolean}
+   */
+  hasOutDatedTemplateCache (mainCompilation) {
+    // Check if cache validation was already computed
+    const isCacheValid = this.cacheVerifiedForCompilation.get(mainCompilation);
+    if (isCacheValid !== undefined) {
+      return isCacheValid;
+    }
+    // If the compilation was never run there is no invalid cache
+    if (!this.compilationStartedTimestamp) {
+      this.cacheVerifiedForCompilation.set(mainCompilation, false);
+      return false;
+    }
+    // Check if any dependent file was changed after the last compilation
+    const fileTimestamps = mainCompilation.fileTimestamps;
+    const isCacheOutOfDate = this.fileDependencies.some((fileDependency) => {
+      const timestamp = fileTimestamps.get(fileDependency);
+      // If the timestamp is not known the file is new
+      // If the timestamp is larger then the file has changed
+      // Otherwise the file is still the same
+      return !timestamp || timestamp > this.compilationStartedTimestamp;
+    });
+    this.cacheVerifiedForCompilation.set(mainCompilation, isCacheOutOfDate);
+    return isCacheOutOfDate;
+  }
 }
 
 /**
@@ -214,10 +271,13 @@ const childCompilerCache = new WeakMap();
  * @param {WebpackCompiler} mainCompiler
  */
 function getChildCompiler (mainCompiler) {
-  if (!childCompilerCache[mainCompiler]) {
-    childCompilerCache[mainCompiler] = new HtmlWebpackChildCompiler();
+  const cachedChildCompiler = childCompilerCache.get(mainCompiler);
+  if (cachedChildCompiler) {
+    return cachedChildCompiler;
   }
-  return childCompilerCache[mainCompiler];
+  const newCompiler = new HtmlWebpackChildCompiler();
+  childCompilerCache.set(mainCompiler, newCompiler);
+  return newCompiler;
 }
 
 /**
@@ -226,7 +286,12 @@ function getChildCompiler (mainCompiler) {
  * @param {WebpackCompiler} mainCompiler
  */
 function clearCache (mainCompiler) {
-  delete (childCompilerCache[mainCompiler]);
+  const childCompiler = getChildCompiler(mainCompiler);
+  // If this childCompiler was already used
+  // remove the entire childCompiler from the cache
+  if (childCompiler.isCompiling() || childCompiler.didCompile()) {
+    childCompilerCache.delete(mainCompiler);
+  }
 }
 
 /**
@@ -235,7 +300,11 @@ function clearCache (mainCompiler) {
  * @param {string} templatePath
  */
 function addTemplateToCompiler (mainCompiler, templatePath) {
-  getChildCompiler(mainCompiler).addTemplate(templatePath);
+  const childCompiler = getChildCompiler(mainCompiler);
+  const isNew = childCompiler.addTemplate(templatePath);
+  if (isNew) {
+    clearCache(mainCompiler);
+  }
 }
 
 /**
@@ -252,6 +321,7 @@ function addTemplateToCompiler (mainCompiler, templatePath) {
 function compileTemplate (templatePath, outputFilename, mainCompilation) {
   const childCompiler = getChildCompiler(mainCompilation.compiler);
   return childCompiler.compileTemplates(mainCompilation).then((compiledTemplates) => {
+    if (!compiledTemplates[templatePath]) console.log(Object.keys(compiledTemplates), templatePath);
     const compiledTemplate = compiledTemplates[templatePath];
     // Replace [hash] placeholders in filename
     const outputName = mainCompilation.mainTemplate.hooks.assetPath.call(outputFilename, {
@@ -269,8 +339,32 @@ function compileTemplate (templatePath, outputFilename, mainCompilation) {
   });
 }
 
+/**
+ * Returns false if the cache is not valid anymore
+ *
+ * @param {WebpackCompilation} compilation
+ * @returns {boolean}
+ */
+function hasOutDatedTemplateCache (compilation) {
+  const childCompiler = childCompilerCache.get(compilation.compiler);
+  return childCompiler ? childCompiler.hasOutDatedTemplateCache(compilation) : false;
+}
+
+/**
+ * Return all file dependencies of the last child compilation
+ *
+ * @param {WebpackCompiler} compiler
+ * @returns {Array<string>}
+ */
+function getFileDependencies (compiler) {
+  const childCompiler = getChildCompiler(compiler);
+  return childCompiler.fileDependencies;
+}
+
 module.exports = {
   addTemplateToCompiler,
   compileTemplate,
-  clearCache
+  hasOutDatedTemplateCache,
+  clearCache,
+  getFileDependencies
 };

lib/loader.js

@@ -5,26 +5,24 @@ const _ = require('lodash');
 const loaderUtils = require('loader-utils');
 
 module.exports = function (source) {
-  if (this.cacheable) {
-    this.cacheable();
-  }
+  // Get templating options
+  const options = this.query !== '' ? loaderUtils.getOptions(this) : {};
+  const force = options.force || false;
+
   const allLoadersButThisOne = this.loaders.filter(function (loader) {
-    // Loader API changed from `loader.module` to `loader.normal` in Webpack 2.
-    return (loader.module || loader.normal) !== module.exports;
+    return loader.normal !== module.exports;
   });
-  // This loader shouldn't kick in if there is any other loader
-  if (allLoadersButThisOne.length > 0) {
+  // This loader shouldn't kick in if there is any other loader (unless it's explicitly enforced)
+  if (allLoadersButThisOne.length > 0 && !force) {
     return source;
   }
-  // Skip .js files
-  if (/\.js$/.test(this.resourcePath)) {
+  // Skip .js files (unless it's explicitly enforced)
+  if (/\.js$/.test(this.resourcePath) && !force) {
     return source;
   }
 
   // The following part renders the template with lodash as aminimalistic loader
   //
-  // Get templating options
-  const options = this.query !== '' ? loaderUtils.getOptions(this) : {};
   const template = _.template(source, _.defaults(options, { interpolate: /<%=([\s\S]+?)%>/g, variable: 'data' }));
   // Require !!lodash - using !! will disable all loaders (e.g. babel)
   return 'var _ = require(' + loaderUtils.stringifyRequest(this, '!!' + require.resolve('lodash')) + ');' +

package.json

@@ -15,7 +15,7 @@
     "posttest": "tsc",
     "commit": "git-cz",
     "build-examples": "node examples/build-examples.js",
-    "test": "jest --runInBand",
+    "test": "jest --runInBand --verbose --coverage",
     "test-watch": "jest --runInBand --watch",
     "release": "standard-version"
   },
@@ -44,9 +44,9 @@
     "typescript": "2.9.1",
     "underscore-template-loader": "^0.7.3",
     "url-loader": "^0.5.7",
-    "webpack": "4.8.3",
+    "webpack": "4.1.0",
     "webpack-cli": "2.0.12",
-    "webpack-recompilation-simulator": "^1.3.0"
+    "webpack-recompilation-simulator": "^3.0.0"
   },
   "dependencies": {
     "@types/tapable": "1.0.2",

spec/basic.spec.js

@@ -33,6 +33,7 @@ var OUTPUT_DIR = path.resolve(__dirname, '../dist/basic-spec');
 
 jest.setTimeout(30000);
 process.on('unhandledRejection', r => console.log(r));
+process.traceDeprecation = true;
 
 function testHtmlPlugin (webpackConfig, expectedResults, outputFile, done, expectErrors, expectWarnings) {
   outputFile = outputFile || 'index.html';