Add incremental compiler to webpack dev server

In order to have quicker compilations while developing we are adding an
incremental compiler to webpack to render routes on demand.

When the developer is working on the dev server and enables the
incremental compiler with DEV_SERVER_INCREMENTAL=true, the new
functionality is enabled.

The biggest problem to solve here: How can we _know_ which entry point
to render and which not? Our current webpack integration with rails
requires the webpack manifest to have a list of all existing entry
points. So our incremental compiler takes the following approach:

Every compilation of webpack will run `generateEntries` which generates
a list of all our entry points. In that function we are able to replace
all page specific entrypoints and point them to an empty file,
unless we explicitly want them to compile.

In the webpack-dev-server itself we register a middleware which keeps
track of all the page specific bundles requested. Whenever a page
specific bundle is requested that hasn't been requested before, we add
it to the list of bundles we want compiled.

This approach allows us to dynamically change the entry points without a
need to restart webpack alltogether _and_ it works with hot module
reloading.

Rather than pointing to a blank javascript we are pointing to one which
renders an overlay to let the user know that webpack compiles the page
for the first time.

Additionally we keep a history of requested routes in `tmp/cache` in
order to keep the list of compiled entry points between sessions. In a
next iteration we can add a bit of logic and e.g. remove entry points
the developer hasn't been visiting in a week.

First results are really promising (on my machine):
 - Memory consumption when idling: 1600MB => 340MB
 - Max memory: ~2200MB => ~1000MB
 - Initial Compilation time: 58s => 15s
 - Recompile afer a file change: 13s => 3s
 - Visiting a new page that hasn't been visited before, it takes about
   four seconds to reload the page, seven seconds to completely load the
   page.

Currently the technique still watches all of the source files, so
changing an unrelated file will trigger a recompilation. This is however
a minor caveat and the same behavior that we currently have, maybe we
can optimize in the future.

app/assets/javascripts/webpack_non_compiled_placeholder.js

+const div = document.createElement('div');
+
+Object.assign(div.style, {
+  width: '100vw',
+  height: '100vh',
+  position: 'fixed',
+  top: 0,
+  left: 0,
+  'z-index': 100000,
+  background: 'rgba(0,0,0,0.9)',
+  'font-size': '25px',
+  'font-family': 'monospace',
+  color: 'white',
+  padding: '2.5em',
+  'text-align': 'center',
+});
+
+div.innerHTML = `
+<h1 style="color:white">🧙 Webpack is doing its magic 🧙</h1>
+<p>If you use Hot Module reloading, the page will reload in a few seconds.</p>
+<p>If you do not use Hot Module reloading, please <a href="">reload the page manually in a few seconds</a></p>
+`;
+
+document.body.append(div);

config/helpers/incremental_webpack_compiler.js

+const fs = require('fs');
+const path = require('path');
+
+const log = (msg, ...rest) => console.log(`IncrementalWebpackCompiler: ${msg}`, ...rest);
+
+// If we force a recompile immediately, the page reload doesn't seem to work.
+// Five seconds seem to work fine and the user can read the message
+const TIMEOUT = 5000;
+
+class NoopCompiler {
+  enabled = false;
+
+  filterEntryPoints(entryPoints) {
+    return entryPoints;
+  }
+
+  logStatus() {}
+
+  setupMiddleware() {}
+}
+
+class IncrementalWebpackCompiler extends NoopCompiler {
+  enabled = true;
+
+  constructor(historyFilePath) {
+    super();
+    this.history = {};
+    this.compiledEntryPoints = new Set([
+      // Login page
+      'pages.sessions.new',
+      // Explore page
+      'pages.root',
+    ]);
+    this.historyFilePath = historyFilePath;
+    this.loadFromHistory();
+  }
+
+  filterEntryPoints(entrypoints) {
+    return Object.fromEntries(
+      Object.entries(entrypoints).map(([key, val]) => {
+        if (this.compiledEntryPoints.has(key)) {
+          return [key, val];
+        }
+        return [key, ['./webpack_non_compiled_placeholder.js']];
+      }),
+    );
+  }
+
+  logStatus(totalCount) {
+    const current = this.compiledEntryPoints.size;
+    log(`Currently compiling route entrypoints: ${current} of ${totalCount}`);
+  }
+
+  setupMiddleware(app, server) {
+    app.use((req, res, next) => {
+      const fileName = path.basename(req.url);
+
+      /**
+       * We are only interested in files that have a name like `pages.foo.bar.chunk.js`
+       * because those are the ones corresponding to our entry points.
+       *
+       * This filters out hot update files that are for example named "pages.foo.bar.[hash].hot-update.js"
+       */
+      if (fileName.startsWith('pages.') && fileName.endsWith('.chunk.js')) {
+        const chunk = fileName.replace(/\.chunk\.js$/, '');
+
+        this.addToHistory(chunk);
+
+        if (!this.compiledEntryPoints.has(chunk)) {
+          log(`First time we are seeing ${chunk}. Adding to compilation.`);
+
+          this.compiledEntryPoints.add(chunk);
+
+          setTimeout(() => {
+            server.middleware.invalidate(() => {
+              if (server.sockets) {
+                server.sockWrite(server.sockets, 'content-changed');
+              }
+            });
+          }, TIMEOUT);
+        }
+      }
+
+      next();
+    });
+  }
+
+  // private methods
+
+  addToHistory(chunk) {
+    if (!this.history[chunk]) {
+      this.history[chunk] = { lastVisit: null, count: 0 };
+    }
+    this.history[chunk].lastVisit = Date.now();
+    this.history[chunk].count += 1;
+
+    try {
+      fs.writeFileSync(this.historyFilePath, JSON.stringify(this.history), 'utf8');
+    } catch (e) {
+      log('Warning – Could not write to history', e.message);
+    }
+  }
+
+  loadFromHistory() {
+    try {
+      this.history = JSON.parse(fs.readFileSync(this.historyFilePath, 'utf8'));
+      const entryPoints = Object.keys(this.history);
+      log(`Successfully loaded history containing ${entryPoints.length} entry points`);
+      /*
+      TODO: Let's ask a few folks to give us their history file after a milestone of usage
+            Then we can make smarter decisions on when to throw out rather than rendering everything
+            Something like top 20/30/40 entries visited in the last 7/10/15 days might be sufficient
+       */
+      this.compiledEntryPoints = new Set([...this.compiledEntryPoints, ...entryPoints]);
+    } catch (e) {
+      log(`No history found...`);
+    }
+  }
+}
+
+module.exports = (enabled, historyFilePath) => {
+  log(`Status – ${enabled ? 'enabled' : 'disabled'}`);
+
+  if (enabled) {
+    return new IncrementalWebpackCompiler(historyFilePath);
+  }
+  return new NoopCompiler();
+};

config/webpack.config.js

@@ -9,6 +9,7 @@ const MonacoWebpackPlugin = require('./plugins/monaco_webpack');
 const BundleAnalyzerPlugin = require('webpack-bundle-analyzer').BundleAnalyzerPlugin;
 const CopyWebpackPlugin = require('copy-webpack-plugin');
 const vendorDllHash = require('./helpers/vendor_dll_hash');
+const createIncrementalWebpackCompiler = require('./helpers/incremental_webpack_compiler');
 
 const ROOT_PATH = path.resolve(__dirname, '..');
 const VENDOR_DLL = process.env.WEBPACK_VENDOR_DLL && process.env.WEBPACK_VENDOR_DLL !== 'false';
@@ -23,6 +24,10 @@ const DEV_SERVER_ALLOWED_HOSTS =
   process.env.DEV_SERVER_ALLOWED_HOSTS && process.env.DEV_SERVER_ALLOWED_HOSTS.split(',');
 const DEV_SERVER_HTTPS = process.env.DEV_SERVER_HTTPS && process.env.DEV_SERVER_HTTPS !== 'false';
 const DEV_SERVER_LIVERELOAD = IS_DEV_SERVER && process.env.DEV_SERVER_LIVERELOAD !== 'false';
+const INCREMENTAL_COMPILER_ENABLED =
+  IS_DEV_SERVER &&
+  process.env.DEV_SERVER_INCREMENTAL &&
+  process.env.DEV_SERVER_INCREMENTAL !== 'false';
 const WEBPACK_REPORT = process.env.WEBPACK_REPORT && process.env.WEBPACK_REPORT !== 'false';
 const WEBPACK_MEMORY_TEST =
   process.env.WEBPACK_MEMORY_TEST && process.env.WEBPACK_MEMORY_TEST !== 'false';
@@ -48,6 +53,11 @@ let autoEntriesCount = 0;
 let watchAutoEntries = [];
 const defaultEntries = ['./main'];
 
+const incrementalCompiler = createIncrementalWebpackCompiler(
+  INCREMENTAL_COMPILER_ENABLED,
+  path.join(CACHE_PATH, 'incremental-webpack-compiler-history.json'),
+);
+
 function generateEntries() {
   // generate automatic entry points
   const autoEntries = {};
@@ -97,7 +107,7 @@ function generateEntries() {
     jira_connect_app: './jira_connect/index.js',
   };
 
-  return Object.assign(manualEntries, autoEntries);
+  return Object.assign(manualEntries, incrementalCompiler.filterEntryPoints(autoEntries));
 }
 
 const alias = {
@@ -495,9 +505,13 @@ module.exports = {
           watchAutoEntries.forEach((watchPath) => compilation.contextDependencies.add(watchPath));
 
           // report our auto-generated bundle count
-          console.log(
-            `${autoEntriesCount} entries from '/pages' automatically added to webpack output.`,
-          );
+          if (incrementalCompiler.enabled) {
+            incrementalCompiler.logStatus(autoEntriesCount);
+          } else {
+            console.log(
+              `${autoEntriesCount} entries from '/pages' automatically added to webpack output.`,
+            );
+          }
 
           callback();
         });
@@ -576,8 +590,10 @@ module.exports = {
     */
     new webpack.IgnorePlugin(/moment/, /pikaday/),
   ].filter(Boolean),
-
   devServer: {
+    before(app, server) {
+      incrementalCompiler.setupMiddleware(app, server);
+    },
     host: DEV_SERVER_HOST,
     port: DEV_SERVER_PORT,
     public: DEV_SERVER_PUBLIC_ADDR,