diff --git a/config/helpers/incremental_webpack_compiler.js b/config/helpers/incremental_webpack_compiler.js
deleted file mode 100644
index 5d4f9bd040d164fe76a47bafd3d701cf546840ac..0000000000000000000000000000000000000000
--- a/config/helpers/incremental_webpack_compiler.js
+++ /dev/null
@@ -1,131 +0,0 @@
-/* eslint-disable max-classes-per-file, no-underscore-dangle */
-const fs = require('fs');
-const path = require('path');
-
-const log = (msg, ...rest) => console.log(`IncrementalWebpackCompiler: ${msg}`, ...rest);
-
-// If we force a recompile immediately, the page reload doesn't seem to work.
-// Five seconds seem to work fine and the user can read the message
-const TIMEOUT = 5000;
-
-/* eslint-disable class-methods-use-this */
-class NoopCompiler {
- constructor() {
- this.enabled = false;
- }
-
- filterEntryPoints(entryPoints) {
- return entryPoints;
- }
-
- logStatus() {}
-
- setupMiddleware() {}
-}
-/* eslint-enable class-methods-use-this */
-
-class IncrementalWebpackCompiler {
- constructor(historyFilePath) {
- this.enabled = true;
- this.history = {};
- this.compiledEntryPoints = new Set([
- // Login page
- 'pages.sessions.new',
- // Explore page
- 'pages.root',
- ]);
- this.historyFilePath = historyFilePath;
- this._loadFromHistory();
- }
-
- filterEntryPoints(entrypoints) {
- return Object.fromEntries(
- Object.entries(entrypoints).map(([key, val]) => {
- if (this.compiledEntryPoints.has(key)) {
- return [key, val];
- }
- return [key, ['./webpack_non_compiled_placeholder.js']];
- }),
- );
- }
-
- logStatus(totalCount) {
- const current = this.compiledEntryPoints.size;
- log(`Currently compiling route entrypoints: ${current} of ${totalCount}`);
- }
-
- setupMiddleware(app, server) {
- app.use((req, res, next) => {
- const fileName = path.basename(req.url);
-
- /**
- * We are only interested in files that have a name like `pages.foo.bar.chunk.js`
- * because those are the ones corresponding to our entry points.
- *
- * This filters out hot update files that are for example named "pages.foo.bar.[hash].hot-update.js"
- */
- if (fileName.startsWith('pages.') && fileName.endsWith('.chunk.js')) {
- const chunk = fileName.replace(/\.chunk\.js$/, '');
-
- this._addToHistory(chunk);
-
- if (!this.compiledEntryPoints.has(chunk)) {
- log(`First time we are seeing ${chunk}. Adding to compilation.`);
-
- this.compiledEntryPoints.add(chunk);
-
- setTimeout(() => {
- server.middleware.invalidate(() => {
- if (server.sockets) {
- server.sockWrite(server.sockets, 'content-changed');
- }
- });
- }, TIMEOUT);
- }
- }
-
- next();
- });
- }
-
- // private methods
-
- _addToHistory(chunk) {
- if (!this.history[chunk]) {
- this.history[chunk] = { lastVisit: null, count: 0 };
- }
- this.history[chunk].lastVisit = Date.now();
- this.history[chunk].count += 1;
-
- try {
- fs.writeFileSync(this.historyFilePath, JSON.stringify(this.history), 'utf8');
- } catch (e) {
- log('Warning – Could not write to history', e.message);
- }
- }
-
- _loadFromHistory() {
- try {
- this.history = JSON.parse(fs.readFileSync(this.historyFilePath, 'utf8'));
- const entryPoints = Object.keys(this.history);
- log(`Successfully loaded history containing ${entryPoints.length} entry points`);
- /*
- TODO: Let's ask a few folks to give us their history file after a milestone of usage
- Then we can make smarter decisions on when to throw out rather than rendering everything
- Something like top 20/30/40 entries visited in the last 7/10/15 days might be sufficient
- */
- this.compiledEntryPoints = new Set([...this.compiledEntryPoints, ...entryPoints]);
- } catch (e) {
- log(`No history found...`);
- }
- }
-}
-
-module.exports = (enabled, historyFilePath) => {
- log(`Status – ${enabled ? 'enabled' : 'disabled'}`);
-
- if (enabled) {
- return new IncrementalWebpackCompiler(historyFilePath);
- }
- return new NoopCompiler();
-};
diff --git a/config/helpers/incremental_webpack_compiler/compiler.js b/config/helpers/incremental_webpack_compiler/compiler.js
new file mode 100644
index 0000000000000000000000000000000000000000..480d7fa3263fa511cf40493b61d572e6f04bb11a
--- /dev/null
+++ b/config/helpers/incremental_webpack_compiler/compiler.js
@@ -0,0 +1,117 @@
+/* eslint-disable max-classes-per-file */
+
+const path = require('path');
+const { History, HistoryWithTTL } = require('./history');
+const log = require('./log');
+
+const onRequestEntryPoint = (app, callback) => {
+ app.use((req, res, next) => {
+ const fileName = path.basename(req.url);
+
+ /**
+ * We are only interested in files that have a name like `pages.foo.bar.chunk.js`
+ * because those are the ones corresponding to our entry points.
+ *
+ * This filters out hot update files that are for example named "pages.foo.bar.[hash].hot-update.js"
+ */
+ if (fileName.startsWith('pages.') && fileName.endsWith('.chunk.js')) {
+ const entryPoint = fileName.replace(/\.chunk\.js$/, '');
+ callback(entryPoint);
+ }
+
+ next();
+ });
+};
+
+/**
+ * The NoopCompiler does nothing, following the null object pattern.
+ */
+class NoopCompiler {
+ constructor() {
+ this.enabled = false;
+ }
+
+ // eslint-disable-next-line class-methods-use-this
+ filterEntryPoints(entryPoints) {
+ return entryPoints;
+ }
+
+ // eslint-disable-next-line class-methods-use-this
+ logStatus() {}
+
+ // eslint-disable-next-line class-methods-use-this
+ setupMiddleware() {}
+}
+
+/**
+ * The HistoryOnlyCompiler only records which entry points have been requested.
+ * This is so that if the user disables incremental compilation, history is
+ * still recorded. If they later enable incremental compilation, that history
+ * can be used.
+ */
+class HistoryOnlyCompiler extends NoopCompiler {
+ constructor(historyFilePath) {
+ super();
+ this.history = new History(historyFilePath);
+ }
+
+ setupMiddleware(app) {
+ onRequestEntryPoint(app, (entryPoint) => {
+ this.history.onRequestEntryPoint(entryPoint);
+ });
+ }
+}
+
+// If we force a recompile immediately, the page reload doesn't seem to work.
+// Five seconds seem to work fine and the user can read the message
+const TIMEOUT = 5000;
+
+/**
+ * The IncrementalWebpackCompiler tracks which entry points have been
+ * requested, and only compiles entry points visited within the last `ttl`
+ * days.
+ */
+class IncrementalWebpackCompiler {
+ constructor(historyFilePath, ttl) {
+ this.enabled = true;
+ this.history = new HistoryWithTTL(historyFilePath, ttl);
+ }
+
+ filterEntryPoints(entrypoints) {
+ return Object.fromEntries(
+ Object.entries(entrypoints).map(([entryPoint, paths]) => {
+ if (this.history.isRecentlyVisited(entryPoint)) {
+ return [entryPoint, paths];
+ }
+ return [entryPoint, ['./webpack_non_compiled_placeholder.js']];
+ }),
+ );
+ }
+
+ logStatus(totalCount) {
+ log(`Currently compiling route entrypoints: ${this.history.size} of ${totalCount}`);
+ }
+
+ setupMiddleware(app, server) {
+ onRequestEntryPoint(app, (entryPoint) => {
+ const wasVisitedRecently = this.history.onRequestEntryPoint(entryPoint);
+ if (!wasVisitedRecently) {
+ log(`Have not visited ${entryPoint} recently. Adding to compilation.`);
+
+ setTimeout(() => {
+ server.middleware.invalidate(() => {
+ if (server.sockets) {
+ server.sockWrite(server.sockets, 'content-changed');
+ }
+ });
+ }, TIMEOUT);
+ }
+ });
+ }
+}
+
+module.exports = {
+ NoopCompiler,
+ HistoryOnlyCompiler,
+ IncrementalWebpackCompiler,
+};
diff --git a/config/helpers/incremental_webpack_compiler/history.js b/config/helpers/incremental_webpack_compiler/history.js
new file mode 100644
index 0000000000000000000000000000000000000000..5ff2a5292a39ab77b0f3594a3ed279f7fb00367b
--- /dev/null
+++ b/config/helpers/incremental_webpack_compiler/history.js
@@ -0,0 +1,125 @@
+/* eslint-disable max-classes-per-file, no-underscore-dangle */
+
+const fs = require('fs');
+const log = require('./log');
+
+/**
+ * The History class is responsible for tracking which entry points have been
+ * requested, and persisting/loading the history to/from disk.
+ */
+class History {
+ constructor(historyFilePath) {
+ this._historyFilePath = historyFilePath;
+ this._history = this._loadHistoryFile();
+ }
+
+ onRequestEntryPoint(entryPoint) {
+ const wasVisitedRecently = this.isRecentlyVisited(entryPoint);
+
+ if (!this._history[entryPoint]) {
+ this._history[entryPoint] = { lastVisit: null, count: 0 };
+ }
+
+ this._history[entryPoint].lastVisit = Date.now();
+ this._history[entryPoint].count += 1;
+
+ this._writeHistoryFile();
+
+ return wasVisitedRecently;
+ }
+
+ // eslint-disable-next-line class-methods-use-this
+ isRecentlyVisited() {
+ return true;
+ }
+
+ // eslint-disable-next-line class-methods-use-this
+ get size() {
+ return 0;
+ }
+
+ // Private methods
+
+ _writeHistoryFile() {
+ try {
+ fs.writeFileSync(this._historyFilePath, JSON.stringify(this._history), 'utf8');
+ } catch (e) {
+ log('Warning – Could not write to history', e.message);
+ }
+ }
+
+ _loadHistoryFile() {
+ let history = {};
+
+ try {
+ history = JSON.parse(fs.readFileSync(this._historyFilePath, 'utf8'));
+ const historySize = Object.keys(history).length;
+ log(`Successfully loaded history containing ${historySize} entry points`);
+ } catch (error) {
+ log(`Could not load history: ${error}`);
+ }
+
+ return history;
+ }
+}
+
+const MS_PER_DAY = 1000 * 60 * 60 * 24;
+
+/**
+ * The HistoryWithTTL class adds LRU-like behaviour onto the base History
+ * behaviour. Entry points visited within the last `ttl` days are considered
+ * "recent", and therefore should be eagerly compiled.
+ */
+class HistoryWithTTL extends History {
+ constructor(historyFilePath, ttl) {
+ super(historyFilePath);
+ this._ttl = ttl;
+ this._calculateRecentEntryPoints();
+ }
+
+ onRequestEntryPoint(entryPoint) {
+ const wasVisitedRecently = super.onRequestEntryPoint(entryPoint);
+
+ this._calculateRecentEntryPoints();
+
+ return wasVisitedRecently;
+ }
+
+ isRecentlyVisited(entryPoint) {
+ return this._recentEntryPoints.has(entryPoint);
+ }
+
+ get size() {
+ return this._recentEntryPoints.size;
+ }
+
+ // Private methods
+
+ _calculateRecentEntryPoints() {
+ const oldestVisitAllowed = Date.now() - MS_PER_DAY * this._ttl;
+
+ const recentEntryPoints = Object.entries(this._history).reduce(
+ (acc, [entryPoint, { lastVisit }]) => {
+ if (lastVisit > oldestVisitAllowed) {
+ acc.push(entryPoint);
+ }
+
+ return acc;
+ },
+ [],
+ );
+
+ this._recentEntryPoints = new Set([
+ // Login page
+ 'pages.sessions.new',
+ // Explore page
+ 'pages.root',
+ ...recentEntryPoints,
+ ]);
+ }
+}
+
+module.exports = {
+ History,
+ HistoryWithTTL,
+};
diff --git a/config/helpers/incremental_webpack_compiler/index.js b/config/helpers/incremental_webpack_compiler/index.js
new file mode 100644
index 0000000000000000000000000000000000000000..818266074904ccc725f46a60b7c0d1fa4def6e21
--- /dev/null
+++ b/config/helpers/incremental_webpack_compiler/index.js
@@ -0,0 +1,17 @@
+const { NoopCompiler, HistoryOnlyCompiler, IncrementalWebpackCompiler } = require('./compiler');
+const log = require('./log');
+
+module.exports = (recordHistory, enabled, historyFilePath, ttl) => {
+ if (!recordHistory) {
+ log(`Status – disabled`);
+ return new NoopCompiler();
+ }
+
+ if (enabled) {
+ log(`Status – enabled, ttl=${ttl}`);
+ return new IncrementalWebpackCompiler(historyFilePath, ttl);
+ }
+
+ log(`Status – history-only`);
+ return new HistoryOnlyCompiler(historyFilePath);
+};
diff --git a/config/helpers/incremental_webpack_compiler/log.js b/config/helpers/incremental_webpack_compiler/log.js
new file mode 100644
index 0000000000000000000000000000000000000000..6336cb0a78b22696a486c5f1148df979fef51930
--- /dev/null
+++ b/config/helpers/incremental_webpack_compiler/log.js
@@ -0,0 +1,3 @@
+const log = (msg, ...rest) => console.log(`IncrementalWebpackCompiler: ${msg}`, ...rest);
+
+module.exports = log;
diff --git a/config/webpack.config.js b/config/webpack.config.js
index b81b56110418f3fb2984507b7d7cbbf6e014f337..b5c508b30944961cbc288a0c1e476bb1872fc871 100644
--- a/config/webpack.config.js
+++ b/config/webpack.config.js
@@ -48,6 +48,8 @@ const INCREMENTAL_COMPILER_ENABLED =
IS_DEV_SERVER &&
process.env.DEV_SERVER_INCREMENTAL &&
process.env.DEV_SERVER_INCREMENTAL !== 'false';
+const INCREMENTAL_COMPILER_TTL = Number(process.env.DEV_SERVER_INCREMENTAL_TTL) || Infinity;
+const INCREMENTAL_COMPILER_RECORD_HISTORY = IS_DEV_SERVER && !process.env.CI;
const WEBPACK_REPORT = process.env.WEBPACK_REPORT && process.env.WEBPACK_REPORT !== 'false';
const WEBPACK_MEMORY_TEST =
process.env.WEBPACK_MEMORY_TEST && process.env.WEBPACK_MEMORY_TEST !== 'false';
@@ -69,8 +71,10 @@ let watchAutoEntries = [];
const defaultEntries = ['./main'];
const incrementalCompiler = createIncrementalWebpackCompiler(
+ INCREMENTAL_COMPILER_RECORD_HISTORY,
INCREMENTAL_COMPILER_ENABLED,
path.join(CACHE_PATH, 'incremental-webpack-compiler-history.json'),
+ INCREMENTAL_COMPILER_TTL,
);
function generateEntries() {