From e1b1760c6c6061d44e0af40645ec33322bb38e8b Mon Sep 17 00:00:00 2001
From: Mike Greiling <mike@pixelcog.com>
Date: Tue, 17 Dec 2019 08:36:27 +0000
Subject: [PATCH] Add basic webpack DLLPlugin support
Introduces the ability to enable webpack DLL support by passing
the flag WEBPACK_VENDOR_DLL=true while running webpack or
webpack-dev-server.
---
.gitlab/ci/frontend.gitlab-ci.yml | 2 +
config/helpers/vendor_dll_hash.js | 23 ++++++++++
config/webpack.config.js | 43 ++++++++++++++++++-
config/webpack.vendor.config.js | 71 +++++++++++++++++++++++++++++++
lib/gitlab/webpack/manifest.rb | 3 +-
lib/tasks/gitlab/assets.rake | 8 ++++
package.json | 1 +
7 files changed, 149 insertions(+), 2 deletions(-)
create mode 100644 config/helpers/vendor_dll_hash.js
create mode 100644 config/webpack.vendor.config.js
diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml
index 0c83f87eac665..5bd19309528fa 100644
--- a/.gitlab/ci/frontend.gitlab-ci.yml
+++ b/.gitlab/ci/frontend.gitlab-ci.yml
@@ -244,7 +244,9 @@ webpack-dev-server:
dependencies: ["setup-test-env", "compile-assets pull-cache"]
variables:
WEBPACK_MEMORY_TEST: "true"
+ WEBPACK_VENDOR_DLL: "true"
script:
+ - yarn webpack-vendor
- node --expose-gc node_modules/.bin/webpack-dev-server --config config/webpack.config.js
artifacts:
name: webpack-dev-server
diff --git a/config/helpers/vendor_dll_hash.js b/config/helpers/vendor_dll_hash.js
new file mode 100644
index 0000000000000..cfd7be66ad33a
--- /dev/null
+++ b/config/helpers/vendor_dll_hash.js
@@ -0,0 +1,23 @@
+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+
+const CACHE_PATHS = [
+ './config/webpack.config.js',
+ './config/webpack.vendor.config.js',
+ './package.json',
+ './yarn.lock',
+];
+
+const resolvePath = file => path.resolve(__dirname, '../..', file);
+const readFile = file => fs.readFileSync(file);
+const fileHash = buffer =>
+ crypto
+ .createHash('md5')
+ .update(buffer)
+ .digest('hex');
+
+module.exports = () => {
+ const fileBuffers = CACHE_PATHS.map(resolvePath).map(readFile);
+ return fileHash(Buffer.concat(fileBuffers)).substr(0, 12);
+};
diff --git a/config/webpack.config.js b/config/webpack.config.js
index c0be2f66ca798..d85fa84c32f63 100644
--- a/config/webpack.config.js
+++ b/config/webpack.config.js
@@ -1,6 +1,6 @@
+const fs = require('fs');
const path = require('path');
const glob = require('glob');
-const fs = require('fs');
const webpack = require('webpack');
const VueLoaderPlugin = require('vue-loader/lib/plugin');
const StatsWriterPlugin = require('webpack-stats-plugin').StatsWriterPlugin;
@@ -8,8 +8,10 @@ const CompressionPlugin = require('compression-webpack-plugin');
const MonacoWebpackPlugin = require('monaco-editor-webpack-plugin');
const BundleAnalyzerPlugin = require('webpack-bundle-analyzer').BundleAnalyzerPlugin;
const CopyWebpackPlugin = require('copy-webpack-plugin');
+const vendorDllHash = require('./helpers/vendor_dll_hash');
const ROOT_PATH = path.resolve(__dirname, '..');
+const VENDOR_DLL = process.env.WEBPACK_VENDOR_DLL && process.env.WEBPACK_VENDOR_DLL !== 'false';
const CACHE_PATH = process.env.WEBPACK_CACHE_PATH || path.join(ROOT_PATH, 'tmp/cache');
const IS_PRODUCTION = process.env.NODE_ENV === 'production';
const IS_DEV_SERVER = process.env.WEBPACK_DEV_SERVER === 'true';
@@ -113,6 +115,25 @@ if (IS_EE) {
});
}
+// if there is a compiled DLL with a matching hash string, use it
+let dll;
+
+if (VENDOR_DLL && !IS_PRODUCTION) {
+ const dllHash = vendorDllHash();
+ const dllCachePath = path.join(ROOT_PATH, `tmp/cache/webpack-dlls/${dllHash}`);
+ if (fs.existsSync(dllCachePath)) {
+ console.log(`Using vendor DLL found at: ${dllCachePath}`);
+ dll = {
+ manifestPath: path.join(dllCachePath, 'vendor.dll.manifest.json'),
+ cacheFrom: dllCachePath,
+ cacheTo: path.join(ROOT_PATH, `public/assets/webpack/dll.${dllHash}/`),
+ publicPath: `dll.${dllHash}/vendor.dll.bundle.js`,
+ };
+ } else {
+ console.log(`Warning: No vendor DLL found at: ${dllCachePath}. DllPlugin disabled.`);
+ }
+}
+
module.exports = {
mode: IS_PRODUCTION ? 'production' : 'development',
@@ -267,6 +288,11 @@ module.exports = {
modules: false,
assets: true,
});
+
+ // tell our rails helper where to find the DLL files
+ if (dll) {
+ stats.dllAssets = dll.publicPath;
+ }
return JSON.stringify(stats, null, 2);
},
}),
@@ -286,6 +312,21 @@ module.exports = {
jQuery: 'jquery',
}),
+ // reference our compiled DLL modules
+ dll &&
+ new webpack.DllReferencePlugin({
+ context: ROOT_PATH,
+ manifest: dll.manifestPath,
+ }),
+
+ dll &&
+ new CopyWebpackPlugin([
+ {
+ from: dll.cacheFrom,
+ to: dll.cacheTo,
+ },
+ ]),
+
!IS_EE &&
new webpack.NormalModuleReplacementPlugin(/^ee_component\/(.*)\.vue/, resource => {
resource.request = path.join(
diff --git a/config/webpack.vendor.config.js b/config/webpack.vendor.config.js
new file mode 100644
index 0000000000000..bddbf067d7c63
--- /dev/null
+++ b/config/webpack.vendor.config.js
@@ -0,0 +1,71 @@
+const path = require('path');
+const webpack = require('webpack');
+const vendorDllHash = require('./helpers/vendor_dll_hash');
+
+const ROOT_PATH = path.resolve(__dirname, '..');
+
+const dllHash = vendorDllHash();
+const dllCachePath = path.join(ROOT_PATH, `tmp/cache/webpack-dlls/${dllHash}`);
+const dllPublicPath = `/assets/webpack/dll.${dllHash}/`;
+
+module.exports = {
+ mode: 'development',
+ resolve: {
+ extensions: ['.js'],
+ },
+
+ context: ROOT_PATH,
+
+ entry: {
+ vendor: [
+ 'jquery',
+ 'pdfjs-dist/build/pdf',
+ 'pdfjs-dist/build/pdf.worker.min',
+ 'sql.js',
+ 'core-js',
+ 'echarts',
+ 'lodash',
+ 'underscore',
+ 'vuex',
+ 'pikaday',
+ 'vue/dist/vue.esm.js',
+ 'at.js',
+ 'jed',
+ 'mermaid',
+ 'katex',
+ 'three',
+ 'select2',
+ 'moment',
+ 'aws-sdk',
+ 'sanitize-html',
+ 'bootstrap/dist/js/bootstrap.js',
+ 'sortablejs/modular/sortable.esm.js',
+ 'popper.js',
+ 'apollo-client',
+ 'source-map',
+ 'mousetrap',
+ ],
+ },
+
+ output: {
+ path: dllCachePath,
+ publicPath: dllPublicPath,
+ filename: '[name].dll.bundle.js',
+ chunkFilename: '[name].dll.chunk.js',
+ library: '[name]_[hash]',
+ },
+
+ plugins: [
+ new webpack.DllPlugin({
+ path: path.join(dllCachePath, '[name].dll.manifest.json'),
+ name: '[name]_[hash]',
+ }),
+ ],
+
+ node: {
+ fs: 'empty', // sqljs requires fs
+ setImmediate: false,
+ },
+
+ devtool: 'cheap-module-source-map',
+};
diff --git a/lib/gitlab/webpack/manifest.rb b/lib/gitlab/webpack/manifest.rb
index 1d2aff5e5b4f1..d2c01bbd55e24 100644
--- a/lib/gitlab/webpack/manifest.rb
+++ b/lib/gitlab/webpack/manifest.rb
@@ -12,11 +12,12 @@ class << self
def entrypoint_paths(source)
raise ::Webpack::Rails::Manifest::WebpackError, manifest["errors"] unless manifest_bundled?
+ dll_assets = manifest.fetch("dllAssets", [])
entrypoint = manifest["entrypoints"][source]
if entrypoint && entrypoint["assets"]
# Can be either a string or an array of strings.
# Do not include source maps as they are not javascript
- [entrypoint["assets"]].flatten.reject { |p| p =~ /.*\.map$/ }.map do |p|
+ [dll_assets, entrypoint["assets"]].flatten.reject { |p| p =~ /.*\.map$/ }.map do |p|
"/#{::Rails.configuration.webpack.public_path}/#{p}"
end
else
diff --git a/lib/tasks/gitlab/assets.rake b/lib/tasks/gitlab/assets.rake
index 7a42e4e92a051..3aa1dc403d6dd 100644
--- a/lib/tasks/gitlab/assets.rake
+++ b/lib/tasks/gitlab/assets.rake
@@ -8,6 +8,7 @@ namespace :gitlab do
yarn:check
gettext:po_to_json
rake:assets:precompile
+ gitlab:assets:vendor
webpack:compile
gitlab:assets:fix_urls
].each(&Gitlab::TaskHelpers.method(:invoke_and_time_task))
@@ -49,5 +50,12 @@ namespace :gitlab do
end
end
end
+
+ desc 'GitLab | Assets | Compile vendor assets'
+ task :vendor do
+ unless system('yarn webpack-vendor')
+ abort 'Error: Unable to compile webpack DLL.'.color(:red)
+ end
+ end
end
end
diff --git a/package.json b/package.json
index e60ae6d5a80ec..aa11e35d3e6b2 100644
--- a/package.json
+++ b/package.json
@@ -27,6 +27,7 @@
"stylelint-create-utility-map": "node scripts/frontend/stylelint/stylelint-utility-map.js",
"test": "node scripts/frontend/test",
"webpack": "NODE_OPTIONS=\"--max-old-space-size=3584\" webpack --config config/webpack.config.js",
+ "webpack-vendor": "NODE_OPTIONS=\"--max-old-space-size=3584\" webpack --config config/webpack.vendor.config.js",
"webpack-prod": "NODE_OPTIONS=\"--max-old-space-size=3584\" NODE_ENV=production webpack --config config/webpack.config.js"
},
"dependencies": {
--
GitLab