Merge branch 'mg-introduce-webpack-dlls' into 'master'

Add basic webpack DLLPlugin support (experimental)

See merge request gitlab-org/gitlab!18407

.gitlab/ci/frontend.gitlab-ci.yml

@@ -244,7 +244,9 @@ webpack-dev-server:
   dependencies: ["setup-test-env", "compile-assets pull-cache"]
   variables:
     WEBPACK_MEMORY_TEST: "true"
+    WEBPACK_VENDOR_DLL: "true"
   script:
+    - yarn webpack-vendor
     - node --expose-gc node_modules/.bin/webpack-dev-server --config config/webpack.config.js
   artifacts:
     name: webpack-dev-server

config/helpers/vendor_dll_hash.js

+const fs = require('fs');
+const path = require('path');
+const crypto = require('crypto');
+
+const CACHE_PATHS = [
+  './config/webpack.config.js',
+  './config/webpack.vendor.config.js',
+  './package.json',
+  './yarn.lock',
+];
+
+const resolvePath = file => path.resolve(__dirname, '../..', file);
+const readFile = file => fs.readFileSync(file);
+const fileHash = buffer =>
+  crypto
+    .createHash('md5')
+    .update(buffer)
+    .digest('hex');
+
+module.exports = () => {
+  const fileBuffers = CACHE_PATHS.map(resolvePath).map(readFile);
+  return fileHash(Buffer.concat(fileBuffers)).substr(0, 12);
+};

config/webpack.config.js

+const fs = require('fs');
 const path = require('path');
 const glob = require('glob');
-const fs = require('fs');
 const webpack = require('webpack');
 const VueLoaderPlugin = require('vue-loader/lib/plugin');
 const StatsWriterPlugin = require('webpack-stats-plugin').StatsWriterPlugin;
@@ -8,8 +8,10 @@ const CompressionPlugin = require('compression-webpack-plugin');
 const MonacoWebpackPlugin = require('monaco-editor-webpack-plugin');
 const BundleAnalyzerPlugin = require('webpack-bundle-analyzer').BundleAnalyzerPlugin;
 const CopyWebpackPlugin = require('copy-webpack-plugin');
+const vendorDllHash = require('./helpers/vendor_dll_hash');
 
 const ROOT_PATH = path.resolve(__dirname, '..');
+const VENDOR_DLL = process.env.WEBPACK_VENDOR_DLL && process.env.WEBPACK_VENDOR_DLL !== 'false';
 const CACHE_PATH = process.env.WEBPACK_CACHE_PATH || path.join(ROOT_PATH, 'tmp/cache');
 const IS_PRODUCTION = process.env.NODE_ENV === 'production';
 const IS_DEV_SERVER = process.env.WEBPACK_DEV_SERVER === 'true';
@@ -113,6 +115,25 @@ if (IS_EE) {
   });
 }
 
+// if there is a compiled DLL with a matching hash string, use it
+let dll;
+
+if (VENDOR_DLL && !IS_PRODUCTION) {
+  const dllHash = vendorDllHash();
+  const dllCachePath = path.join(ROOT_PATH, `tmp/cache/webpack-dlls/${dllHash}`);
+  if (fs.existsSync(dllCachePath)) {
+    console.log(`Using vendor DLL found at: ${dllCachePath}`);
+    dll = {
+      manifestPath: path.join(dllCachePath, 'vendor.dll.manifest.json'),
+      cacheFrom: dllCachePath,
+      cacheTo: path.join(ROOT_PATH, `public/assets/webpack/dll.${dllHash}/`),
+      publicPath: `dll.${dllHash}/vendor.dll.bundle.js`,
+    };
+  } else {
+    console.log(`Warning: No vendor DLL found at: ${dllCachePath}. DllPlugin disabled.`);
+  }
+}
+
 module.exports = {
   mode: IS_PRODUCTION ? 'production' : 'development',
 
@@ -267,6 +288,11 @@ module.exports = {
           modules: false,
           assets: true,
         });
+
+        // tell our rails helper where to find the DLL files
+        if (dll) {
+          stats.dllAssets = dll.publicPath;
+        }
         return JSON.stringify(stats, null, 2);
       },
     }),
@@ -286,6 +312,21 @@ module.exports = {
       jQuery: 'jquery',
     }),
 
+    // reference our compiled DLL modules
+    dll &&
+      new webpack.DllReferencePlugin({
+        context: ROOT_PATH,
+        manifest: dll.manifestPath,
+      }),
+
+    dll &&
+      new CopyWebpackPlugin([
+        {
+          from: dll.cacheFrom,
+          to: dll.cacheTo,
+        },
+      ]),
+
     !IS_EE &&
       new webpack.NormalModuleReplacementPlugin(/^ee_component\/(.*)\.vue/, resource => {
         resource.request = path.join(

config/webpack.vendor.config.js

+const path = require('path');
+const webpack = require('webpack');
+const vendorDllHash = require('./helpers/vendor_dll_hash');
+
+const ROOT_PATH = path.resolve(__dirname, '..');
+
+const dllHash = vendorDllHash();
+const dllCachePath = path.join(ROOT_PATH, `tmp/cache/webpack-dlls/${dllHash}`);
+const dllPublicPath = `/assets/webpack/dll.${dllHash}/`;
+
+module.exports = {
+  mode: 'development',
+  resolve: {
+    extensions: ['.js'],
+  },
+
+  context: ROOT_PATH,
+
+  entry: {
+    vendor: [
+      'jquery',
+      'pdfjs-dist/build/pdf',
+      'pdfjs-dist/build/pdf.worker.min',
+      'sql.js',
+      'core-js',
+      'echarts',
+      'lodash',
+      'underscore',
+      'vuex',
+      'pikaday',
+      'vue/dist/vue.esm.js',
+      'at.js',
+      'jed',
+      'mermaid',
+      'katex',
+      'three',
+      'select2',
+      'moment',
+      'aws-sdk',
+      'sanitize-html',
+      'bootstrap/dist/js/bootstrap.js',
+      'sortablejs/modular/sortable.esm.js',
+      'popper.js',
+      'apollo-client',
+      'source-map',
+      'mousetrap',
+    ],
+  },
+
+  output: {
+    path: dllCachePath,
+    publicPath: dllPublicPath,
+    filename: '[name].dll.bundle.js',
+    chunkFilename: '[name].dll.chunk.js',
+    library: '[name]_[hash]',
+  },
+
+  plugins: [
+    new webpack.DllPlugin({
+      path: path.join(dllCachePath, '[name].dll.manifest.json'),
+      name: '[name]_[hash]',
+    }),
+  ],
+
+  node: {
+    fs: 'empty', // sqljs requires fs
+    setImmediate: false,
+  },
+
+  devtool: 'cheap-module-source-map',
+};

lib/gitlab/webpack/manifest.rb

@@ -12,11 +12,12 @@ module Gitlab
         def entrypoint_paths(source)
           raise ::Webpack::Rails::Manifest::WebpackError, manifest["errors"] unless manifest_bundled?
 
+          dll_assets = manifest.fetch("dllAssets", [])
           entrypoint = manifest["entrypoints"][source]
           if entrypoint && entrypoint["assets"]
             # Can be either a string or an array of strings.
             # Do not include source maps as they are not javascript
-            [entrypoint["assets"]].flatten.reject { |p| p =~ /.*\.map$/ }.map do |p|
+            [dll_assets, entrypoint["assets"]].flatten.reject { |p| p =~ /.*\.map$/ }.map do |p|
               "/#{::Rails.configuration.webpack.public_path}/#{p}"
             end
           else

lib/tasks/gitlab/assets.rake

@@ -8,6 +8,7 @@ namespace :gitlab do
         yarn:check
         gettext:po_to_json
         rake:assets:precompile
+        gitlab:assets:vendor
         webpack:compile
         gitlab:assets:fix_urls
       ].each(&Gitlab::TaskHelpers.method(:invoke_and_time_task))
@@ -49,5 +50,12 @@ namespace :gitlab do
         end
       end
     end
+
+    desc 'GitLab | Assets | Compile vendor assets'
+    task :vendor do
+      unless system('yarn webpack-vendor')
+        abort 'Error: Unable to compile webpack DLL.'.color(:red)
+      end
+    end
   end
 end

package.json

@@ -27,6 +27,7 @@
     "stylelint-create-utility-map": "node scripts/frontend/stylelint/stylelint-utility-map.js",
     "test": "node scripts/frontend/test",
     "webpack": "NODE_OPTIONS=\"--max-old-space-size=3584\" webpack --config config/webpack.config.js",
+    "webpack-vendor": "NODE_OPTIONS=\"--max-old-space-size=3584\" webpack --config config/webpack.vendor.config.js",
     "webpack-prod": "NODE_OPTIONS=\"--max-old-space-size=3584\" NODE_ENV=production webpack --config config/webpack.config.js"
   },
   "dependencies": {