Merge branch '34092-remove-webpack-rails-gem' into 'master'

Resolve "Remove webpack-rails gem"

Closes #34092

See merge request gitlab-org/gitlab!42485

Gemfile

@@ -284,7 +284,6 @@ gem 'fast_blank'
 gem 'gitlab-chronic', '~> 0.10.5'
 gem 'gitlab_chronic_duration', '~> 0.10.6.2'
 
-gem 'webpack-rails', '~> 0.9.10'
 gem 'rack-proxy', '~> 0.6.0'
 
 gem 'sassc-rails', '~> 2.1.0'

Gemfile.lock

@@ -1218,8 +1218,6 @@ GEM
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff
-    webpack-rails (0.9.11)
-      railties (>= 3.2.0)
     websocket-driver (0.7.1)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
@@ -1499,7 +1497,6 @@ DEPENDENCIES
   vmstat (~> 2.3.0)
   webauthn (~> 2.3)
   webmock (~> 3.5.1)
-  webpack-rails (~> 0.9.10)
   wikicloth (= 0.8.1)
   yajl-ruby (~> 1.4.1)
 

config/application.rb

@@ -152,12 +152,18 @@ module Gitlab
     config.active_record.schema_format = :sql
 
     # Configure webpack
+    config.webpack = ActiveSupport::OrderedOptions.new
     config.webpack.config_file = "config/webpack.config.js"
     config.webpack.output_dir  = "public/assets/webpack"
     config.webpack.public_path = "assets/webpack"
+    config.webpack.manifest_filename = "manifest.json"
 
     # Webpack dev server configuration is handled in initializers/static_files.rb
+    config.webpack.dev_server = ActiveSupport::OrderedOptions.new
     config.webpack.dev_server.enabled = false
+    config.webpack.dev_server.host = 'localhost'
+    config.webpack.dev_server.port = 3808
+    config.webpack.dev_server.https = false
 
     config.action_mailer.delivery_job = "ActionMailer::MailDeliveryJob"
 

config/initializers/static_files.rb

@@ -21,9 +21,7 @@ if app.config.public_file_server.enabled
     settings = {
       enabled: true,
       host: dev_server.host,
-      port: dev_server.port,
-      manifest_host: dev_server.host,
-      manifest_port: dev_server.port
+      port: dev_server.port
     }
 
     if Rails.env.development?

lib/gitlab/webpack/manifest.rb

 # frozen_string_literal: true
 
-require 'webpack/rails/manifest'
+require 'net/http'
+require 'uri'
 
 module Gitlab
   module Webpack
-    class Manifest < ::Webpack::Rails::Manifest
-      # Raised if a supplied asset does not exist in the webpack manifest
+    class Manifest
+      # Raised if we can't read our webpack manifest for whatever reason
+      class ManifestLoadError < StandardError
+        def initialize(message, orig)
+          super "#{message} (original error #{orig})"
+        end
+      end
+
+      # Raised if webpack couldn't build one of your entry points
+      class WebpackError < StandardError
+        def initialize(errors)
+          super "Error in webpack compile, details follow below:\n#{errors.join("\n\n")}"
+        end
+      end
+
+      # Raised if a supplied entry point does not exist in the webpack manifest
       AssetMissingError = Class.new(StandardError)
 
       class << self
         def entrypoint_paths(source)
-          raise ::Webpack::Rails::Manifest::WebpackError, manifest["errors"] unless manifest_bundled?
+          raise WebpackError, manifest["errors"] unless manifest_bundled?
 
           dll_assets = manifest.fetch("dllAssets", [])
           entrypoint = manifest["entrypoints"][source]
@@ -20,10 +35,79 @@ module Gitlab
             [dll_assets, entrypoint["assets"]].flatten.reject { |p| p =~ /.*\.map$/ }.map do |p|
               "/#{::Rails.configuration.webpack.public_path}/#{p}"
             end
+          else
+            raise AssetMissingError, "Can't find asset '#{source}' in webpack manifest"
+          end
+        end
+
+        def asset_paths(source)
+          raise WebpackError, manifest["errors"] unless manifest_bundled?
+
+          paths = manifest["assetsByChunkName"][source]
+          if paths
+            # Can be either a string or an array of strings.
+            # Do not include source maps as they are not javascript
+            [paths].flatten.reject { |p| p =~ /.*\.map$/ }.map do |p|
+              "/#{::Rails.configuration.webpack.public_path}/#{p}"
+            end
           else
             raise AssetMissingError, "Can't find entry point '#{source}' in webpack manifest"
           end
         end
+
+        private
+
+        def manifest_bundled?
+          !manifest["errors"].any? { |error| error.include? "Module build failed" }
+        end
+
+        def manifest
+          if ::Rails.configuration.webpack.dev_server.enabled
+            # Don't cache if we're in dev server mode, manifest may change ...
+            load_manifest
+          else
+            # ... otherwise cache at class level, as JSON loading/parsing can be expensive
+            @manifest ||= load_manifest
+          end
+        end
+
+        def load_manifest
+          data = if ::Rails.configuration.webpack.dev_server.enabled
+                   load_dev_server_manifest
+                 else
+                   load_static_manifest
+                 end
+
+          Gitlab::Json.parse(data)
+        end
+
+        def load_dev_server_manifest
+          host = ::Rails.configuration.webpack.dev_server.host
+          port = ::Rails.configuration.webpack.dev_server.port
+          http = Net::HTTP.new(host, port)
+          http.use_ssl = ::Rails.configuration.webpack.dev_server.https
+          http.verify_mode = OpenSSL::SSL::VERIFY_NONE
+          http.get(dev_server_path).body
+        rescue => e
+          raise ManifestLoadError.new("Could not load manifest from webpack-dev-server at http://#{host}:#{port}#{dev_server_path} - is it running, and is stats-webpack-plugin loaded?", e)
+        end
+
+        def load_static_manifest
+          File.read(static_manifest_path)
+        rescue => e
+          raise ManifestLoadError.new("Could not load compiled manifest from #{static_manifest_path} - have you run `rake webpack:compile`?", e)
+        end
+
+        def static_manifest_path
+          ::Rails.root.join(
+            ::Rails.configuration.webpack.output_dir,
+            ::Rails.configuration.webpack.manifest_filename
+          )
+        end
+
+        def dev_server_path
+          "/#{::Rails.configuration.webpack.public_path}/#{::Rails.configuration.webpack.manifest_filename}"
+        end
       end
     end
   end

lib/tasks/gitlab/assets.rake

@@ -81,7 +81,7 @@ namespace :gitlab do
 
       if head_assets_md5 != master_assets_md5 || !public_assets_webpack_dir_exists
         FileUtils.rm_r(Tasks::Gitlab::Assets::PUBLIC_ASSETS_WEBPACK_DIR) if public_assets_webpack_dir_exists
-        Rake::Task['webpack:compile'].invoke
+        system('yarn webpack')
       end
     end
 

spec/lib/gitlab/webpack/manifest_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require 'json'
+
+RSpec.describe Gitlab::Webpack::Manifest do
+  let(:manifest) do
+    <<-EOF
+      {
+        "errors": [],
+        "assetsByChunkName": {
+          "entry1": [ "entry1.js", "entry1-a.js" ],
+          "entry2": "entry2.js"
+        }
+      }
+    EOF
+  end
+
+  shared_examples_for "a valid manifest" do
+    it "returns single entry asset paths from the manifest" do
+      expect(Gitlab::Webpack::Manifest.asset_paths("entry2")).to eq(["/public_path/entry2.js"])
+    end
+
+    it "returns multiple entry asset paths from the manifest" do
+      expect(Gitlab::Webpack::Manifest.asset_paths("entry1")).to eq(["/public_path/entry1.js", "/public_path/entry1-a.js"])
+    end
+
+    it "errors on a missing entry point" do
+      expect { Gitlab::Webpack::Manifest.asset_paths("herp") }.to raise_error(Gitlab::Webpack::Manifest::AssetMissingError)
+    end
+  end
+
+  before do
+    # Test that config variables work while we're here
+    ::Rails.configuration.webpack.dev_server.host = 'hostname'
+    ::Rails.configuration.webpack.dev_server.port = 2000
+    ::Rails.configuration.webpack.manifest_filename = "my_manifest.json"
+    ::Rails.configuration.webpack.public_path = "public_path"
+    ::Rails.configuration.webpack.output_dir = "manifest_output"
+  end
+
+  context "with dev server enabled" do
+    before do
+      ::Rails.configuration.webpack.dev_server.enabled = true
+
+      stub_request(:get, "http://hostname:2000/public_path/my_manifest.json").to_return(body: manifest, status: 200)
+    end
+
+    describe ".asset_paths" do
+      it_behaves_like "a valid manifest"
+
+      it "errors if we can't find the manifest" do
+        ::Rails.configuration.webpack.manifest_filename = "broken.json"
+        stub_request(:get, "http://hostname:2000/public_path/broken.json").to_raise(SocketError)
+
+        expect { Gitlab::Webpack::Manifest.asset_paths("entry1") }.to raise_error(Gitlab::Webpack::Manifest::ManifestLoadError)
+      end
+
+      describe "webpack errors" do
+        context "when webpack has 'Module build failed' errors in its manifest" do
+          it "errors" do
+            error_manifest = Gitlab::Json.parse(manifest).merge("errors" => [
+              "somethingModule build failed something",
+              "I am an error"
+            ]).to_json
+            stub_request(:get, "http://hostname:2000/public_path/my_manifest.json").to_return(body: error_manifest, status: 200)
+
+            expect { Gitlab::Webpack::Manifest.asset_paths("entry1") }.to raise_error(Gitlab::Webpack::Manifest::WebpackError)
+          end
+        end
+
+        context "when webpack does not have 'Module build failed' errors in its manifest" do
+          it "does not error" do
+            error_manifest = Gitlab::Json.parse(manifest).merge("errors" => ["something went wrong"]).to_json
+            stub_request(:get, "http://hostname:2000/public_path/my_manifest.json").to_return(body: error_manifest, status: 200)
+
+            expect { Gitlab::Webpack::Manifest.asset_paths("entry1") }.not_to raise_error
+          end
+        end
+
+        it "does not error if errors is present but empty" do
+          error_manifest = Gitlab::Json.parse(manifest).merge("errors" => []).to_json
+          stub_request(:get, "http://hostname:2000/public_path/my_manifest.json").to_return(body: error_manifest, status: 200)
+          expect { Gitlab::Webpack::Manifest.asset_paths("entry1") }.not_to raise_error
+        end
+      end
+    end
+  end
+
+  context "with dev server disabled" do
+    before do
+      ::Rails.configuration.webpack.dev_server.enabled = false
+      allow(File).to receive(:read).with(::Rails.root.join("manifest_output/my_manifest.json")).and_return(manifest)
+    end
+
+    describe ".asset_paths" do
+      it_behaves_like "a valid manifest"
+
+      it "errors if we can't find the manifest" do
+        ::Rails.configuration.webpack.manifest_filename = "broken.json"
+        allow(File).to receive(:read).with(::Rails.root.join("manifest_output/broken.json")).and_raise(Errno::ENOENT)
+        expect { Gitlab::Webpack::Manifest.asset_paths("entry1") }.to raise_error(Gitlab::Webpack::Manifest::ManifestLoadError)
+      end
+    end
+  end
+end