Merge branch 'stanhu/gitlab-ce-add-eager-load-lib' into 'master'

Add eager load paths to help prevent dependency load issues with Sidekiq workers

_Originally opened at !3545 by @stanhu._

- - -

Relevant resources:

- https://github.com/mperham/sidekiq/wiki/FAQ#why-doesnt-sidekiq-autoload-my-rails-application-code
- https://github.com/mperham/sidekiq/issues/1281#issuecomment-27129904
- http://blog.arkency.com/2014/11/dont-forget-about-eager-load-when-extending-autoload
- https://github.com/rails/rails/blob/52ce6ece8c8f74064bb64e0a0b1ddd83092718e1/railties/lib/rails/engine.rb#L472-L479
- https://github.com/rails/rails/blob/v4.2.6/railties/lib/rails/paths.rb

Attempts to address #3661, #11896, #12769, #13521, #14131, #14589, #14759, #14825.

See merge request !3724

CHANGELOG

@@ -32,6 +32,7 @@ v 8.8.0 (unreleased)
   - API: Expose Issue#user_notes_count. !3126 (Anton Popov)
   - Files over 5MB can only be viewed in their raw form, files over 1MB without highlighting !3718
   - Add support for supressing text diffs using .gitattributes on the default branch (Matt Oakes)
+  - Add eager load paths to help prevent dependency load issues in Sidekiq workers. !3724
   - Added multiple colors for labels in dropdowns when dups happen.
   - Improve description for the Two-factor Authentication sign-in screen. (Connor Shea)
   - API support for the 'since' and 'until' operators on commit requests (Paco Guzman)

app/models/repository.rb

@@ -81,7 +81,7 @@ class Repository
   def commit(id = 'HEAD')
     return nil unless exists?
     commit = Gitlab::Git::Commit.find(raw_repository, id)
-    commit = Commit.new(commit, @project) if commit
+    commit = ::Commit.new(commit, @project) if commit
     commit
   rescue Rugged::OdbError
     nil

config/application.rb

 require File.expand_path('../boot', __FILE__)
 
 require 'rails/all'
-require 'devise'
-I18n.config.enforce_available_locales = false
+
 Bundler.require(:default, Rails.env)
-require_relative '../lib/gitlab/redis'
 
 module Gitlab
   class Application < Rails::Application
+    require_dependency Rails.root.join('lib/gitlab/redis')
+
     # Settings in config/environments/* take precedence over those specified here.
     # Application configuration should go into files in config/initializers
     # -- all .rb files in that directory are automatically loaded.
 
-    # Custom directories with classes and modules you want to be autoloadable.
-    config.autoload_paths.push(*%W(#{config.root}/lib
-                                   #{config.root}/app/models/hooks
-                                   #{config.root}/app/models/concerns
-                                   #{config.root}/app/models/project_services
-                                   #{config.root}/app/models/members))
+    # Sidekiq uses eager loading, but directories not in the standard Rails
+    # directories must be added to the eager load paths:
+    # https://github.com/mperham/sidekiq/wiki/FAQ#why-doesnt-sidekiq-autoload-my-rails-application-code
+    # Also, there is no need to add `lib` to autoload_paths since autoloading is
+    # configured to check for eager loaded paths:
+    # https://github.com/rails/rails/blob/v4.2.6/railties/lib/rails/engine.rb#L687
+    # This is a nice reference article on autoloading/eager loading:
+    # http://blog.arkency.com/2014/11/dont-forget-about-eager-load-when-extending-autoload
+    config.eager_load_paths.push(*%W(#{config.root}/lib
+                                     #{config.root}/app/models/ci
+                                     #{config.root}/app/models/hooks
+                                     #{config.root}/app/models/members
+                                     #{config.root}/app/models/project_services))
 
     # Only load the plugins named here, in the order given (default is alphabetical).
     # :all can be used as a placeholder for all plugins not explicitly named.

config/initializers/1_settings.rb

-require 'gitlab' # Load lib/gitlab.rb as soon as possible
+require_dependency Rails.root.join('lib/gitlab') # Load Gitlab as soon as possible
 
 class Settings < Settingslogic
   source ENV.fetch('GITLAB_CONFIG') { "#{Rails.root}/config/gitlab.yml" }

config/initializers/5_backend.rb

 # GIT over HTTP
-require Rails.root.join("lib", "gitlab", "backend", "grack_auth")
+require_dependency Rails.root.join('lib/gitlab/backend/grack_auth')
 
 # GIT over SSH
-require Rails.root.join("lib", "gitlab", "backend", "shell")
+require_dependency Rails.root.join('lib/gitlab/backend/shell')
 
 # GitLab shell adapter
-require Rails.root.join("lib", "gitlab", "backend", "shell_adapter")
+require_dependency Rails.root.join('lib/gitlab/backend/shell_adapter')
 
 required_version = Gitlab::VersionInfo.parse(Gitlab::Shell.version_required)
 current_version = Gitlab::VersionInfo.parse(Gitlab::Shell.new.version)

lib/api/api.rb

-Dir["#{Rails.root}/lib/api/*.rb"].each {|file| require file}
-
 module API
   class API < Grape::API
     include APIGuard
@@ -25,38 +23,39 @@ module API
     format :json
     content_type :txt, "text/plain"
 
-    helpers Helpers
-
-    mount Groups
-    mount GroupMembers
-    mount Users
-    mount Projects
-    mount Repositories
-    mount Issues
-    mount Milestones
-    mount Session
-    mount MergeRequests
-    mount Notes
-    mount Internal
-    mount SystemHooks
-    mount ProjectSnippets
-    mount ProjectMembers
-    mount DeployKeys
-    mount ProjectHooks
-    mount Services
-    mount Files
-    mount Commits
-    mount CommitStatus
-    mount Namespaces
-    mount Branches
-    mount Labels
-    mount Settings
-    mount Keys
-    mount Tags
-    mount Triggers
-    mount Builds
-    mount Variables
-    mount Runners
-    mount Licenses
+    # Ensure the namespace is right, otherwise we might load Grape::API::Helpers
+    helpers ::API::Helpers
+
+    mount ::API::Groups
+    mount ::API::GroupMembers
+    mount ::API::Users
+    mount ::API::Projects
+    mount ::API::Repositories
+    mount ::API::Issues
+    mount ::API::Milestones
+    mount ::API::Session
+    mount ::API::MergeRequests
+    mount ::API::Notes
+    mount ::API::Internal
+    mount ::API::SystemHooks
+    mount ::API::ProjectSnippets
+    mount ::API::ProjectMembers
+    mount ::API::DeployKeys
+    mount ::API::ProjectHooks
+    mount ::API::Services
+    mount ::API::Files
+    mount ::API::Commits
+    mount ::API::CommitStatuses
+    mount ::API::Namespaces
+    mount ::API::Branches
+    mount ::API::Labels
+    mount ::API::Settings
+    mount ::API::Keys
+    mount ::API::Tags
+    mount ::API::Triggers
+    mount ::API::Builds
+    mount ::API::Variables
+    mount ::API::Runners
+    mount ::API::Licenses
   end
 end

lib/api/api_guard.rb

@@ -2,171 +2,175 @@
 
 require 'rack/oauth2'
 
-module APIGuard
-  extend ActiveSupport::Concern
+module API
+  module APIGuard
+    extend ActiveSupport::Concern
 
-  included do |base|
-    # OAuth2 Resource Server Authentication
-    use Rack::OAuth2::Server::Resource::Bearer, 'The API' do |request|
-      # The authenticator only fetches the raw token string
+    included do |base|
+      # OAuth2 Resource Server Authentication
+      use Rack::OAuth2::Server::Resource::Bearer, 'The API' do |request|
+        # The authenticator only fetches the raw token string
 
-      # Must yield access token to store it in the env
-      request.access_token
-    end
+        # Must yield access token to store it in the env
+        request.access_token
+      end
 
-    helpers HelperMethods
+      helpers HelperMethods
 
-    install_error_responders(base)
-  end
+      install_error_responders(base)
+    end
 
-  # Helper Methods for Grape Endpoint
-  module HelperMethods
-    # Invokes the doorkeeper guard.
-    #
-    # If token is presented and valid, then it sets @current_user.
-    #
-    # If the token does not have sufficient scopes to cover the requred scopes,
-    # then it raises InsufficientScopeError.
-    #
-    # If the token is expired, then it raises ExpiredError.
-    #
-    # If the token is revoked, then it raises RevokedError.
-    #
-    # If the token is not found (nil), then it raises TokenNotFoundError.
-    #
-    # Arguments:
-    #
-    #   scopes: (optional) scopes required for this guard.
-    #           Defaults to empty array.
-    #
-    def doorkeeper_guard!(scopes: [])
-      if (access_token = find_access_token).nil?
-        raise TokenNotFoundError
-
-      else
-        case validate_access_token(access_token, scopes)
-        when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE
-          raise InsufficientScopeError.new(scopes)
-        when Oauth2::AccessTokenValidationService::EXPIRED
-          raise ExpiredError
-        when Oauth2::AccessTokenValidationService::REVOKED
-          raise RevokedError
-        when Oauth2::AccessTokenValidationService::VALID
-          @current_user = User.find(access_token.resource_owner_id)
+    # Helper Methods for Grape Endpoint
+    module HelperMethods
+      # Invokes the doorkeeper guard.
+      #
+      # If token is presented and valid, then it sets @current_user.
+      #
+      # If the token does not have sufficient scopes to cover the requred scopes,
+      # then it raises InsufficientScopeError.
+      #
+      # If the token is expired, then it raises ExpiredError.
+      #
+      # If the token is revoked, then it raises RevokedError.
+      #
+      # If the token is not found (nil), then it raises TokenNotFoundError.
+      #
+      # Arguments:
+      #
+      #   scopes: (optional) scopes required for this guard.
+      #           Defaults to empty array.
+      #
+      def doorkeeper_guard!(scopes: [])
+        if (access_token = find_access_token).nil?
+          raise TokenNotFoundError
+
+        else
+          case validate_access_token(access_token, scopes)
+          when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE
+            raise InsufficientScopeError.new(scopes)
+          when Oauth2::AccessTokenValidationService::EXPIRED
+            raise ExpiredError
+          when Oauth2::AccessTokenValidationService::REVOKED
+            raise RevokedError
+          when Oauth2::AccessTokenValidationService::VALID
+            @current_user = User.find(access_token.resource_owner_id)
+          end
         end
       end
-    end
 
-    def doorkeeper_guard(scopes: [])
-      if access_token = find_access_token
-        case validate_access_token(access_token, scopes)
-        when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE
-          raise InsufficientScopeError.new(scopes)
+      def doorkeeper_guard(scopes: [])
+        if access_token = find_access_token
+          case validate_access_token(access_token, scopes)
+          when Oauth2::AccessTokenValidationService::INSUFFICIENT_SCOPE
+            raise InsufficientScopeError.new(scopes)
 
-        when Oauth2::AccessTokenValidationService::EXPIRED
-          raise ExpiredError
+          when Oauth2::AccessTokenValidationService::EXPIRED
+            raise ExpiredError
 
-        when Oauth2::AccessTokenValidationService::REVOKED
-          raise RevokedError
+          when Oauth2::AccessTokenValidationService::REVOKED
+            raise RevokedError
 
-        when Oauth2::AccessTokenValidationService::VALID
-          @current_user = User.find(access_token.resource_owner_id)
+          when Oauth2::AccessTokenValidationService::VALID
+            @current_user = User.find(access_token.resource_owner_id)
+          end
         end
       end
-    end
 
-    def current_user
-      @current_user
-    end
+      def current_user
+        @current_user
+      end
 
-    private
-    def find_access_token
-      @access_token ||= Doorkeeper.authenticate(doorkeeper_request, Doorkeeper.configuration.access_token_methods)
-    end
+      private
 
-    def doorkeeper_request
-      @doorkeeper_request ||= ActionDispatch::Request.new(env)
-    end
+      def find_access_token
+        @access_token ||= Doorkeeper.authenticate(doorkeeper_request, Doorkeeper.configuration.access_token_methods)
+      end
 
-    def validate_access_token(access_token, scopes)
-      Oauth2::AccessTokenValidationService.validate(access_token, scopes: scopes)
-    end
-  end
+      def doorkeeper_request
+        @doorkeeper_request ||= ActionDispatch::Request.new(env)
+      end
 
-  module ClassMethods
-    # Installs the doorkeeper guard on the whole Grape API endpoint.
-    #
-    # Arguments:
-    #
-    #   scopes: (optional) scopes required for this guard.
-    #           Defaults to empty array.
-    #
-    def guard_all!(scopes: [])
-      before do
-        guard! scopes: scopes
+      def validate_access_token(access_token, scopes)
+        Oauth2::AccessTokenValidationService.validate(access_token, scopes: scopes)
       end
     end
 
-    private
-    def install_error_responders(base)
-      error_classes = [ MissingTokenError, TokenNotFoundError,
-                        ExpiredError, RevokedError, InsufficientScopeError]
+    module ClassMethods
+      # Installs the doorkeeper guard on the whole Grape API endpoint.
+      #
+      # Arguments:
+      #
+      #   scopes: (optional) scopes required for this guard.
+      #           Defaults to empty array.
+      #
+      def guard_all!(scopes: [])
+        before do
+          guard! scopes: scopes
+        end
+      end
 
-      base.send :rescue_from, *error_classes, oauth2_bearer_token_error_handler
-    end
+      private
 
-    def oauth2_bearer_token_error_handler
-      Proc.new do |e|
-        response =
-          case e
-          when MissingTokenError
-            Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new
-
-          when TokenNotFoundError
-            Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new(
-              :invalid_token,
-              "Bad Access Token.")
-
-          when ExpiredError
-            Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new(
-              :invalid_token,
-              "Token is expired. You can either do re-authorization or token refresh.")
-
-          when RevokedError
-            Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new(
-              :invalid_token,
-              "Token was revoked. You have to re-authorize from the user.")
-
-          when InsufficientScopeError
-            # FIXME: ForbiddenError (inherited from Bearer::Forbidden of Rack::Oauth2)
-            # does not include WWW-Authenticate header, which breaks the standard.
-            Rack::OAuth2::Server::Resource::Bearer::Forbidden.new(
-              :insufficient_scope,
-              Rack::OAuth2::Server::Resource::ErrorMethods::DEFAULT_DESCRIPTION[:insufficient_scope],
-              { scope: e.scopes })
-          end
+      def install_error_responders(base)
+        error_classes = [ MissingTokenError, TokenNotFoundError,
+                          ExpiredError, RevokedError, InsufficientScopeError]
 
-        response.finish
+        base.send :rescue_from, *error_classes, oauth2_bearer_token_error_handler
+      end
+
+      def oauth2_bearer_token_error_handler
+        Proc.new do |e|
+          response =
+            case e
+            when MissingTokenError
+              Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new
+
+            when TokenNotFoundError
+              Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new(
+                :invalid_token,
+                "Bad Access Token.")
+
+            when ExpiredError
+              Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new(
+                :invalid_token,
+                "Token is expired. You can either do re-authorization or token refresh.")
+
+            when RevokedError
+              Rack::OAuth2::Server::Resource::Bearer::Unauthorized.new(
+                :invalid_token,
+                "Token was revoked. You have to re-authorize from the user.")
+
+            when InsufficientScopeError
+              # FIXME: ForbiddenError (inherited from Bearer::Forbidden of Rack::Oauth2)
+              # does not include WWW-Authenticate header, which breaks the standard.
+              Rack::OAuth2::Server::Resource::Bearer::Forbidden.new(
+                :insufficient_scope,
+                Rack::OAuth2::Server::Resource::ErrorMethods::DEFAULT_DESCRIPTION[:insufficient_scope],
+                { scope: e.scopes })
+            end
+
+          response.finish
+        end
       end
     end
-  end
 
-  #
-  # Exceptions
-  #
+    #
+    # Exceptions
+    #
 
-  class MissingTokenError < StandardError; end
+    class MissingTokenError < StandardError; end
 
-  class TokenNotFoundError < StandardError; end
+    class TokenNotFoundError < StandardError; end
 
-  class ExpiredError < StandardError; end
+    class ExpiredError < StandardError; end
 
-  class RevokedError < StandardError; end
+    class RevokedError < StandardError; end
 
-  class InsufficientScopeError < StandardError
-    attr_reader :scopes
-    def initialize(scopes)
-      @scopes = scopes
+    class InsufficientScopeError < StandardError
+      attr_reader :scopes
+      def initialize(scopes)
+        @scopes = scopes
+      end
     end
   end
 end

lib/api/commit_statuses.rb

@@ -2,7 +2,7 @@ require 'mime/types'
 
 module API
   # Project commit statuses API
-  class CommitStatus < Grape::API
+  class CommitStatuses < Grape::API
     resource :projects do
       before { authenticate! }
 

lib/ci/api/api.rb

-Dir["#{Rails.root}/lib/ci/api/*.rb"].each {|file| require file}
-
 module Ci
   module API
     class API < Grape::API
-      include APIGuard
+      include ::API::APIGuard
       version 'v1', using: :path
 
       rescue_from ActiveRecord::RecordNotFound do
@@ -31,9 +29,9 @@ module Ci
       helpers ::API::Helpers
       helpers Gitlab::CurrentSettings
 
-      mount Builds
-      mount Runners
-      mount Triggers
+      mount ::Ci::API::Builds
+      mount ::Ci::API::Runners
+      mount ::Ci::API::Triggers
     end
   end
 end

lib/gitlab.rb

-require 'gitlab/git'
+require_dependency 'gitlab/git'
 
 module Gitlab
   def self.com?

spec/requests/api/commit_status_spec.rb → spec/requests/api/commit_statuses_spec.rb

 require 'spec_helper'
 
-describe API::CommitStatus, api: true do
+describe API::CommitStatuses, api: true do
   include ApiHelpers
 
   let!(:project) { create(:project) }