Merge branch 'add_reliable_fetcher' into 'master'

Add reliable sidekiq fetcher

See merge request gitlab-org/gitlab-ce!21715

Gemfile

@@ -295,6 +295,7 @@ gem 'peek-mysql2', '~> 1.1.0', group: :mysql
 gem 'peek-pg', '~> 1.3.0', group: :postgres
 gem 'peek-rblineprof', '~> 0.2.0'
 gem 'peek-redis', '~> 1.2.0'
+gem 'gitlab-sidekiq-fetcher', require: 'sidekiq-reliable-fetch'
 
 # Metrics
 group :metrics do

Gemfile.lock

@@ -301,6 +301,8 @@ GEM
       mime-types (>= 1.16)
       posix-spawn (~> 0.3)
     gitlab-markup (1.6.4)
+    gitlab-sidekiq-fetcher (0.3.0)
+      sidekiq (~> 5)
     gitlab-styles (2.4.1)
       rubocop (~> 0.54.0)
       rubocop-gitlab-security (~> 0.1.0)
@@ -1031,6 +1033,7 @@ DEPENDENCIES
   gitlab-flowdock-git-hook (~> 1.0.1)
   gitlab-gollum-lib (~> 4.2)
   gitlab-markup (~> 1.6.4)
+  gitlab-sidekiq-fetcher
   gitlab-styles (~> 2.4)
   gitlab_omniauth-ldap (~> 2.0.4)
   gon (~> 6.2)

Gemfile.rails5.lock

@@ -304,6 +304,8 @@ GEM
       mime-types (>= 1.16)
       posix-spawn (~> 0.3)
     gitlab-markup (1.6.4)
+    gitlab-sidekiq-fetcher (0.3.0)
+      sidekiq (~> 5)
     gitlab-styles (2.4.1)
       rubocop (~> 0.54.0)
       rubocop-gitlab-security (~> 0.1.0)
@@ -1040,6 +1042,7 @@ DEPENDENCIES
   gitlab-flowdock-git-hook (~> 1.0.1)
   gitlab-gollum-lib (~> 4.2)
   gitlab-markup (~> 1.6.4)
+  gitlab-sidekiq-fetcher
   gitlab-styles (~> 2.4)
   gitlab_omniauth-ldap (~> 2.0.4)
   gon (~> 6.2)

changelogs/unreleased/add_reliable_fetcher.yml

+---
+title: Use Reliable Sidekiq fetch
+merge_request: 21715
+author:
+type: fixed

config/initializers/sidekiq.rb

@@ -40,6 +40,10 @@ Sidekiq.configure_server do |config|
     ActiveRecord::Base.clear_all_connections!
   end
 
+  if Feature.enabled?(:gitlab_sidekiq_reliable_fetcher)
+    Sidekiq::ReliableFetcher.setup_reliable_fetch!(config)
+  end
+
   # Sidekiq-cron: load recurring jobs from gitlab.yml
   # UGLY Hack to get nested hash from settingslogic
   cron_jobs = JSON.parse(Gitlab.config.cron_jobs.to_json)
@@ -57,10 +61,10 @@ Sidekiq.configure_server do |config|
 
   Gitlab::SidekiqVersioning.install!
 
-  config = Gitlab::Database.config ||
+  db_config = Gitlab::Database.config ||
     Rails.application.config.database_configuration[Rails.env]
-  config['pool'] = Sidekiq.options[:concurrency]
-  ActiveRecord::Base.establish_connection(config)
+  db_config['pool'] = Sidekiq.options[:concurrency]
+  ActiveRecord::Base.establish_connection(db_config)
   Rails.logger.debug("Connection Pool size for Sidekiq Server is now: #{ActiveRecord::Base.connection.pool.instance_variable_get('@size')}")
 
   # Avoid autoload issue such as 'Mail::Parsers::AddressStruct'