Commit 0e1e4288 authored by Douwe Maan's avatar Douwe Maan

Merge branch 'sidekiq-job-throttling' into 'master'

Allow certain Sidekiq jobs to be throttled

## What does this MR do?

Allows certain slow running Sidekiq jobs to be throttled. It is disabled by default and can be enabled via the Application Settings. 

![Screen_Shot_2016-11-04_at_4.51.24_PM](/uploads/a1f1d24c693fcdb278602765cd404d94/Screen_Shot_2016-11-04_at_4.51.24_PM.png)

## Does this MR meet the acceptance criteria?

- [x] [CHANGELOG](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/CHANGELOG.md) entry added
- [x] [Documentation created/updated](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/doc/development/doc_styleguide.md)
- Tests
  - [x] Added for this feature/bug
  - [x] All builds are passing

## What are the relevant issue numbers?

Related to #23352

See merge request !7292
parents 52f29501 e840749b
...@@ -137,6 +137,7 @@ gem 'acts-as-taggable-on', '~> 4.0' ...@@ -137,6 +137,7 @@ gem 'acts-as-taggable-on', '~> 4.0'
gem 'sidekiq', '~> 4.2' gem 'sidekiq', '~> 4.2'
gem 'sidekiq-cron', '~> 0.4.0' gem 'sidekiq-cron', '~> 0.4.0'
gem 'redis-namespace', '~> 1.5.2' gem 'redis-namespace', '~> 1.5.2'
gem 'sidekiq-limit_fetch', '~> 3.4'
# HTTP requests # HTTP requests
gem 'httparty', '~> 0.13.3' gem 'httparty', '~> 0.13.3'
......
...@@ -685,6 +685,8 @@ GEM ...@@ -685,6 +685,8 @@ GEM
redis-namespace (>= 1.5.2) redis-namespace (>= 1.5.2)
rufus-scheduler (>= 2.0.24) rufus-scheduler (>= 2.0.24)
sidekiq (>= 4.0.0) sidekiq (>= 4.0.0)
sidekiq-limit_fetch (3.4.0)
sidekiq (>= 4)
simplecov (0.12.0) simplecov (0.12.0)
docile (~> 1.1.0) docile (~> 1.1.0)
json (>= 1.8, < 3) json (>= 1.8, < 3)
...@@ -961,6 +963,7 @@ DEPENDENCIES ...@@ -961,6 +963,7 @@ DEPENDENCIES
shoulda-matchers (~> 2.8.0) shoulda-matchers (~> 2.8.0)
sidekiq (~> 4.2) sidekiq (~> 4.2)
sidekiq-cron (~> 0.4.0) sidekiq-cron (~> 0.4.0)
sidekiq-limit_fetch (~> 3.4)
simplecov (= 0.12.0) simplecov (= 0.12.0)
slack-notifier (~> 1.2.0) slack-notifier (~> 1.2.0)
spinach-rails (~> 0.2.1) spinach-rails (~> 0.2.1)
......
...@@ -117,6 +117,8 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController ...@@ -117,6 +117,8 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
:send_user_confirmation_email, :send_user_confirmation_email,
:container_registry_token_expire_delay, :container_registry_token_expire_delay,
:enabled_git_access_protocol, :enabled_git_access_protocol,
:sidekiq_throttling_enabled,
:sidekiq_throttling_factor,
:housekeeping_enabled, :housekeeping_enabled,
:housekeeping_bitmaps_enabled, :housekeeping_bitmaps_enabled,
:housekeeping_incremental_repack_period, :housekeeping_incremental_repack_period,
...@@ -125,7 +127,8 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController ...@@ -125,7 +127,8 @@ class Admin::ApplicationSettingsController < Admin::ApplicationController
repository_storages: [], repository_storages: [],
restricted_visibility_levels: [], restricted_visibility_levels: [],
import_sources: [], import_sources: [],
disabled_oauth_sign_in_sources: [] disabled_oauth_sign_in_sources: [],
sidekiq_throttling_queues: []
) )
end end
end end
...@@ -100,4 +100,8 @@ module ApplicationSettingsHelper ...@@ -100,4 +100,8 @@ module ApplicationSettingsHelper
options_for_select(options, @application_setting.repository_storages) options_for_select(options, @application_setting.repository_storages)
end end
def sidekiq_queue_options_for_select
options_for_select(Sidekiq::Queue.all.map(&:name), @application_setting.sidekiq_throttling_queues)
end
end end
...@@ -19,6 +19,7 @@ class ApplicationSetting < ActiveRecord::Base ...@@ -19,6 +19,7 @@ class ApplicationSetting < ActiveRecord::Base
serialize :domain_whitelist, Array serialize :domain_whitelist, Array
serialize :domain_blacklist, Array serialize :domain_blacklist, Array
serialize :repository_storages serialize :repository_storages
serialize :sidekiq_throttling_queues, Array
cache_markdown_field :sign_in_text cache_markdown_field :sign_in_text
cache_markdown_field :help_page_text cache_markdown_field :help_page_text
...@@ -85,6 +86,15 @@ class ApplicationSetting < ActiveRecord::Base ...@@ -85,6 +86,15 @@ class ApplicationSetting < ActiveRecord::Base
presence: { message: 'Domain blacklist cannot be empty if Blacklist is enabled.' }, presence: { message: 'Domain blacklist cannot be empty if Blacklist is enabled.' },
if: :domain_blacklist_enabled? if: :domain_blacklist_enabled?
validates :sidekiq_throttling_factor,
numericality: { greater_than: 0, less_than: 1 },
presence: { message: 'Throttling factor cannot be empty if Sidekiq Throttling is enabled.' },
if: :sidekiq_throttling_enabled?
validates :sidekiq_throttling_queues,
presence: { message: 'Queues to throttle cannot be empty if Sidekiq Throttling is enabled.' },
if: :sidekiq_throttling_enabled?
validates :housekeeping_incremental_repack_period, validates :housekeeping_incremental_repack_period,
presence: true, presence: true,
numericality: { only_integer: true, greater_than: 0 } numericality: { only_integer: true, greater_than: 0 }
...@@ -180,6 +190,7 @@ class ApplicationSetting < ActiveRecord::Base ...@@ -180,6 +190,7 @@ class ApplicationSetting < ActiveRecord::Base
container_registry_token_expire_delay: 5, container_registry_token_expire_delay: 5,
repository_storages: ['default'], repository_storages: ['default'],
user_default_external: false, user_default_external: false,
sidekiq_throttling_enabled: false,
housekeeping_enabled: true, housekeeping_enabled: true,
housekeeping_bitmaps_enabled: true, housekeeping_bitmaps_enabled: true,
housekeeping_incremental_repack_period: 10, housekeeping_incremental_repack_period: 10,
......
...@@ -283,6 +283,31 @@ ...@@ -283,6 +283,31 @@
The amount of points to store in a single UDP packet. More points The amount of points to store in a single UDP packet. More points
results in fewer but larger UDP packets being sent. results in fewer but larger UDP packets being sent.
%fieldset
%legend Background Jobs
%p
These settings require a restart to take effect.
.form-group
.col-sm-offset-2.col-sm-10
.checkbox
= f.label :sidekiq_throttling_enabled do
= f.check_box :sidekiq_throttling_enabled
Enable Sidekiq Job Throttling
.help-block
Limit the amount of resources slow running jobs are assigned.
.form-group
= f.label :sidekiq_throttling_queues, 'Sidekiq queues to throttle', class: 'control-label col-sm-2'
.col-sm-10
= f.select :sidekiq_throttling_queues, sidekiq_queue_options_for_select, { include_hidden: false }, multiple: true, class: 'select2 select-wide', data: { field: 'sidekiq_throttling_queues' }
.help-block
Choose which queues you wish to throttle.
.form-group
= f.label :sidekiq_throttling_factor, 'Throttling Factor', class: 'control-label col-sm-2'
.col-sm-10
= f.number_field :sidekiq_throttling_factor, class: 'form-control', min: '0.01', max: '0.99', step: '0.01'
.help-block
The factor by which the queues should be throttled. A value between 0.0 and 1.0, exclusive.
%fieldset %fieldset
%legend Spam and Anti-bot Protection %legend Spam and Anti-bot Protection
.form-group .form-group
......
---
title: Added ability to throttle Sidekiq Jobs
merge_request: 7292
author:
...@@ -29,6 +29,8 @@ Sidekiq.configure_server do |config| ...@@ -29,6 +29,8 @@ Sidekiq.configure_server do |config|
end end
Sidekiq::Cron::Job.load_from_hash! cron_jobs Sidekiq::Cron::Job.load_from_hash! cron_jobs
Gitlab::SidekiqThrottler.execute!
# Database pool should be at least `sidekiq_concurrency` + 2 # Database pool should be at least `sidekiq_concurrency` + 2
# For more info, see: https://github.com/mperham/sidekiq/blob/master/4.0-Upgrade.md # For more info, see: https://github.com/mperham/sidekiq/blob/master/4.0-Upgrade.md
config = ActiveRecord::Base.configurations[Rails.env] || config = ActiveRecord::Base.configurations[Rails.env] ||
......
# See http://doc.gitlab.com/ce/development/migration_style_guide.html
# for more information on how to write migrations for GitLab.
class AddSidekiqThrottlingToApplicationSettings < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers
# Set this constant to true if this migration requires downtime.
DOWNTIME = false
# When a migration requires downtime you **must** uncomment the following
# constant and define a short and easy to understand explanation as to why the
# migration requires downtime.
# DOWNTIME_REASON = ''
# When using the methods "add_concurrent_index" or "add_column_with_default"
# you must disable the use of transactions as these methods can not run in an
# existing transaction. When using "add_concurrent_index" make sure that this
# method is the _only_ method called in the migration, any other changes
# should go in a separate migration. This ensures that upon failure _only_ the
# index creation fails and can be retried or reverted easily.
#
# To disable transactions uncomment the following line and remove these
# comments:
# disable_ddl_transaction!
def change
add_column :application_settings, :sidekiq_throttling_enabled, :boolean, default: false
add_column :application_settings, :sidekiq_throttling_queues, :string
add_column :application_settings, :sidekiq_throttling_factor, :decimal
end
end
...@@ -98,6 +98,9 @@ ActiveRecord::Schema.define(version: 20161109150329) do ...@@ -98,6 +98,9 @@ ActiveRecord::Schema.define(version: 20161109150329) do
t.text "help_page_text_html" t.text "help_page_text_html"
t.text "shared_runners_text_html" t.text "shared_runners_text_html"
t.text "after_sign_up_text_html" t.text "after_sign_up_text_html"
t.boolean "sidekiq_throttling_enabled", default: false
t.string "sidekiq_throttling_queues"
t.decimal "sidekiq_throttling_factor"
t.boolean "housekeeping_enabled", default: true, null: false t.boolean "housekeeping_enabled", default: true, null: false
t.boolean "housekeeping_bitmaps_enabled", default: true, null: false t.boolean "housekeeping_bitmaps_enabled", default: true, null: false
t.integer "housekeeping_incremental_repack_period", default: 10, null: false t.integer "housekeeping_incremental_repack_period", default: 10, null: false
......
# GitLab operations # GitLab operations
- [Sidekiq MemoryKiller](operations/sidekiq_memory_killer.md) - [Sidekiq MemoryKiller](operations/sidekiq_memory_killer.md)
- [Sidekiq Job throttling](operations/sidekiq_job_throttling.md)
- [Cleaning up Redis sessions](operations/cleaning_up_redis_sessions.md) - [Cleaning up Redis sessions](operations/cleaning_up_redis_sessions.md)
- [Understanding Unicorn and unicorn-worker-killer](operations/unicorn.md) - [Understanding Unicorn and unicorn-worker-killer](operations/unicorn.md)
- [Moving repositories to a new location](operations/moving_repositories.md) - [Moving repositories to a new location](operations/moving_repositories.md)
# Sidekiq Job throttling
> Note: Introduced with GitLab 8.14
When your GitLab installation needs to handle tens of thousands of background
jobs, it can be convenient to throttle queues that do not need to be executed
immediately, e.g. long running jobs like Pipelines, thus allowing jobs that do
need to be executed immediately to have access to more resources.
In order to accomplish this, you can limit the amount of workers that certain
slow running queues can have available. This is what we call Sidekiq Job
Throttling. Depending on your infrastructure, you might have different slow
running queues, which is why you can choose which queues you want to throttle
and by how much you want to throttle them.
These settings are available in the Application Settings of your GitLab
installation.
![Sidekiq Job Throttling](img/sidekiq_job_throttling.png)
The throttle factor determines the maximum number of workers a queue can run on.
This value gets multiplied by `:concurrency` value set in the Sidekiq settings
and rounded up to the closest full integer.
So, for example, you set the `:concurrency` to 25 and the `Throttling factor` to
0.1, the maximum workers assigned to the selected queues would be 3.
```ruby
queue_limit = (factor * Sidekiq.options[:concurrency]).ceil
```
After enabling the job throttling, you will need to restart your GitLab
instance, in order for the changes to take effect.
\ No newline at end of file
...@@ -23,6 +23,10 @@ module Gitlab ...@@ -23,6 +23,10 @@ module Gitlab
settings || fake_application_settings settings || fake_application_settings
end end
def sidekiq_throttling_enabled?
current_application_settings.sidekiq_throttling_enabled
end
def fake_application_settings def fake_application_settings
OpenStruct.new( OpenStruct.new(
default_projects_limit: Settings.gitlab['default_projects_limit'], default_projects_limit: Settings.gitlab['default_projects_limit'],
...@@ -50,6 +54,7 @@ module Gitlab ...@@ -50,6 +54,7 @@ module Gitlab
repository_checks_enabled: true, repository_checks_enabled: true,
container_registry_token_expire_delay: 5, container_registry_token_expire_delay: 5,
user_default_external: false, user_default_external: false,
sidekiq_throttling_enabled: false,
) )
end end
......
module Gitlab
class SidekiqThrottler
class << self
def execute!
if Gitlab::CurrentSettings.sidekiq_throttling_enabled?
Gitlab::CurrentSettings.current_application_settings.sidekiq_throttling_queues.each do |queue|
Sidekiq::Queue[queue].limit = queue_limit
end
end
end
private
def queue_limit
@queue_limit ||=
begin
factor = Gitlab::CurrentSettings.current_application_settings.sidekiq_throttling_factor
(factor * Sidekiq.options[:concurrency]).ceil
end
end
end
end
end
require 'spec_helper'
describe Gitlab::SidekiqThrottler do
before do
Sidekiq.options[:concurrency] = 35
stub_application_setting(
sidekiq_throttling_enabled: true,
sidekiq_throttling_factor: 0.1,
sidekiq_throttling_queues: %w[build project_cache]
)
end
describe '#execute!' do
it 'sets limits on the selected queues' do
Gitlab::SidekiqThrottler.execute!
expect(Sidekiq::Queue['build'].limit).to eq 4
expect(Sidekiq::Queue['project_cache'].limit).to eq 4
end
it 'does not set limits on other queues' do
Gitlab::SidekiqThrottler.execute!
expect(Sidekiq::Queue['merge'].limit).to be_nil
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment