Commit 93571c43 authored by Robert Speicher's avatar Robert Speicher

Merge branch 'redis-connection-pool' into 'master'

Redis connection pool

Split from https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/3232

Having an easily accessible Redis connection pool allows us to do more
cool stuff with Redis in GitLab (instead of having to go through e.g.
the Rails cache).

See merge request !3521
parents 833808d7 cf669551
...@@ -149,6 +149,10 @@ gem 'version_sorter', '~> 2.0.0' ...@@ -149,6 +149,10 @@ gem 'version_sorter', '~> 2.0.0'
# Cache # Cache
gem "redis-rails", '~> 4.0.0' gem "redis-rails", '~> 4.0.0'
# Redis
gem 'redis', '~> 3.2'
gem 'connection_pool', '~> 2.0'
# Campfire integration # Campfire integration
gem 'tinder', '~> 1.10.0' gem 'tinder', '~> 1.10.0'
...@@ -229,7 +233,6 @@ group :metrics do ...@@ -229,7 +233,6 @@ group :metrics do
gem 'allocations', '~> 1.0', require: false, platform: :mri gem 'allocations', '~> 1.0', require: false, platform: :mri
gem 'method_source', '~> 0.8', require: false gem 'method_source', '~> 0.8', require: false
gem 'influxdb', '~> 0.2', require: false gem 'influxdb', '~> 0.2', require: false
gem 'connection_pool', '~> 2.0', require: false
end end
group :development do group :development do
......
...@@ -999,6 +999,7 @@ DEPENDENCIES ...@@ -999,6 +999,7 @@ DEPENDENCIES
rdoc (~> 3.6) rdoc (~> 3.6)
recaptcha recaptcha
redcarpet (~> 3.3.3) redcarpet (~> 3.3.3)
redis (~> 3.2)
redis-namespace redis-namespace
redis-rails (~> 4.0.0) redis-rails (~> 4.0.0)
request_store (~> 1.3.0) request_store (~> 1.3.0)
......
...@@ -4,11 +4,9 @@ require 'rails/all' ...@@ -4,11 +4,9 @@ require 'rails/all'
require 'devise' require 'devise'
I18n.config.enforce_available_locales = false I18n.config.enforce_available_locales = false
Bundler.require(:default, Rails.env) Bundler.require(:default, Rails.env)
require_relative '../lib/gitlab/redis_config' require_relative '../lib/gitlab/redis'
module Gitlab module Gitlab
REDIS_CACHE_NAMESPACE = 'cache:gitlab'
class Application < Rails::Application class Application < Rails::Application
# Settings in config/environments/* take precedence over those specified here. # Settings in config/environments/* take precedence over those specified here.
# Application configuration should go into files in config/initializers # Application configuration should go into files in config/initializers
...@@ -69,8 +67,8 @@ module Gitlab ...@@ -69,8 +67,8 @@ module Gitlab
end end
end end
redis_config_hash = Gitlab::RedisConfig.redis_store_options redis_config_hash = Gitlab::Redis.redis_store_options
redis_config_hash[:namespace] = REDIS_CACHE_NAMESPACE redis_config_hash[:namespace] = Gitlab::Redis::CACHE_NAMESPACE
redis_config_hash[:expires_in] = 2.weeks # Cache should not grow forever redis_config_hash[:expires_in] = 2.weeks # Cache should not grow forever
config.cache_store = :redis_store, redis_config_hash config.cache_store = :redis_store, redis_config_hash
......
...@@ -13,7 +13,7 @@ end ...@@ -13,7 +13,7 @@ end
if Rails.env.test? if Rails.env.test?
Gitlab::Application.config.session_store :cookie_store, key: "_gitlab_session" Gitlab::Application.config.session_store :cookie_store, key: "_gitlab_session"
else else
redis_config = Gitlab::RedisConfig.redis_store_options redis_config = Gitlab::Redis.redis_store_options
redis_config[:namespace] = 'session:gitlab' redis_config[:namespace] = 'session:gitlab'
Gitlab::Application.config.session_store( Gitlab::Application.config.session_store(
......
...@@ -2,7 +2,7 @@ SIDEKIQ_REDIS_NAMESPACE = 'resque:gitlab' ...@@ -2,7 +2,7 @@ SIDEKIQ_REDIS_NAMESPACE = 'resque:gitlab'
Sidekiq.configure_server do |config| Sidekiq.configure_server do |config|
config.redis = { config.redis = {
url: Gitlab::RedisConfig.url, url: Gitlab::Redis.url,
namespace: SIDEKIQ_REDIS_NAMESPACE namespace: SIDEKIQ_REDIS_NAMESPACE
} }
...@@ -29,7 +29,7 @@ end ...@@ -29,7 +29,7 @@ end
Sidekiq.configure_client do |config| Sidekiq.configure_client do |config|
config.redis = { config.redis = {
url: Gitlab::RedisConfig.url, url: Gitlab::Redis.url,
namespace: SIDEKIQ_REDIS_NAMESPACE namespace: SIDEKIQ_REDIS_NAMESPACE
} }
end end
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
<% <%
require "yaml" require "yaml"
require "json" require "json"
require_relative "lib/gitlab/redis_config" require_relative "lib/gitlab/redis"
rails_env = ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development" rails_env = ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development"
...@@ -18,7 +18,7 @@ if File.exists?(config_file) ...@@ -18,7 +18,7 @@ if File.exists?(config_file)
config['mailbox'] = "inbox" if config['mailbox'].nil? config['mailbox'] = "inbox" if config['mailbox'].nil?
if config['enabled'] && config['address'] if config['enabled'] && config['address']
redis_url = Gitlab::RedisConfig.new(rails_env).url redis_url = Gitlab::Redis.new(rails_env).url
%> %>
- -
:host: <%= config['host'].to_json %> :host: <%= config['host'].to_json %>
......
...@@ -43,7 +43,9 @@ module Gitlab ...@@ -43,7 +43,9 @@ module Gitlab
# false if the lease is already taken. # false if the lease is already taken.
def try_obtain def try_obtain
# Performing a single SET is atomic # Performing a single SET is atomic
!!redis.set(redis_key, '1', nx: true, ex: @timeout) Gitlab::Redis.with do |redis|
!!redis.set(redis_key, '1', nx: true, ex: @timeout)
end
end end
# No #cancel method. See comments above! # No #cancel method. See comments above!
......
module Gitlab module Gitlab
class RedisConfig class Redis
CACHE_NAMESPACE = 'cache:gitlab'
attr_reader :url attr_reader :url
# To be thread-safe we must be careful when writing the class instance
# variables @url and @pool. Because @pool depends on @url we need two
# mutexes to prevent deadlock.
URL_MUTEX = Mutex.new
POOL_MUTEX = Mutex.new
private_constant :URL_MUTEX, :POOL_MUTEX
def self.url def self.url
new.url @url || URL_MUTEX.synchronize { @url = new.url }
end
def self.with
if @pool.nil?
POOL_MUTEX.synchronize do
@pool = ConnectionPool.new { ::Redis.new(url: url) }
end
end
@pool.with { |redis| yield redis }
end end
def self.redis_store_options def self.redis_store_options
url = new.url url = new.url
redis_config_hash = Redis::Store::Factory.extract_host_options_from_uri(url) redis_config_hash = ::Redis::Store::Factory.extract_host_options_from_uri(url)
# Redis::Store does not handle Unix sockets well, so let's do it for them # Redis::Store does not handle Unix sockets well, so let's do it for them
redis_uri = URI.parse(url) redis_uri = URI.parse(url)
if redis_uri.scheme == 'unix' if redis_uri.scheme == 'unix'
......
...@@ -4,18 +4,19 @@ namespace :cache do ...@@ -4,18 +4,19 @@ namespace :cache do
desc "GitLab | Clear redis cache" desc "GitLab | Clear redis cache"
task :clear => :environment do task :clear => :environment do
redis = Redis.new(url: Gitlab::RedisConfig.url) Gitlab::Redis.with do |redis|
cursor = REDIS_SCAN_START_STOP cursor = REDIS_SCAN_START_STOP
loop do loop do
cursor, keys = redis.scan( cursor, keys = redis.scan(
cursor, cursor,
match: "#{Gitlab::REDIS_CACHE_NAMESPACE}*", match: "#{Gitlab::Redis::CACHE_NAMESPACE}*",
count: CLEAR_BATCH_SIZE count: CLEAR_BATCH_SIZE
) )
redis.del(*keys) if keys.any? redis.del(*keys) if keys.any?
break if cursor == REDIS_SCAN_START_STOP break if cursor == REDIS_SCAN_START_STOP
end
end end
end end
end end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment