Commit ee721e12 authored by Douwe Maan's avatar Douwe Maan

Merge branch 'brodock/gitlab-ce-feature/redis-sentinel'

# Conflicts:
#	lib/gitlab/redis.rb
parents bd1b8ae9 ed0a7c25
...@@ -20,6 +20,7 @@ v 8.11.0 (unreleased) ...@@ -20,6 +20,7 @@ v 8.11.0 (unreleased)
- Optimize maximum user access level lookup in loading of notes - Optimize maximum user access level lookup in loading of notes
- Add "No one can push" as an option for protected branches. !5081 - Add "No one can push" as an option for protected branches. !5081
- Improve performance of AutolinkFilter#text_parse by using XPath - Improve performance of AutolinkFilter#text_parse by using XPath
- Add experimental Redis Sentinel support !1877
- Environments have an url to link to - Environments have an url to link to
- Update `timeago` plugin to use multiple string/locale settings - Update `timeago` plugin to use multiple string/locale settings
- Remove unused images (ClemMakesApps) - Remove unused images (ClemMakesApps)
......
...@@ -107,7 +107,8 @@ module Gitlab ...@@ -107,7 +107,8 @@ module Gitlab
end end
end end
redis_config_hash = Gitlab::Redis.redis_store_options # Use Redis caching across all environments
redis_config_hash = Gitlab::Redis.params
redis_config_hash[:namespace] = Gitlab::Redis::CACHE_NAMESPACE redis_config_hash[:namespace] = Gitlab::Redis::CACHE_NAMESPACE
redis_config_hash[:expires_in] = 2.weeks # Cache should not grow forever redis_config_hash[:expires_in] = 2.weeks # Cache should not grow forever
config.cache_store = :redis_store, redis_config_hash config.cache_store = :redis_store, redis_config_hash
......
...@@ -13,7 +13,7 @@ end ...@@ -13,7 +13,7 @@ end
if Rails.env.test? if Rails.env.test?
Gitlab::Application.config.session_store :cookie_store, key: "_gitlab_session" Gitlab::Application.config.session_store :cookie_store, key: "_gitlab_session"
else else
redis_config = Gitlab::Redis.redis_store_options redis_config = Gitlab::Redis.params
redis_config[:namespace] = Gitlab::Redis::SESSION_NAMESPACE redis_config[:namespace] = Gitlab::Redis::SESSION_NAMESPACE
Gitlab::Application.config.session_store( Gitlab::Application.config.session_store(
......
# Custom Redis configuration
redis_config_hash = Gitlab::Redis.params
redis_config_hash[:namespace] = Gitlab::Redis::SIDEKIQ_NAMESPACE
Sidekiq.configure_server do |config| Sidekiq.configure_server do |config|
config.redis = { config.redis = redis_config_hash
url: Gitlab::Redis.url,
namespace: Gitlab::Redis::SIDEKIQ_NAMESPACE
}
config.server_middleware do |chain| config.server_middleware do |chain|
chain.add Gitlab::SidekiqMiddleware::ArgumentsLogger if ENV['SIDEKIQ_LOG_ARGUMENTS'] chain.add Gitlab::SidekiqMiddleware::ArgumentsLogger if ENV['SIDEKIQ_LOG_ARGUMENTS']
...@@ -39,8 +40,5 @@ Sidekiq.configure_server do |config| ...@@ -39,8 +40,5 @@ Sidekiq.configure_server do |config|
end end
Sidekiq.configure_client do |config| Sidekiq.configure_client do |config|
config.redis = { config.redis = redis_config_hash
url: Gitlab::Redis.url,
namespace: Gitlab::Redis::SIDEKIQ_NAMESPACE
}
end end
# If you change this file in a Merge Request, please also create
# a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests
#
:mailboxes: :mailboxes:
<% <%
require "yaml" require_relative "lib/gitlab/mail_room" unless defined?(Gitlab::MailRoom)
require "json" config = Gitlab::MailRoom.config
require_relative "lib/gitlab/redis" unless defined?(Gitlab::Redis)
rails_env = ENV["RAILS_ENV"] || ENV["RACK_ENV"] || "development" if Gitlab::MailRoom.enabled?
config_file = ENV["MAIL_ROOM_GITLAB_CONFIG_FILE"] || "config/gitlab.yml"
if File.exists?(config_file)
all_config = YAML.load_file(config_file)[rails_env]
config = all_config["incoming_email"] || {}
config['enabled'] = false if config['enabled'].nil?
config['port'] = 143 if config['port'].nil?
config['ssl'] = false if config['ssl'].nil?
config['start_tls'] = false if config['start_tls'].nil?
config['mailbox'] = "inbox" if config['mailbox'].nil?
if config['enabled'] && config['address']
redis_url = Gitlab::Redis.new(rails_env).url
%> %>
- -
:host: <%= config['host'].to_json %> :host: <%= config[:host].to_json %>
:port: <%= config['port'].to_json %> :port: <%= config[:port].to_json %>
:ssl: <%= config['ssl'].to_json %> :ssl: <%= config[:ssl].to_json %>
:start_tls: <%= config['start_tls'].to_json %> :start_tls: <%= config[:start_tls].to_json %>
:email: <%= config['user'].to_json %> :email: <%= config[:user].to_json %>
:password: <%= config['password'].to_json %> :password: <%= config[:password].to_json %>
:idle_timeout: 60
:name: <%= config['mailbox'].to_json %> :name: <%= config[:mailbox].to_json %>
:delete_after_delivery: true :delete_after_delivery: true
:delivery_method: sidekiq :delivery_method: sidekiq
:delivery_options: :delivery_options:
:redis_url: <%= redis_url.to_json %> :redis_url: <%= config[:redis_url].to_json %>
:namespace: resque:gitlab :namespace: <%= Gitlab::Redis::SIDEKIQ_NAMESPACE %>
:queue: incoming_email :queue: incoming_email
:worker: EmailReceiverWorker :worker: EmailReceiverWorker
:arbitration_method: redis :arbitration_method: redis
:arbitration_options: :arbitration_options:
:redis_url: <%= redis_url.to_json %> :redis_url: <%= config[:redis_url].to_json %>
:namespace: mail_room:gitlab :namespace: <%= Gitlab::Redis::MAILROOM_NAMESPACE %>
<% end %> <% end %>
<% end %>
# If you change this file in a Merge Request, please also create # If you change this file in a Merge Request, please also create
# a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests # a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests
# #
development: redis://localhost:6379 development:
test: redis://localhost:6379 url: redis://localhost:6379
production: unix:/var/run/redis/redis.sock # sentinels:
# -
# host: localhost
# port: 26380 # point to sentinel, not to redis port
# -
# host: slave2
# port: 26381 # point to sentinel, not to redis port
test:
url: redis://localhost:6379
production:
# Redis (single instance)
url: unix:/var/run/redis/redis.sock
##
# Redis + Sentinel (for HA)
#
# Please read instructions carefully before using it as you may lose data:
# http://redis.io/topics/sentinel
#
# You must specify a list of a few sentinels that will handle client connection
# please read here for more information: https://docs.gitlab.com/ce/administration/high_availability/redis.html
##
# url: redis://master:6379
# sentinels:
# -
# host: slave1
# port: 26379 # point to sentinel, not to redis port
# -
# host: slave2
# port: 26379 # point to sentinel, not to redis port
...@@ -588,15 +588,17 @@ for the changes to take effect. ...@@ -588,15 +588,17 @@ for the changes to take effect.
### Custom Redis Connection ### Custom Redis Connection
If you'd like Resque to connect to a Redis server on a non-standard port or on a different host, you can configure its connection string via the `config/resque.yml` file. If you'd like to connect to a Redis server on a non-standard port or on a different host, you can configure its connection string via the `config/resque.yml` file.
# example # example
production: redis://redis.example.tld:6379 production:
url: redis://redis.example.tld:6379
If you want to connect the Redis server via socket, then use the "unix:" URL scheme and the path to the Redis socket file in the `config/resque.yml` file. If you want to connect the Redis server via socket, then use the "unix:" URL scheme and the path to the Redis socket file in the `config/resque.yml` file.
# example # example
production: unix:/path/to/redis/socket production:
url: unix:/path/to/redis/socket
### Custom SSH Connection ### Custom SSH Connection
......
require 'yaml'
require 'json'
require_relative 'redis' unless defined?(Gitlab::Redis)
module Gitlab
module MailRoom
class << self
def enabled?
config[:enabled] && config[:address]
end
def config
@config ||= fetch_config
end
def reset_config!
@config = nil
end
private
def fetch_config
return {} unless File.exist?(config_file)
rails_env = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || 'development'
all_config = YAML.load_file(config_file)[rails_env].deep_symbolize_keys
config = all_config[:incoming_email] || {}
config[:enabled] = false if config[:enabled].nil?
config[:port] = 143 if config[:port].nil?
config[:ssl] = false if config[:ssl].nil?
config[:start_tls] = false if config[:start_tls].nil?
config[:mailbox] = 'inbox' if config[:mailbox].nil?
if config[:enabled] && config[:address]
config[:redis_url] = Gitlab::Redis.new(rails_env).url
end
config
end
def config_file
ENV['MAIL_ROOM_GITLAB_CONFIG_FILE'] || File.expand_path('../../../config/gitlab.yml', __FILE__)
end
end
end
end
# This file should not have any direct dependency on Rails environment
# please require all dependencies below:
require 'active_support/core_ext/hash/keys'
module Gitlab module Gitlab
class Redis class Redis
CACHE_NAMESPACE = 'cache:gitlab' CACHE_NAMESPACE = 'cache:gitlab'
SESSION_NAMESPACE = 'session:gitlab' SESSION_NAMESPACE = 'session:gitlab'
SIDEKIQ_NAMESPACE = 'resque:gitlab' SIDEKIQ_NAMESPACE = 'resque:gitlab'
MAILROOM_NAMESPACE = 'mail_room:gitlab'
attr_reader :url DEFAULT_REDIS_URL = 'redis://localhost:6379'
# To be thread-safe we must be careful when writing the class instance # To be thread-safe we must be careful when writing the class instance
# variables @url and @pool. Because @pool depends on @url we need two # variables @url and @pool. Because @pool depends on @url we need two
# mutexes to prevent deadlock. # mutexes to prevent deadlock.
URL_MUTEX = Mutex.new PARAMS_MUTEX = Mutex.new
POOL_MUTEX = Mutex.new POOL_MUTEX = Mutex.new
private_constant :URL_MUTEX, :POOL_MUTEX private_constant :PARAMS_MUTEX, :POOL_MUTEX
class << self
def params
@params || PARAMS_MUTEX.synchronize { @params = new.params }
end
def self.url # @deprecated Use .params instead to get sentinel support
@url || URL_MUTEX.synchronize { @url = new.url } def url
new.url
end end
def self.with def with
if @pool.nil? if @pool.nil?
POOL_MUTEX.synchronize do POOL_MUTEX.synchronize do
@pool = ConnectionPool.new { ::Redis.new(url: url) } @pool = ConnectionPool.new { ::Redis.new(params) }
end end
end end
@pool.with { |redis| yield redis } @pool.with { |redis| yield redis }
end end
def self.redis_store_options def reset_params!
url = new.url @params = nil
redis_config_hash = ::Redis::Store::Factory.extract_host_options_from_uri(url)
# Redis::Store does not handle Unix sockets well, so let's do it for them
redis_uri = URI.parse(url)
if redis_uri.scheme == 'unix'
redis_config_hash[:path] = redis_uri.path
end end
redis_config_hash
end end
def initialize(rails_env = nil) def initialize(rails_env = nil)
rails_env ||= Rails.env @rails_env = rails_env || ::Rails.env
config_file = File.expand_path('../../../config/resque.yml', __FILE__) end
def params
redis_store_options
end
@url = "redis://localhost:6379" def url
if File.exist?(config_file) raw_config_hash[:url]
@url = YAML.load_file(config_file)[rails_env]
end end
private
def redis_store_options
config = raw_config_hash
redis_url = config.delete(:url)
redis_uri = URI.parse(redis_url)
if redis_uri.scheme == 'unix'
# Redis::Store does not handle Unix sockets well, so let's do it for them
config[:path] = redis_uri.path
config
else
redis_hash = ::Redis::Store::Factory.extract_host_options_from_uri(redis_url)
# order is important here, sentinels must be after the connection keys.
# {url: ..., port: ..., sentinels: [...]}
redis_hash.merge(config)
end
end
def raw_config_hash
config_data = fetch_config
if config_data
config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys
else
{ url: DEFAULT_REDIS_URL }
end
end
def fetch_config
file = config_file
File.exist?(file) ? YAML.load_file(file)[@rails_env] : false
end
def config_file
File.expand_path('../../../config/resque.yml', __FILE__)
end end
end end
end end
require "spec_helper" require 'spec_helper'
describe "mail_room.yml" do describe 'mail_room.yml' do
let(:config_path) { "config/mail_room.yml" } let(:config_path) { 'config/mail_room.yml' }
let(:configuration) { YAML.load(ERB.new(File.read(config_path)).result) } let(:configuration) { YAML.load(ERB.new(File.read(config_path)).result) }
context "when incoming email is disabled" do context 'when incoming email is disabled' do
before do before do
ENV["MAIL_ROOM_GITLAB_CONFIG_FILE"] = Rails.root.join("spec/fixtures/mail_room_disabled.yml").to_s ENV['MAIL_ROOM_GITLAB_CONFIG_FILE'] = Rails.root.join('spec/fixtures/mail_room_disabled.yml').to_s
Gitlab::MailRoom.reset_config!
end end
after do after do
ENV["MAIL_ROOM_GITLAB_CONFIG_FILE"] = nil ENV['MAIL_ROOM_GITLAB_CONFIG_FILE'] = nil
end end
it "contains no configuration" do it 'contains no configuration' do
expect(configuration[:mailboxes]).to be_nil expect(configuration[:mailboxes]).to be_nil
end end
end end
context "when incoming email is enabled" do context 'when incoming email is enabled' do
before do before do
ENV["MAIL_ROOM_GITLAB_CONFIG_FILE"] = Rails.root.join("spec/fixtures/mail_room_enabled.yml").to_s ENV['MAIL_ROOM_GITLAB_CONFIG_FILE'] = Rails.root.join('spec/fixtures/mail_room_enabled.yml').to_s
Gitlab::MailRoom.reset_config!
end end
after do after do
ENV["MAIL_ROOM_GITLAB_CONFIG_FILE"] = nil ENV['MAIL_ROOM_GITLAB_CONFIG_FILE'] = nil
end end
it "contains the intended configuration" do it 'contains the intended configuration' do
expect(configuration[:mailboxes].length).to eq(1) expect(configuration[:mailboxes].length).to eq(1)
mailbox = configuration[:mailboxes].first mailbox = configuration[:mailboxes].first
expect(mailbox[:host]).to eq("imap.gmail.com") expect(mailbox[:host]).to eq('imap.gmail.com')
expect(mailbox[:port]).to eq(993) expect(mailbox[:port]).to eq(993)
expect(mailbox[:ssl]).to eq(true) expect(mailbox[:ssl]).to eq(true)
expect(mailbox[:start_tls]).to eq(false) expect(mailbox[:start_tls]).to eq(false)
expect(mailbox[:email]).to eq("gitlab-incoming@gmail.com") expect(mailbox[:email]).to eq('gitlab-incoming@gmail.com')
expect(mailbox[:password]).to eq("[REDACTED]") expect(mailbox[:password]).to eq('[REDACTED]')
expect(mailbox[:name]).to eq("inbox") expect(mailbox[:name]).to eq('inbox')
redis_config_file = Rails.root.join('config', 'resque.yml') redis_url = Gitlab::Redis.url
redis_url =
if File.exist?(redis_config_file)
YAML.load_file(redis_config_file)[Rails.env]
else
"redis://localhost:6379"
end
expect(mailbox[:delivery_options][:redis_url]).to eq(redis_url) expect(mailbox[:delivery_options][:redis_url]).to eq(redis_url)
expect(mailbox[:arbitration_options][:redis_url]).to eq(redis_url) expect(mailbox[:arbitration_options][:redis_url]).to eq(redis_url)
......
# redis://[:password@]host[:port][/db-number][?option=value]
# more details: http://www.iana.org/assignments/uri-schemes/prov/redis
development:
url: redis://:mynewpassword@localhost:6379/99
sentinels:
-
host: localhost
port: 26380 # point to sentinel, not to redis port
-
host: slave2
port: 26381 # point to sentinel, not to redis port
test:
url: redis://:mynewpassword@localhost:6379/99
sentinels:
-
host: localhost
port: 26380 # point to sentinel, not to redis port
-
host: slave2
port: 26381 # point to sentinel, not to redis port
production:
url: redis://:mynewpassword@localhost:6379/99
sentinels:
-
host: slave1
port: 26380 # point to sentinel, not to redis port
-
host: slave2
port: 26381 # point to sentinel, not to redis port
development:
url: unix:/path/to/redis.sock
test:
url: unix:/path/to/redis.sock
production:
url: unix:/path/to/redis.sock
# redis://[:password@]host[:port][/db-number][?option=value]
# more details: http://www.iana.org/assignments/uri-schemes/prov/redis
development: redis://:mypassword@localhost:6379/99
test: redis://:mypassword@localhost:6379/99
production: redis://:mypassword@localhost:6379/99
development: unix:/path/to/old/redis.sock
test: unix:/path/to/old/redis.sock
production: unix:/path/to/old/redis.sock
require 'spec_helper'
describe Gitlab::Redis do
let(:redis_config) { Rails.root.join('config', 'resque.yml').to_s }
before(:each) { described_class.reset_params! }
after(:each) { described_class.reset_params! }
describe '.params' do
subject { described_class.params }
context 'when url contains unix socket reference' do
let(:config_old) { Rails.root.join('spec/fixtures/config/redis_old_format_socket.yml').to_s }
let(:config_new) { Rails.root.join('spec/fixtures/config/redis_new_format_socket.yml').to_s }
context 'with old format' do
it 'returns path key instead' do
expect_any_instance_of(described_class).to receive(:config_file) { config_old }
is_expected.to include(path: '/path/to/old/redis.sock')
is_expected.not_to have_key(:url)
end
end
context 'with new format' do
it 'returns path key instead' do
expect_any_instance_of(described_class).to receive(:config_file) { config_new }
is_expected.to include(path: '/path/to/redis.sock')
is_expected.not_to have_key(:url)
end
end
end
context 'when url is host based' do
let(:config_old) { Rails.root.join('spec/fixtures/config/redis_old_format_host.yml') }
let(:config_new) { Rails.root.join('spec/fixtures/config/redis_new_format_host.yml') }
context 'with old format' do
it 'returns hash with host, port, db, and password' do
expect_any_instance_of(described_class).to receive(:config_file) { config_old }
is_expected.to include(host: 'localhost', password: 'mypassword', port: 6379, db: 99)
is_expected.not_to have_key(:url)
end
end
context 'with new format' do
it 'returns hash with host, port, db, and password' do
expect_any_instance_of(described_class).to receive(:config_file) { config_new }
is_expected.to include(host: 'localhost', password: 'mynewpassword', port: 6379, db: 99)
is_expected.not_to have_key(:url)
end
end
end
end
describe '#raw_config_hash' do
it 'returns default redis url when no config file is present' do
expect(subject).to receive(:fetch_config) { false }
expect(subject.send(:raw_config_hash)).to eq(url: Gitlab::Redis::DEFAULT_REDIS_URL)
end
it 'returns old-style single url config in a hash' do
expect(subject).to receive(:fetch_config) { 'redis://myredis:6379' }
expect(subject.send(:raw_config_hash)).to eq(url: 'redis://myredis:6379')
end
end
describe '#fetch_config' do
it 'returns false when no config file is present' do
allow(File).to receive(:exist?).with(redis_config) { false }
expect(subject.send(:fetch_config)).to be_falsey
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment