Commit 0b434daf authored by Robert Speicher's avatar Robert Speicher

Merge branch '30392_support_for_multiple_key_value_stores' into 'master'

Support multiple Redis instances based on queue type

See merge request !11573
parents 4daa6da5 cb3b4a15
...@@ -3,3 +3,4 @@ lib/gitlab/sanitizers/svg/whitelist.rb ...@@ -3,3 +3,4 @@ lib/gitlab/sanitizers/svg/whitelist.rb
lib/gitlab/diff/position_tracer.rb lib/gitlab/diff/position_tracer.rb
app/policies/project_policy.rb app/policies/project_policy.rb
app/models/concerns/relative_positioning.rb app/models/concerns/relative_positioning.rb
lib/gitlab/redis/*.rb
...@@ -31,6 +31,9 @@ eslint-report.html ...@@ -31,6 +31,9 @@ eslint-report.html
/config/initializers/smtp_settings.rb /config/initializers/smtp_settings.rb
/config/initializers/relative_url.rb /config/initializers/relative_url.rb
/config/resque.yml /config/resque.yml
/config/redis.cache.yml
/config/redis.queues.yml
/config/redis.shared_state.yml
/config/unicorn.rb /config/unicorn.rb
/config/secrets.yml /config/secrets.yml
/config/sidekiq.yml /config/sidekiq.yml
......
...@@ -4,7 +4,10 @@ class HealthController < ActionController::Base ...@@ -4,7 +4,10 @@ class HealthController < ActionController::Base
CHECKS = [ CHECKS = [
Gitlab::HealthChecks::DbCheck, Gitlab::HealthChecks::DbCheck,
Gitlab::HealthChecks::RedisCheck, Gitlab::HealthChecks::Redis::RedisCheck,
Gitlab::HealthChecks::Redis::CacheCheck,
Gitlab::HealthChecks::Redis::QueuesCheck,
Gitlab::HealthChecks::Redis::SharedStateCheck,
Gitlab::HealthChecks::FsShardsCheck Gitlab::HealthChecks::FsShardsCheck
].freeze ].freeze
......
...@@ -149,7 +149,7 @@ module Ci ...@@ -149,7 +149,7 @@ module Ci
private private
def cleanup_runner_queue def cleanup_runner_queue
Gitlab::Redis.with do |redis| Gitlab::Redis::Queues.with do |redis|
redis.del(runner_queue_key) redis.del(runner_queue_key)
end end
end end
......
...@@ -1386,15 +1386,15 @@ class Project < ActiveRecord::Base ...@@ -1386,15 +1386,15 @@ class Project < ActiveRecord::Base
end end
def pushes_since_gc def pushes_since_gc
Gitlab::Redis.with { |redis| redis.get(pushes_since_gc_redis_key).to_i } Gitlab::Redis::SharedState.with { |redis| redis.get(pushes_since_gc_redis_shared_state_key).to_i }
end end
def increment_pushes_since_gc def increment_pushes_since_gc
Gitlab::Redis.with { |redis| redis.incr(pushes_since_gc_redis_key) } Gitlab::Redis::SharedState.with { |redis| redis.incr(pushes_since_gc_redis_shared_state_key) }
end end
def reset_pushes_since_gc def reset_pushes_since_gc
Gitlab::Redis.with { |redis| redis.del(pushes_since_gc_redis_key) } Gitlab::Redis::SharedState.with { |redis| redis.del(pushes_since_gc_redis_shared_state_key) }
end end
def route_map_for(commit_sha) def route_map_for(commit_sha)
...@@ -1457,7 +1457,7 @@ class Project < ActiveRecord::Base ...@@ -1457,7 +1457,7 @@ class Project < ActiveRecord::Base
from && self != from from && self != from
end end
def pushes_since_gc_redis_key def pushes_since_gc_redis_shared_state_key
"projects/#{id}/pushes_since_gc" "projects/#{id}/pushes_since_gc"
end end
......
...@@ -3,7 +3,10 @@ require 'prometheus/client/formats/text' ...@@ -3,7 +3,10 @@ require 'prometheus/client/formats/text'
class MetricsService class MetricsService
CHECKS = [ CHECKS = [
Gitlab::HealthChecks::DbCheck, Gitlab::HealthChecks::DbCheck,
Gitlab::HealthChecks::RedisCheck, Gitlab::HealthChecks::Redis::RedisCheck,
Gitlab::HealthChecks::Redis::CacheCheck,
Gitlab::HealthChecks::Redis::QueuesCheck,
Gitlab::HealthChecks::Redis::SharedStateCheck,
Gitlab::HealthChecks::FsShardsCheck Gitlab::HealthChecks::FsShardsCheck
].freeze ].freeze
......
module Milestones module Milestones
class DestroyService < Milestones::BaseService class DestroyService < Milestones::BaseService
def execute(milestone) def execute(milestone)
return unless milestone.is_project_milestone?
Milestone.transaction do Milestone.transaction do
update_params = { milestone: nil } update_params = { milestone: nil }
milestone.issues.each do |issue| milestone.issues.each do |issue|
Issues::UpdateService.new(parent, current_user, update_params).execute(issue) Issues::UpdateService.new(project, current_user, update_params).execute(issue)
end end
milestone.merge_requests.each do |merge_request| milestone.merge_requests.each do |merge_request|
MergeRequests::UpdateService.new(parent, current_user, update_params).execute(merge_request) MergeRequests::UpdateService.new(project, current_user, update_params).execute(merge_request)
end end
event_service.destroy_milestone(milestone, current_user) event_service.destroy_milestone(milestone, current_user)
......
...@@ -19,4 +19,132 @@ an ERB file and then loads the resulting YML as its configuration. ...@@ -19,4 +19,132 @@ an ERB file and then loads the resulting YML as its configuration.
This file is called `resque.yml` for historical reasons. We are **NOT** This file is called `resque.yml` for historical reasons. We are **NOT**
using Resque at the moment. It is used to specify Redis configuration using Resque at the moment. It is used to specify Redis configuration
values instead. values when a single database instance of Redis is desired.
# Advanced Redis configuration files
In more advanced configurations of Redis key-value storage, it is desirable
to separate the keys by lifecycle and intended use to ease provisioning and
management of scalable Redis clusters.
These settings provide routing and other configuration data (such as sentinel,
persistence policies, and other Redis customization) for connections
to Redis single instances, Redis sentinel, and Redis clusters.
If desired, the routing URL provided by these settings can be used with:
1. Unix Socket
1. named socket for each Redis instance desired.
2. `database number` for each Redis instance desired.
2. TCP Socket
1. `host name` or IP for each Redis instance desired
2. TCP port number for each Redis instance desired
3. `database number` for each Redis instance desired
## Example URL attribute formats for GitLab Redis `.yml` configuration files
* Unix Socket, default Redis database (0)
* `url: unix:/path/to/redis.sock`
* `url: unix:/path/to/redis.sock?db=`
* Unix Socket, Redis database 44
* `url: unix:/path/to/redis.sock?db=44`
* `url: unix:/path/to/redis.sock?extra=foo&db=44`
* TCP Socket for Redis on localhost, port 6379, database 33
* `url: redis://:mynewpassword@localhost:6379/33`
* TCP Socket for Redis on remote host `myserver`, port 6379, database 33
* `url: redis://:mynewpassword@myserver:6379/33`
## redis.cache.yml
If configured, `redis.cache.yml` overrides the
`resque.yml` settings to configure the Redis database instance
used for `Rails.cache` and other volatile non-persistent data which enhances
the performance of GitLab.
Settings here can be overridden by the environment variable
`GITLAB_REDIS_CACHE_CONFIG_FILE` which provides
an alternate location for configuration settings.
The order of precedence for the URL used to connect to the Redis instance
used for `cache` is:
1. URL from a configuration file pointed to by the
`GITLAB_REDIS_CACHE_CONFIG_FILE` environment variable
2. URL from `redis.cache.yml`
3. URL from a configuration file pointed to by the
`GITLAB_REDIS_CONFIG_FILE` environment variable
4. URL from `resque.yml`
5. `redis://localhost:6380`
The order of precedence for all other configuration settings for `cache`
are selected from only the first of the following files found (if a setting
is not provided in an earlier file, the remainder of the files are not
searched):
1. the configuration file pointed to by the
`GITLAB_REDIS_CACHE_CONFIG_FILE` environment variable
2. the configuration file `redis.cache.yml`
3. the configuration file pointed to by the
`GITLAB_REDIS_CONFIG_FILE` environment variable
4. the configuration file `resque.yml`
## redis.queues.yml
If configured, `redis.queues.yml` overrides the
`resque.yml` settings to configure the Redis database instance
used for clients of `::Gitlab::Redis::Queues`.
These queues are intended to be the foundation
of reliable inter-process communication between modules, whether on the same
host node, or within a cluster. The primary clients of the queues are
SideKiq, Mailroom, CI Runner, Workhorse, and push services. Settings here can
be overridden by the environment variable
`GITLAB_REDIS_QUEUES_CONFIG_FILE` which provides an alternate location for
configuration settings.
The order of precedence for the URL used to connect to the Redis instance
used for `queues` is:
1. URL from a configuration file pointed to by the
`GITLAB_REDIS_QUEUES_CONFIG_FILE` environment variable
2. URL from `redis.queues.yml`
3. URL from a configuration file pointed to by the
`GITLAB_REDIS_CONFIG_FILE` environment variable
4. URL from `resque.yml`
5. `redis://localhost:6381`
The order of precedence for all other configuration settings for `queues`
are selected from only the first of the following files found (if a setting
is not provided in an earlier file, the remainder of the files are not
searched):
1. the configuration file pointed to by the
`GITLAB_REDIS_QUEUES_CONFIG_FILE` environment variable
2. the configuration file `redis.queues.yml`
3. the configuration file pointed to by the
`GITLAB_REDIS_CONFIG_FILE` environment variable
4. the configuration file `resque.yml`
## redis.shared_state.yml
If configured, `redis.shared_state.yml` overrides the
`resque.yml` settings to configure the Redis database instance
used for clients of `::Gitlab::Redis::SharedState` such as session state,
and rate limiting.
Settings here can be overridden by the environment variable
`GITLAB_REDIS_SHARED_STATE_CONFIG_FILE` which provides
an alternate location for configuration settings.
The order of precedence for the URL used to connect to the Redis instance
used for `shared_state` is:
1. URL from a configuration file pointed to by the
`GITLAB_REDIS_SHARED_STATE_CONFIG_FILE` environment variable
2. URL from `redis.shared_state.yml`
3. URL from a configuration file pointed to by the
`GITLAB_REDIS_CONFIG_FILE` environment variable
4. URL from `resque.yml`
5. `redis://localhost:6382`
The order of precedence for all other configuration settings for `shared_state`
are selected from only the first of the following files found (if a setting
is not provided in an earlier file, the remainder of the files are not
searched):
1. the configuration file pointed to by the
`GITLAB_REDIS_SHARED_STATE_CONFIG_FILE` environment variable
2. the configuration file `redis.shared_state.yml`
3. the configuration file pointed to by the
`GITLAB_REDIS_CONFIG_FILE` environment variable
4. the configuration file `resque.yml`
...@@ -6,7 +6,9 @@ Bundler.require(:default, Rails.env) ...@@ -6,7 +6,9 @@ Bundler.require(:default, Rails.env)
module Gitlab module Gitlab
class Application < Rails::Application class Application < Rails::Application
require_dependency Rails.root.join('lib/gitlab/redis') require_dependency Rails.root.join('lib/gitlab/redis/cache')
require_dependency Rails.root.join('lib/gitlab/redis/queues')
require_dependency Rails.root.join('lib/gitlab/redis/shared_state')
require_dependency Rails.root.join('lib/gitlab/request_context') require_dependency Rails.root.join('lib/gitlab/request_context')
# Settings in config/environments/* take precedence over those specified here. # Settings in config/environments/* take precedence over those specified here.
...@@ -142,15 +144,15 @@ module Gitlab ...@@ -142,15 +144,15 @@ module Gitlab
end end
end end
# Use Redis caching across all environments # Use caching across all environments
redis_config_hash = Gitlab::Redis.params caching_config_hash = Gitlab::Redis::Cache.params
redis_config_hash[:namespace] = Gitlab::Redis::CACHE_NAMESPACE caching_config_hash[:namespace] = Gitlab::Redis::Cache::CACHE_NAMESPACE
redis_config_hash[:expires_in] = 2.weeks # Cache should not grow forever caching_config_hash[:expires_in] = 2.weeks # Cache should not grow forever
if Sidekiq.server? # threaded context if Sidekiq.server? # threaded context
redis_config_hash[:pool_size] = Sidekiq.options[:concurrency] + 5 caching_config_hash[:pool_size] = Sidekiq.options[:concurrency] + 5
redis_config_hash[:pool_timeout] = 1 caching_config_hash[:pool_timeout] = 1
end end
config.cache_store = :redis_store, redis_config_hash config.cache_store = :redis_store, caching_config_hash
config.active_record.raise_in_transactional_callbacks = true config.active_record.raise_in_transactional_callbacks = true
......
# Make sure we initialize a Redis connection pool before Sidekiq starts # Make sure we initialize a Redis connection pool before multi-threaded
# multi-threaded execution. # execution starts by
Gitlab::Redis.with { nil } # 1. Sidekiq
# 2. Rails.cache
# 3. HTTP clients
Gitlab::Redis::Cache.with { nil }
Gitlab::Redis::Queues.with { nil }
Gitlab::Redis::SharedState.with { nil }
Rails.application.config.peek.adapter = :redis, { client: ::Redis.new(Gitlab::Redis.params) } Rails.application.config.peek.adapter = :redis, { client: ::Redis.new(Gitlab::Redis::Cache.params) }
Peek.into Peek::Views::Host Peek.into Peek::Views::Host
Peek.into Peek::Views::PerformanceBar Peek.into Peek::Views::PerformanceBar
......
...@@ -19,12 +19,12 @@ cookie_key = if Rails.env.development? ...@@ -19,12 +19,12 @@ cookie_key = if Rails.env.development?
if Rails.env.test? if Rails.env.test?
Gitlab::Application.config.session_store :cookie_store, key: "_gitlab_session" Gitlab::Application.config.session_store :cookie_store, key: "_gitlab_session"
else else
redis_config = Gitlab::Redis.params sessions_config = Gitlab::Redis::SharedState.params
redis_config[:namespace] = Gitlab::Redis::SESSION_NAMESPACE sessions_config[:namespace] = Gitlab::Redis::SharedState::SESSION_NAMESPACE
Gitlab::Application.config.session_store( Gitlab::Application.config.session_store(
:redis_store, # Using the cookie_store would enable session replay attacks. :redis_store, # Using the cookie_store would enable session replay attacks.
servers: redis_config, servers: sessions_config,
key: cookie_key, key: cookie_key,
secure: Gitlab.config.gitlab.https, secure: Gitlab.config.gitlab.https,
httponly: true, httponly: true,
......
# Custom Redis configuration # Custom Queues configuration
redis_config_hash = Gitlab::Redis.params queues_config_hash = Gitlab::Redis::Queues.params
redis_config_hash[:namespace] = Gitlab::Redis::SIDEKIQ_NAMESPACE queues_config_hash[:namespace] = Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE
# Default is to retry 25 times with exponential backoff. That's too much. # Default is to retry 25 times with exponential backoff. That's too much.
Sidekiq.default_worker_options = { retry: 3 } Sidekiq.default_worker_options = { retry: 3 }
Sidekiq.configure_server do |config| Sidekiq.configure_server do |config|
config.redis = redis_config_hash config.redis = queues_config_hash
config.server_middleware do |chain| config.server_middleware do |chain|
chain.add Gitlab::SidekiqMiddleware::ArgumentsLogger if ENV['SIDEKIQ_LOG_ARGUMENTS'] chain.add Gitlab::SidekiqMiddleware::ArgumentsLogger if ENV['SIDEKIQ_LOG_ARGUMENTS']
...@@ -54,7 +54,7 @@ Sidekiq.configure_server do |config| ...@@ -54,7 +54,7 @@ Sidekiq.configure_server do |config|
end end
Sidekiq.configure_client do |config| Sidekiq.configure_client do |config|
config.redis = redis_config_hash config.redis = queues_config_hash
config.client_middleware do |chain| config.client_middleware do |chain|
chain.add Gitlab::SidekiqStatus::ClientMiddleware chain.add Gitlab::SidekiqStatus::ClientMiddleware
......
...@@ -21,7 +21,7 @@ ...@@ -21,7 +21,7 @@
:delivery_method: sidekiq :delivery_method: sidekiq
:delivery_options: :delivery_options:
:redis_url: <%= config[:redis_url].to_json %> :redis_url: <%= config[:redis_url].to_json %>
:namespace: <%= Gitlab::Redis::SIDEKIQ_NAMESPACE %> :namespace: <%= Gitlab::Redis::Queues::SIDEKIQ_NAMESPACE %>
:queue: email_receiver :queue: email_receiver
:worker: EmailReceiverWorker :worker: EmailReceiverWorker
<% if config[:sentinels] %> <% if config[:sentinels] %>
...@@ -36,7 +36,7 @@ ...@@ -36,7 +36,7 @@
:arbitration_method: redis :arbitration_method: redis
:arbitration_options: :arbitration_options:
:redis_url: <%= config[:redis_url].to_json %> :redis_url: <%= config[:redis_url].to_json %>
:namespace: <%= Gitlab::Redis::MAILROOM_NAMESPACE %> :namespace: <%= Gitlab::Redis::Queues::MAILROOM_NAMESPACE %>
<% if config[:sentinels] %> <% if config[:sentinels] %>
:sentinels: :sentinels:
<% config[:sentinels].each do |sentinel| %> <% config[:sentinels].each do |sentinel| %>
......
# If you change this file in a Merge Request, please also create
# a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests
#
development:
url: redis://localhost:6379/10
#
# url: redis://localhost:6380
# sentinels:
# -
# host: localhost
# port: 26380 # point to sentinel, not to redis port
# -
# host: slave2
# port: 26380 # point to sentinel, not to redis port
test:
url: redis://localhost:6379/10
#
# url: redis://localhost:6380
production:
# Redis (single instance)
url: unix:/var/run/redis/redis.cache.sock
##
# Redis + Sentinel (for HA)
#
# Please read instructions carefully before using it as you may lose data:
# http://redis.io/topics/sentinel
#
# You must specify a list of a few sentinels that will handle client connection
# please read here for more information: https://docs.gitlab.com/ce/administration/high_availability/redis.html
##
# url: redis://master:6380
# sentinels:
# -
# host: slave1
# port: 26380 # point to sentinel, not to redis port
# -
# host: slave2
# port: 26380 # point to sentinel, not to redis port
# If you change this file in a Merge Request, please also create
# a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests
#
development:
url: redis://localhost:6379/11
#
# url: redis://localhost:6381
# sentinels:
# -
# host: localhost
# port: 26381 # point to sentinel, not to redis port
# -
# host: slave2
# port: 26381 # point to sentinel, not to redis port
test:
url: redis://localhost:6379/11
#
# url: redis://localhost:6381
production:
# Redis (single instance)
url: unix:/var/run/redis/redis.queues.sock
##
# Redis + Sentinel (for HA)
#
# Please read instructions carefully before using it as you may lose data:
# http://redis.io/topics/sentinel
#
# You must specify a list of a few sentinels that will handle client connection
# please read here for more information: https://docs.gitlab.com/ce/administration/high_availability/redis.html
##
# url: redis://master:6381
# sentinels:
# -
# host: slave1
# port: 26381 # point to sentinel, not to redis port
# -
# host: slave2
# port: 26381 # point to sentinel, not to redis port
# If you change this file in a Merge Request, please also create
# a Merge Request on https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests
#
development:
url: redis://localhost:6379/12
#
# url: redis://localhost:6382
# sentinels:
# -
# host: localhost
# port: 26382 # point to sentinel, not to redis port
# -
# host: slave2
# port: 26382 # point to sentinel, not to redis port
test:
url: redis://localhost:6379/12
#
# url: redis://localhost:6382
production:
# Redis (single instance)
url: unix:/var/run/redis/redis.shared_state.sock
##
# Redis + Sentinel (for HA)
#
# Please read instructions carefully before using it as you may lose data:
# http://redis.io/topics/sentinel
#
# You must specify a list of a few sentinels that will handle client connection
# please read here for more information: https://docs.gitlab.com/ce/administration/high_availability/redis.html
##
# url: redis://master:6382
# sentinels:
# -
# host: slave1
# port: 26382 # point to sentinel, not to redis port
# -
# host: slave2
# port: 26382 # point to sentinel, not to redis port
...@@ -56,7 +56,7 @@ class MigrateUserActivitiesToUsersLastActivityOn < ActiveRecord::Migration ...@@ -56,7 +56,7 @@ class MigrateUserActivitiesToUsersLastActivityOn < ActiveRecord::Migration
end end
def activities(from, to, page: 1) def activities(from, to, page: 1)
Gitlab::Redis.with do |redis| Gitlab::Redis::SharedState.with do |redis|
redis.zrangebyscore(USER_ACTIVITY_SET_KEY, from.to_i, to.to_i, redis.zrangebyscore(USER_ACTIVITY_SET_KEY, from.to_i, to.to_i,
with_scores: true, with_scores: true,
limit: limit(page)) limit: limit(page))
...@@ -64,7 +64,7 @@ class MigrateUserActivitiesToUsersLastActivityOn < ActiveRecord::Migration ...@@ -64,7 +64,7 @@ class MigrateUserActivitiesToUsersLastActivityOn < ActiveRecord::Migration
end end
def activities_count(from, to) def activities_count(from, to)
Gitlab::Redis.with do |redis| Gitlab::Redis::SharedState.with do |redis|
redis.zcount(USER_ACTIVITY_SET_KEY, from.to_i, to.to_i) redis.zcount(USER_ACTIVITY_SET_KEY, from.to_i, to.to_i)
end end
end end
......
...@@ -4,6 +4,11 @@ This is the documentation for configuring a Highly Available Redis setup when ...@@ -4,6 +4,11 @@ This is the documentation for configuring a Highly Available Redis setup when
you have installed Redis all by yourself and not using the bundled one that you have installed Redis all by yourself and not using the bundled one that
comes with the Omnibus packages. comes with the Omnibus packages.
Note also that you may elect to override all references to
`/home/git/gitlab/config/resque.yml` in accordance with the advanced Redis
settings outlined in
[Configuration Files Documentation](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/README.md).
We cannot stress enough the importance of reading the We cannot stress enough the importance of reading the
[Overview section](redis.md#overview) of the Omnibus Redis HA as it provides [Overview section](redis.md#overview) of the Omnibus Redis HA as it provides
some invaluable information to the configuration of Redis. Please proceed to some invaluable information to the configuration of Redis. Please proceed to
......
...@@ -15,6 +15,12 @@ prefixed with 'session:gitlab:', so they would look like ...@@ -15,6 +15,12 @@ prefixed with 'session:gitlab:', so they would look like
'session:gitlab:976aa289e2189b17d7ef525a6702ace9'. Below we describe how to 'session:gitlab:976aa289e2189b17d7ef525a6702ace9'. Below we describe how to
remove the keys in the old format. remove the keys in the old format.
**Note:** the instructions below must be modified in accordance with your
configuration settings if you have used the advanced Redis
settings outlined in
[Configuration Files Documentation](https://gitlab.com/gitlab-org/gitlab-ce/blob/master/config/README.md).
First we define a shell function with the proper Redis connection details. First we define a shell function with the proper Redis connection details.
``` ```
......
...@@ -420,12 +420,6 @@ GitLab Shell is an SSH access and repository management software developed speci ...@@ -420,12 +420,6 @@ GitLab Shell is an SSH access and repository management software developed speci
**Note:** Make sure your hostname can be resolved on the machine itself by either a proper DNS record or an additional line in /etc/hosts ("127.0.0.1 hostname"). This might be necessary for example if you set up GitLab behind a reverse proxy. If the hostname cannot be resolved, the final installation check will fail with "Check GitLab API access: FAILED. code: 401" and pushing commits will be rejected with "[remote rejected] master -> master (hook declined)". **Note:** Make sure your hostname can be resolved on the machine itself by either a proper DNS record or an additional line in /etc/hosts ("127.0.0.1 hostname"). This might be necessary for example if you set up GitLab behind a reverse proxy. If the hostname cannot be resolved, the final installation check will fail with "Check GitLab API access: FAILED. code: 401" and pushing commits will be rejected with "[remote rejected] master -> master (hook declined)".
**Note:** GitLab Shell application startup time can be greatly reduced by disabling RubyGems. This can be done in several manners:
* Export `RUBYOPT=--disable-gems` environment variable for the processes
* Compile Ruby with `configure --disable-rubygems` to disable RubyGems by default. Not recommened for system-wide Ruby.
* Omnibus GitLab [replaces the *shebang* line of the `gitlab-shell/bin/*` scripts](https://gitlab.com/gitlab-org/omnibus-gitlab/merge_requests/1707)
### Install gitlab-workhorse ### Install gitlab-workhorse
GitLab-Workhorse uses [GNU Make](https://www.gnu.org/software/make/). The GitLab-Workhorse uses [GNU Make](https://www.gnu.org/software/make/). The
......
...@@ -27,7 +27,7 @@ module Gitlab ...@@ -27,7 +27,7 @@ module Gitlab
time = Time.now.utc.to_i time = Time.now.utc.to_i
key = "#{USER_UNIQUE_IPS_PREFIX}:#{user_id}" key = "#{USER_UNIQUE_IPS_PREFIX}:#{user_id}"
Gitlab::Redis.with do |redis| Gitlab::Redis::SharedState.with do |redis|
unique_ips_count = nil unique_ips_count = nil
redis.multi do |r| redis.multi do |r|
r.zadd(key, time, ip) r.zadd(key, time, ip)
......
...@@ -23,7 +23,7 @@ module Gitlab ...@@ -23,7 +23,7 @@ module Gitlab
end end
def self.cached_results_for_projects(projects) def self.cached_results_for_projects(projects)
result = Gitlab::Redis.with do |redis| result = Gitlab::Redis::Cache.with do |redis|
redis.multi do redis.multi do
projects.each do |project| projects.each do |project|
cache_key = cache_key_for_project(project) cache_key = cache_key_for_project(project)
...@@ -100,19 +100,19 @@ module Gitlab ...@@ -100,19 +100,19 @@ module Gitlab
end end
def load_from_cache def load_from_cache
Gitlab::Redis.with do |redis| Gitlab::Redis::Cache.with do |redis|
self.sha, self.status, self.ref = redis.hmget(cache_key, :sha, :status, :ref) self.sha, self.status, self.ref = redis.hmget(cache_key, :sha, :status, :ref)
end end
end end
def store_in_cache def store_in_cache
Gitlab::Redis.with do |redis| Gitlab::Redis::Cache.with do |redis|
redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref })
end end
end end
def delete_from_cache def delete_from_cache
Gitlab::Redis.with do |redis| Gitlab::Redis::Cache.with do |redis|
redis.del(cache_key) redis.del(cache_key)
end end
end end
...@@ -120,7 +120,7 @@ module Gitlab ...@@ -120,7 +120,7 @@ module Gitlab
def has_cache? def has_cache?
return self.loaded unless self.loaded.nil? return self.loaded unless self.loaded.nil?
Gitlab::Redis.with do |redis| Gitlab::Redis::Cache.with do |redis|
redis.exists(cache_key) redis.exists(cache_key)
end end
end end
......
...@@ -12,23 +12,23 @@ module Gitlab ...@@ -12,23 +12,23 @@ module Gitlab
end end
def get def get
Gitlab::Redis.with do |redis| Gitlab::Redis::SharedState.with do |redis|
data = redis.get(redis_key) data = redis.get(redis_shared_state_key)
JSON.parse(data, symbolize_names: true) if data JSON.parse(data, symbolize_names: true) if data
end end
end end
def store!(params) def store!(params)
Gitlab::Redis.with do |redis| Gitlab::Redis::SharedState.with do |redis|
params = params.to_json params = params.to_json
redis.set(redis_key, params, ex: EXPIRY_TIME) redis.set(redis_shared_state_key, params, ex: EXPIRY_TIME)
token token
end end
end end
def delete def delete
Gitlab::Redis.with do |redis| Gitlab::Redis::SharedState.with do |redis|
redis.del(redis_key) redis.del(redis_shared_state_key)
end end
end end
...@@ -38,7 +38,7 @@ module Gitlab ...@@ -38,7 +38,7 @@ module Gitlab
Devise.friendly_token(TOKEN_LENGTH) Devise.friendly_token(TOKEN_LENGTH)
end end
def redis_key def redis_shared_state_key
"gitlab:chat_names:#{token}" "gitlab:chat_names:#{token}"
end end
end end
......
...@@ -25,7 +25,7 @@ module Gitlab ...@@ -25,7 +25,7 @@ module Gitlab
def cached_application_settings def cached_application_settings
begin begin
::ApplicationSetting.cached ::ApplicationSetting.cached
rescue ::Redis::BaseError, ::Errno::ENOENT, ::Errno::EADDRNOTAVAIL rescue ::Redis::BaseError, ::Errno::ENOENT
# In case Redis isn't running or the Redis UNIX socket file is not available # In case Redis isn't running or the Redis UNIX socket file is not available
end end
end end
......
...@@ -145,7 +145,7 @@ module Gitlab ...@@ -145,7 +145,7 @@ module Gitlab
def track_rename(type, old_path, new_path) def track_rename(type, old_path, new_path)
key = redis_key_for_type(type) key = redis_key_for_type(type)
Gitlab::Redis.with do |redis| Gitlab::Redis::SharedState.with do |redis|
redis.lpush(key, [old_path, new_path].to_json) redis.lpush(key, [old_path, new_path].to_json)
redis.expire(key, 2.weeks.to_i) redis.expire(key, 2.weeks.to_i)
end end
...@@ -155,7 +155,7 @@ module Gitlab ...@@ -155,7 +155,7 @@ module Gitlab
def reverts_for_type(type) def reverts_for_type(type)
key = redis_key_for_type(type) key = redis_key_for_type(type)
Gitlab::Redis.with do |redis| Gitlab::Redis::SharedState.with do |redis|
failed_reverts = [] failed_reverts = []
while rename_info = redis.lpop(key) while rename_info = redis.lpop(key)
......
...@@ -2,17 +2,17 @@ module Gitlab ...@@ -2,17 +2,17 @@ module Gitlab
module EtagCaching module EtagCaching
class Store class Store
EXPIRY_TIME = 20.minutes EXPIRY_TIME = 20.minutes
REDIS_NAMESPACE = 'etag:'.freeze SHARED_STATE_NAMESPACE = 'etag:'.freeze
def get(key) def get(key)
Gitlab::Redis.with { |redis| redis.get(redis_key(key)) } Gitlab::Redis::SharedState.with { |redis| redis.get(redis_shared_state_key(key)) }
end end
def touch(key, only_if_missing: false) def touch(key, only_if_missing: false)
etag = generate_etag etag = generate_etag
Gitlab::Redis.with do |redis| Gitlab::Redis::SharedState.with do |redis|
redis.set(redis_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing) redis.set(redis_shared_state_key(key), etag, ex: EXPIRY_TIME, nx: only_if_missing)
end end
etag etag
...@@ -24,10 +24,10 @@ module Gitlab ...@@ -24,10 +24,10 @@ module Gitlab
SecureRandom.hex SecureRandom.hex
end end
def redis_key(key) def redis_shared_state_key(key)
raise 'Invalid key' if !Rails.env.production? && !Gitlab::EtagCaching::Router.match(key) raise 'Invalid key' if !Rails.env.production? && !Gitlab::EtagCaching::Router.match(key)
"#{REDIS_NAMESPACE}#{key}" "#{SHARED_STATE_NAMESPACE}#{key}"
end end
end end
end end
......
...@@ -26,17 +26,17 @@ module Gitlab ...@@ -26,17 +26,17 @@ module Gitlab
EOS EOS
def self.cancel(key, uuid) def self.cancel(key, uuid)
Gitlab::Redis.with do |redis| Gitlab::Redis::SharedState.with do |redis|
redis.eval(LUA_CANCEL_SCRIPT, keys: [redis_key(key)], argv: [uuid]) redis.eval(LUA_CANCEL_SCRIPT, keys: [redis_shared_state_key(key)], argv: [uuid])
end end
end end
def self.redis_key(key) def self.redis_shared_state_key(key)
"gitlab:exclusive_lease:#{key}" "gitlab:exclusive_lease:#{key}"
end end
def initialize(key, timeout:) def initialize(key, timeout:)
@redis_key = self.class.redis_key(key) @redis_shared_state_key = self.class.redis_shared_state_key(key)
@timeout = timeout @timeout = timeout
@uuid = SecureRandom.uuid @uuid = SecureRandom.uuid
end end
...@@ -45,24 +45,24 @@ module Gitlab ...@@ -45,24 +45,24 @@ module Gitlab
# false if the lease is already taken. # false if the lease is already taken.
def try_obtain def try_obtain
# Performing a single SET is atomic # Performing a single SET is atomic
Gitlab::Redis.with do |redis| Gitlab::Redis::SharedState.with do |redis|
redis.set(@redis_key, @uuid, nx: true, ex: @timeout) && @uuid redis.set(@redis_shared_state_key, @uuid, nx: true, ex: @timeout) && @uuid
end end
end end
# Try to renew an existing lease. Return lease UUID on success, # Try to renew an existing lease. Return lease UUID on success,
# false if the lease is taken by a different UUID or inexistent. # false if the lease is taken by a different UUID or inexistent.
def renew def renew
Gitlab::Redis.with do |redis| Gitlab::Redis::SharedState.with do |redis|
result = redis.eval(LUA_RENEW_SCRIPT, keys: [@redis_key], argv: [@uuid, @timeout]) result = redis.eval(LUA_RENEW_SCRIPT, keys: [@redis_shared_state_key], argv: [@uuid, @timeout])
result == @uuid result == @uuid
end end
end end
# Returns true if the key for this lease is set. # Returns true if the key for this lease is set.
def exists? def exists?
Gitlab::Redis.with do |redis| Gitlab::Redis::SharedState.with do |redis|
redis.exists(@redis_key) redis.exists(@redis_shared_state_key)
end end
end end
end end
......
module Gitlab
module HealthChecks
module Redis
class CacheCheck
extend SimpleAbstractCheck
class << self
def check_up
check
end
private
def metric_prefix
'redis_cache_ping'
end
def is_successful?(result)
result == 'PONG'
end
def check
catch_timeout 10.seconds do
Gitlab::Redis::Cache.with(&:ping)
end
end
end
end
end
end
end
module Gitlab
module HealthChecks
module Redis
class QueuesCheck
extend SimpleAbstractCheck
class << self
def check_up
check
end
private
def metric_prefix
'redis_queues_ping'
end
def is_successful?(result)
result == 'PONG'
end
def check
catch_timeout 10.seconds do
Gitlab::Redis::Queues.with(&:ping)
end
end
end
end
end
end
end
module Gitlab
module HealthChecks
module Redis
class RedisCheck
extend SimpleAbstractCheck
class << self
private
def metric_prefix
'redis_ping'
end
def is_successful?(result)
result == 'PONG'
end
def check
::Gitlab::HealthChecks::Redis::CacheCheck.check_up &&
::Gitlab::HealthChecks::Redis::QueuesCheck.check_up &&
::Gitlab::HealthChecks::Redis::SharedStateCheck.check_up
end
end
end
end
end
end
module Gitlab
module HealthChecks
module Redis
class SharedStateCheck
extend SimpleAbstractCheck
class << self
def check_up
check
end
private
def metric_prefix
'redis_shared_state_ping'
end
def is_successful?(result)
result == 'PONG'
end
def check
catch_timeout 10.seconds do
Gitlab::Redis::SharedState.with(&:ping)
end
end
end
end
end
end
end
module Gitlab
module HealthChecks
class RedisCheck
extend SimpleAbstractCheck
class << self
private
def metric_prefix
'redis_ping'
end
def is_successful?(result)
result == 'PONG'
end
def check
catch_timeout 10.seconds do
Gitlab::Redis.with(&:ping)
end
end
end
end
end
end
...@@ -18,10 +18,10 @@ module Gitlab ...@@ -18,10 +18,10 @@ module Gitlab
end end
def token def token
Gitlab::Redis.with do |redis| Gitlab::Redis::SharedState.with do |redis|
token = redis.get(redis_key) token = redis.get(redis_shared_state_key)
token ||= Devise.friendly_token(TOKEN_LENGTH) token ||= Devise.friendly_token(TOKEN_LENGTH)
redis.set(redis_key, token, ex: EXPIRY_TIME) redis.set(redis_shared_state_key, token, ex: EXPIRY_TIME)
token token
end end
...@@ -41,7 +41,7 @@ module Gitlab ...@@ -41,7 +41,7 @@ module Gitlab
private private
def redis_key def redis_shared_state_key
"gitlab:lfs_token:#{actor.class.name.underscore}_#{actor.id}" if actor "gitlab:lfs_token:#{actor.class.name.underscore}_#{actor.id}" if actor
end end
end end
......
require 'yaml' require 'yaml'
require 'json' require 'json'
require_relative 'redis' unless defined?(Gitlab::Redis) require_relative 'redis/queues' unless defined?(Gitlab::Redis::Queues)
module Gitlab module Gitlab
module MailRoom module MailRoom
...@@ -34,11 +34,11 @@ module Gitlab ...@@ -34,11 +34,11 @@ module Gitlab
config[:idle_timeout] = 60 if config[:idle_timeout].nil? config[:idle_timeout] = 60 if config[:idle_timeout].nil?
if config[:enabled] && config[:address] if config[:enabled] && config[:address]
gitlab_redis = Gitlab::Redis.new(rails_env) gitlab_redis_queues = Gitlab::Redis::Queues.new(rails_env)
config[:redis_url] = gitlab_redis.url config[:redis_url] = gitlab_redis_queues.url
if gitlab_redis.sentinels? if gitlab_redis_queues.sentinels?
config[:sentinels] = gitlab_redis.sentinels config[:sentinels] = gitlab_redis_queues.sentinels
end end
end end
......
# This file should not have any direct dependency on Rails environment
# please require all dependencies below:
require 'active_support/core_ext/hash/keys'
require 'active_support/core_ext/module/delegation'
module Gitlab
class Redis
CACHE_NAMESPACE = 'cache:gitlab'.freeze
SESSION_NAMESPACE = 'session:gitlab'.freeze
SIDEKIQ_NAMESPACE = 'resque:gitlab'.freeze
MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze
DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze
class << self
delegate :params, :url, to: :new
def with
@pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) }
@pool.with { |redis| yield redis }
end
def pool_size
if Sidekiq.server?
# the pool will be used in a multi-threaded context
Sidekiq.options[:concurrency] + 5
else
# probably this is a Unicorn process, so single threaded
5
end
end
def _raw_config
return @_raw_config if defined?(@_raw_config)
begin
@_raw_config = ERB.new(File.read(config_file)).result.freeze
rescue Errno::ENOENT
@_raw_config = false
end
@_raw_config
end
def config_file
ENV['GITLAB_REDIS_CONFIG_FILE'] || File.expand_path('../../config/resque.yml', __dir__)
end
end
def initialize(rails_env = nil)
@rails_env = rails_env || ::Rails.env
end
def params
redis_store_options
end
def url
raw_config_hash[:url]
end
def sentinels
raw_config_hash[:sentinels]
end
def sentinels?
sentinels && !sentinels.empty?
end
private
def redis_store_options
config = raw_config_hash
redis_url = config.delete(:url)
redis_uri = URI.parse(redis_url)
if redis_uri.scheme == 'unix'
# Redis::Store does not handle Unix sockets well, so let's do it for them
config[:path] = redis_uri.path
config
else
redis_hash = ::Redis::Store::Factory.extract_host_options_from_uri(redis_url)
# order is important here, sentinels must be after the connection keys.
# {url: ..., port: ..., sentinels: [...]}
redis_hash.merge(config)
end
end
def raw_config_hash
config_data = fetch_config
if config_data
config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys
else
{ url: DEFAULT_REDIS_URL }
end
end
def fetch_config
self.class._raw_config ? YAML.load(self.class._raw_config)[@rails_env] : false
end
end
end
# please require all dependencies below:
require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper)
module Gitlab
module Redis
class Cache < ::Gitlab::Redis::Wrapper
CACHE_NAMESPACE = 'cache:gitlab'.freeze
DEFAULT_REDIS_CACHE_URL = 'redis://localhost:6380'.freeze
REDIS_CACHE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CACHE_CONFIG_FILE'.freeze
if defined?(::Rails) && ::Rails.root.present?
DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.cache.yml').freeze
end
class << self
def default_url
DEFAULT_REDIS_CACHE_URL
end
def config_file_name
# if ENV set for this class, use it even if it points to a file does not exist
file_name = ENV[REDIS_CACHE_CONFIG_ENV_VAR_NAME]
return file_name unless file_name.nil?
# otherwise, if config files exists for this class, use it
file_name = File.expand_path(DEFAULT_REDIS_CACHE_CONFIG_FILE_NAME, __dir__)
return file_name if File.file?(file_name)
# this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent
super
end
end
end
end
end
# please require all dependencies below:
require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper)
module Gitlab
module Redis
class Queues < ::Gitlab::Redis::Wrapper
SIDEKIQ_NAMESPACE = 'resque:gitlab'.freeze
MAILROOM_NAMESPACE = 'mail_room:gitlab'.freeze
DEFAULT_REDIS_QUEUES_URL = 'redis://localhost:6381'.freeze
REDIS_QUEUES_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_QUEUES_CONFIG_FILE'.freeze
if defined?(::Rails) && ::Rails.root.present?
DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.queues.yml').freeze
end
class << self
def default_url
DEFAULT_REDIS_QUEUES_URL
end
def config_file_name
# if ENV set for this class, use it even if it points to a file does not exist
file_name = ENV[REDIS_QUEUES_CONFIG_ENV_VAR_NAME]
return file_name if file_name
# otherwise, if config files exists for this class, use it
file_name = File.expand_path(DEFAULT_REDIS_QUEUES_CONFIG_FILE_NAME, __dir__)
return file_name if File.file?(file_name)
# this will force use of DEFAULT_REDIS_QUEUES_URL when config file is absent
super
end
end
end
end
end
# please require all dependencies below:
require_relative 'wrapper' unless defined?(::Gitlab::Redis::Wrapper)
module Gitlab
module Redis
class SharedState < ::Gitlab::Redis::Wrapper
SESSION_NAMESPACE = 'session:gitlab'.freeze
DEFAULT_REDIS_SHARED_STATE_URL = 'redis://localhost:6382'.freeze
REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_SHARED_STATE_CONFIG_FILE'.freeze
if defined?(::Rails) && ::Rails.root.present?
DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME = ::Rails.root.join('config', 'redis.shared_state.yml').freeze
end
class << self
def default_url
DEFAULT_REDIS_SHARED_STATE_URL
end
def config_file_name
# if ENV set for this class, use it even if it points to a file does not exist
file_name = ENV[REDIS_SHARED_STATE_CONFIG_ENV_VAR_NAME]
return file_name if file_name
# otherwise, if config files exists for this class, use it
file_name = File.expand_path(DEFAULT_REDIS_SHARED_STATE_CONFIG_FILE_NAME, __dir__)
return file_name if File.file?(file_name)
# this will force use of DEFAULT_REDIS_SHARED_STATE_URL when config file is absent
super
end
end
end
end
end
# This file should only be used by sub-classes, not directly by any clients of the sub-classes
# please require all dependencies below:
require 'active_support/core_ext/hash/keys'
require 'active_support/core_ext/module/delegation'
module Gitlab
module Redis
class Wrapper
DEFAULT_REDIS_URL = 'redis://localhost:6379'.freeze
REDIS_CONFIG_ENV_VAR_NAME = 'GITLAB_REDIS_CONFIG_FILE'.freeze
if defined?(::Rails) && ::Rails.root.present?
DEFAULT_REDIS_CONFIG_FILE_NAME = ::Rails.root.join('config', 'resque.yml').freeze
end
class << self
delegate :params, :url, to: :new
def with
@pool ||= ConnectionPool.new(size: pool_size) { ::Redis.new(params) }
@pool.with { |redis| yield redis }
end
def pool_size
# heuristic constant 5 should be a config setting somewhere -- related to CPU count?
size = 5
if Sidekiq.server?
# the pool will be used in a multi-threaded context
size += Sidekiq.options[:concurrency]
end
size
end
def _raw_config
return @_raw_config if defined?(@_raw_config)
begin
@_raw_config = ERB.new(File.read(config_file_name)).result.freeze
rescue Errno::ENOENT
@_raw_config = false
end
@_raw_config
end
def default_url
DEFAULT_REDIS_URL
end
def config_file_name
# if ENV set for wrapper class, use it even if it points to a file does not exist
file_name = ENV[REDIS_CONFIG_ENV_VAR_NAME]
return file_name unless file_name.nil?
# otherwise, if config files exists for wrapper class, use it
file_name = File.expand_path(DEFAULT_REDIS_CONFIG_FILE_NAME, __dir__)
return file_name if File.file?(file_name)
# nil will force use of DEFAULT_REDIS_URL when config file is absent
nil
end
end
def initialize(rails_env = nil)
@rails_env = rails_env || ::Rails.env
end
def params
redis_store_options
end
def url
raw_config_hash[:url]
end
def sentinels
raw_config_hash[:sentinels]
end
def sentinels?
sentinels && !sentinels.empty?
end
private
def redis_store_options
config = raw_config_hash
redis_url = config.delete(:url)
redis_uri = URI.parse(redis_url)
if redis_uri.scheme == 'unix'
# Redis::Store does not handle Unix sockets well, so let's do it for them
config[:path] = redis_uri.path
query = redis_uri.query
unless query.nil?
queries = CGI.parse(redis_uri.query)
db_numbers = queries["db"] if queries.key?("db")
config[:db] = db_numbers[0].to_i if db_numbers.any?
end
config
else
redis_hash = ::Redis::Store::Factory.extract_host_options_from_uri(redis_url)
# order is important here, sentinels must be after the connection keys.
# {url: ..., port: ..., sentinels: [...]}
redis_hash.merge(config)
end
end
def raw_config_hash
config_data = fetch_config
if config_data
config_data.is_a?(String) ? { url: config_data } : config_data.deep_symbolize_keys
else
{ url: self.class.default_url }
end
end
def fetch_config
self.class._raw_config ? YAML.load(self.class._raw_config)[@rails_env] : false
end
end
end
end
...@@ -6,13 +6,13 @@ module Gitlab ...@@ -6,13 +6,13 @@ module Gitlab
BATCH_SIZE = 500 BATCH_SIZE = 500
def self.record(key, time = Time.now) def self.record(key, time = Time.now)
Gitlab::Redis.with do |redis| Gitlab::Redis::SharedState.with do |redis|
redis.hset(KEY, key, time.to_i) redis.hset(KEY, key, time.to_i)
end end
end end
def delete(*keys) def delete(*keys)
Gitlab::Redis.with do |redis| Gitlab::Redis::SharedState.with do |redis|
redis.hdel(KEY, keys) redis.hdel(KEY, keys)
end end
end end
...@@ -21,7 +21,7 @@ module Gitlab ...@@ -21,7 +21,7 @@ module Gitlab
cursor = 0 cursor = 0
loop do loop do
cursor, pairs = cursor, pairs =
Gitlab::Redis.with do |redis| Gitlab::Redis::SharedState.with do |redis|
redis.hscan(KEY, cursor, count: BATCH_SIZE) redis.hscan(KEY, cursor, count: BATCH_SIZE)
end end
......
...@@ -176,7 +176,7 @@ module Gitlab ...@@ -176,7 +176,7 @@ module Gitlab
end end
def set_key_and_notify(key, value, expire: nil, overwrite: true) def set_key_and_notify(key, value, expire: nil, overwrite: true)
Gitlab::Redis.with do |redis| Gitlab::Redis::Queues.with do |redis|
result = redis.set(key, value, ex: expire, nx: !overwrite) result = redis.set(key, value, ex: expire, nx: !overwrite)
if result if result
redis.publish(NOTIFICATION_CHANNEL, "#{key}=#{value}") redis.publish(NOTIFICATION_CHANNEL, "#{key}=#{value}")
......
...@@ -5,12 +5,12 @@ namespace :cache do ...@@ -5,12 +5,12 @@ namespace :cache do
desc "GitLab | Clear redis cache" desc "GitLab | Clear redis cache"
task redis: :environment do task redis: :environment do
Gitlab::Redis.with do |redis| Gitlab::Redis::Cache.with do |redis|
cursor = REDIS_SCAN_START_STOP cursor = REDIS_SCAN_START_STOP
loop do loop do
cursor, keys = redis.scan( cursor, keys = redis.scan(
cursor, cursor,
match: "#{Gitlab::Redis::CACHE_NAMESPACE}*", match: "#{Gitlab::Redis::Cache::CACHE_NAMESPACE}*",
count: REDIS_CLEAR_BATCH_SIZE count: REDIS_CLEAR_BATCH_SIZE
) )
......
...@@ -12,9 +12,6 @@ fi ...@@ -12,9 +12,6 @@ fi
# gems could not be found under some circumstance. No idea why, hours wasted. # gems could not be found under some circumstance. No idea why, hours wasted.
retry gem install knapsack fog-aws mime-types retry gem install knapsack fog-aws mime-types
cp config/resque.yml.example config/resque.yml
sed -i 's/localhost/redis/g' config/resque.yml
cp config/gitlab.yml.example config/gitlab.yml cp config/gitlab.yml.example config/gitlab.yml
# Determine the database by looking at the job name. # Determine the database by looking at the job name.
...@@ -37,6 +34,18 @@ else # Assume it's mysql ...@@ -37,6 +34,18 @@ else # Assume it's mysql
sed -i 's/# host:.*/host: mysql/g' config/database.yml sed -i 's/# host:.*/host: mysql/g' config/database.yml
fi fi
cp config/resque.yml.example config/resque.yml
sed -i 's/localhost/redis/g' config/resque.yml
cp config/redis.cache.yml.example config/redis.cache.yml
sed -i 's/localhost/redis/g' config/redis.cache.yml
cp config/redis.queues.yml.example config/redis.queues.yml
sed -i 's/localhost/redis/g' config/redis.queues.yml
cp config/redis.shared_state.yml.example config/redis.shared_state.yml
sed -i 's/localhost/redis/g' config/redis.shared_state.yml
if [ "$SETUP_DB" != "false" ]; then if [ "$SETUP_DB" != "false" ]; then
bundle exec rake db:drop db:create db:schema:load db:migrate bundle exec rake db:drop db:create db:schema:load db:migrate
......
...@@ -5,12 +5,12 @@ describe 'mail_room.yml' do ...@@ -5,12 +5,12 @@ describe 'mail_room.yml' do
let(:mailroom_config_path) { 'config/mail_room.yml' } let(:mailroom_config_path) { 'config/mail_room.yml' }
let(:gitlab_config_path) { 'config/mail_room.yml' } let(:gitlab_config_path) { 'config/mail_room.yml' }
let(:redis_config_path) { 'config/resque.yml' } let(:queues_config_path) { 'config/redis.queues.yml' }
let(:configuration) do let(:configuration) do
vars = { vars = {
'MAIL_ROOM_GITLAB_CONFIG_FILE' => absolute_path(gitlab_config_path), 'MAIL_ROOM_GITLAB_CONFIG_FILE' => absolute_path(gitlab_config_path),
'GITLAB_REDIS_CONFIG_FILE' => absolute_path(redis_config_path) 'GITLAB_REDIS_QUEUES_CONFIG_FILE' => absolute_path(queues_config_path)
} }
cmd = "puts ERB.new(File.read(#{absolute_path(mailroom_config_path).inspect})).result" cmd = "puts ERB.new(File.read(#{absolute_path(mailroom_config_path).inspect})).result"
...@@ -21,12 +21,12 @@ describe 'mail_room.yml' do ...@@ -21,12 +21,12 @@ describe 'mail_room.yml' do
end end
before(:each) do before(:each) do
stub_env('GITLAB_REDIS_CONFIG_FILE', absolute_path(redis_config_path)) stub_env('GITLAB_REDIS_QUEUES_CONFIG_FILE', absolute_path(queues_config_path))
clear_redis_raw_config clear_queues_raw_config
end end
after(:each) do after(:each) do
clear_redis_raw_config clear_queues_raw_config
end end
context 'when incoming email is disabled' do context 'when incoming email is disabled' do
...@@ -39,9 +39,9 @@ describe 'mail_room.yml' do ...@@ -39,9 +39,9 @@ describe 'mail_room.yml' do
context 'when incoming email is enabled' do context 'when incoming email is enabled' do
let(:gitlab_config_path) { 'spec/fixtures/config/mail_room_enabled.yml' } let(:gitlab_config_path) { 'spec/fixtures/config/mail_room_enabled.yml' }
let(:redis_config_path) { 'spec/fixtures/config/redis_new_format_host.yml' } let(:queues_config_path) { 'spec/fixtures/config/redis_queues_new_format_host.yml' }
let(:gitlab_redis) { Gitlab::Redis.new(Rails.env) } let(:gitlab_redis_queues) { Gitlab::Redis::Queues.new(Rails.env) }
it 'contains the intended configuration' do it 'contains the intended configuration' do
expect(configuration[:mailboxes].length).to eq(1) expect(configuration[:mailboxes].length).to eq(1)
...@@ -56,8 +56,8 @@ describe 'mail_room.yml' do ...@@ -56,8 +56,8 @@ describe 'mail_room.yml' do
expect(mailbox[:name]).to eq('inbox') expect(mailbox[:name]).to eq('inbox')
expect(mailbox[:idle_timeout]).to eq(60) expect(mailbox[:idle_timeout]).to eq(60)
redis_url = gitlab_redis.url redis_url = gitlab_redis_queues.url
sentinels = gitlab_redis.sentinels sentinels = gitlab_redis_queues.sentinels
expect(mailbox[:delivery_options][:redis_url]).to be_present expect(mailbox[:delivery_options][:redis_url]).to be_present
expect(mailbox[:delivery_options][:redis_url]).to eq(redis_url) expect(mailbox[:delivery_options][:redis_url]).to eq(redis_url)
...@@ -73,8 +73,8 @@ describe 'mail_room.yml' do ...@@ -73,8 +73,8 @@ describe 'mail_room.yml' do
end end
end end
def clear_redis_raw_config def clear_queues_raw_config
Gitlab::Redis.remove_instance_variable(:@_raw_config) Gitlab::Redis::Queues.remove_instance_variable(:@_raw_config)
rescue NameError rescue NameError
# raised if @_raw_config was not set; ignore # raised if @_raw_config was not set; ignore
end end
......
...@@ -19,7 +19,9 @@ describe HealthController do ...@@ -19,7 +19,9 @@ describe HealthController do
it 'returns proper response' do it 'returns proper response' do
get :readiness get :readiness
expect(json_response['db_check']['status']).to eq('ok') expect(json_response['db_check']['status']).to eq('ok')
expect(json_response['redis_check']['status']).to eq('ok') expect(json_response['cache_check']['status']).to eq('ok')
expect(json_response['queues_check']['status']).to eq('ok')
expect(json_response['shared_state_check']['status']).to eq('ok')
expect(json_response['fs_shards_check']['status']).to eq('ok') expect(json_response['fs_shards_check']['status']).to eq('ok')
expect(json_response['fs_shards_check']['labels']['shard']).to eq('default') expect(json_response['fs_shards_check']['labels']['shard']).to eq('default')
end end
...@@ -42,7 +44,9 @@ describe HealthController do ...@@ -42,7 +44,9 @@ describe HealthController do
it 'returns proper response' do it 'returns proper response' do
get :liveness get :liveness
expect(json_response['db_check']['status']).to eq('ok') expect(json_response['db_check']['status']).to eq('ok')
expect(json_response['redis_check']['status']).to eq('ok') expect(json_response['cache_check']['status']).to eq('ok')
expect(json_response['queues_check']['status']).to eq('ok')
expect(json_response['shared_state_check']['status']).to eq('ok')
expect(json_response['fs_shards_check']['status']).to eq('ok') expect(json_response['fs_shards_check']['status']).to eq('ok')
end end
end end
......
...@@ -35,6 +35,30 @@ describe MetricsController do ...@@ -35,6 +35,30 @@ describe MetricsController do
expect(response.body).to match(/^redis_ping_latency [0-9\.]+$/) expect(response.body).to match(/^redis_ping_latency [0-9\.]+$/)
end end
it 'returns Caching ping metrics' do
get :index
expect(response.body).to match(/^redis_cache_ping_timeout 0$/)
expect(response.body).to match(/^redis_cache_ping_success 1$/)
expect(response.body).to match(/^redis_cache_ping_latency [0-9\.]+$/)
end
it 'returns Queues ping metrics' do
get :index
expect(response.body).to match(/^redis_queues_ping_timeout 0$/)
expect(response.body).to match(/^redis_queues_ping_success 1$/)
expect(response.body).to match(/^redis_queues_ping_latency [0-9\.]+$/)
end
it 'returns SharedState ping metrics' do
get :index
expect(response.body).to match(/^redis_shared_state_ping_timeout 0$/)
expect(response.body).to match(/^redis_shared_state_ping_success 1$/)
expect(response.body).to match(/^redis_shared_state_ping_latency [0-9\.]+$/)
end
it 'returns file system check metrics' do it 'returns file system check metrics' do
get :index get :index
......
...@@ -47,7 +47,7 @@ describe SessionsController do ...@@ -47,7 +47,7 @@ describe SessionsController do
end end
end end
context 'when using valid password', :redis do context 'when using valid password', :clean_gitlab_redis_shared_state do
include UserActivitiesHelpers include UserActivitiesHelpers
let(:user) { create(:user) } let(:user) { create(:user) }
......
require 'spec_helper' require 'spec_helper'
describe 'Navigation bar counter', feature: true, caching: true do describe 'Navigation bar counter', :use_clean_rails_memory_store_caching, feature: true do
let(:user) { create(:user) } let(:user) { create(:user) }
let(:project) { create(:empty_project, namespace: user.namespace) } let(:project) { create(:empty_project, namespace: user.namespace) }
let(:issue) { create(:issue, project: project) } let(:issue) { create(:issue, project: project) }
......
...@@ -61,7 +61,7 @@ feature 'Dashboard Projects' do ...@@ -61,7 +61,7 @@ feature 'Dashboard Projects' do
end end
end end
describe 'with a pipeline', redis: true do describe "with a pipeline", clean_gitlab_redis_shared_state: true do
let(:pipeline) { create(:ci_pipeline, project: project, sha: project.commit.sha) } let(:pipeline) { create(:ci_pipeline, project: project, sha: project.commit.sha) }
before do before do
......
...@@ -68,7 +68,7 @@ feature 'Groups > Members > Sort members', feature: true do ...@@ -68,7 +68,7 @@ feature 'Groups > Members > Sort members', feature: true do
expect(page).to have_css('.member-sort-dropdown .dropdown-toggle-text', text: 'Name, descending') expect(page).to have_css('.member-sort-dropdown .dropdown-toggle-text', text: 'Name, descending')
end end
scenario 'sorts by recent sign in', :redis do scenario 'sorts by recent sign in', :clean_gitlab_redis_shared_state do
visit_members_list(sort: :recent_sign_in) visit_members_list(sort: :recent_sign_in)
expect(first_member).to include(owner.name) expect(first_member).to include(owner.name)
...@@ -76,7 +76,7 @@ feature 'Groups > Members > Sort members', feature: true do ...@@ -76,7 +76,7 @@ feature 'Groups > Members > Sort members', feature: true do
expect(page).to have_css('.member-sort-dropdown .dropdown-toggle-text', text: 'Recent sign in') expect(page).to have_css('.member-sort-dropdown .dropdown-toggle-text', text: 'Recent sign in')
end end
scenario 'sorts by oldest sign in', :redis do scenario 'sorts by oldest sign in', :clean_gitlab_redis_shared_state do
visit_members_list(sort: :oldest_sign_in) visit_members_list(sort: :oldest_sign_in)
expect(first_member).to include(developer.name) expect(first_member).to include(developer.name)
......
...@@ -41,7 +41,7 @@ feature 'Login', feature: true do ...@@ -41,7 +41,7 @@ feature 'Login', feature: true do
expect(page).to have_content('Your account has been blocked.') expect(page).to have_content('Your account has been blocked.')
end end
it 'does not update Devise trackable attributes', :redis do it 'does not update Devise trackable attributes', :clean_gitlab_redis_shared_state do
user = create(:user, :blocked) user = create(:user, :blocked)
expect { gitlab_sign_in(user) }.not_to change { user.reload.sign_in_count } expect { gitlab_sign_in(user) }.not_to change { user.reload.sign_in_count }
...@@ -55,7 +55,7 @@ feature 'Login', feature: true do ...@@ -55,7 +55,7 @@ feature 'Login', feature: true do
expect(page).to have_content('Invalid Login or password.') expect(page).to have_content('Invalid Login or password.')
end end
it 'does not update Devise trackable attributes', :redis do it 'does not update Devise trackable attributes', :clean_gitlab_redis_shared_state do
expect { gitlab_sign_in(User.ghost) }.not_to change { User.ghost.reload.sign_in_count } expect { gitlab_sign_in(User.ghost) }.not_to change { User.ghost.reload.sign_in_count }
end end
end end
......
...@@ -67,7 +67,7 @@ feature 'Projects > Members > Sorting', feature: true do ...@@ -67,7 +67,7 @@ feature 'Projects > Members > Sorting', feature: true do
expect(page).to have_css('.member-sort-dropdown .dropdown-toggle-text', text: 'Name, descending') expect(page).to have_css('.member-sort-dropdown .dropdown-toggle-text', text: 'Name, descending')
end end
scenario 'sorts by recent sign in', :redis do scenario 'sorts by recent sign in', :clean_gitlab_redis_shared_state do
visit_members_list(sort: :recent_sign_in) visit_members_list(sort: :recent_sign_in)
expect(first_member).to include(master.name) expect(first_member).to include(master.name)
...@@ -75,7 +75,7 @@ feature 'Projects > Members > Sorting', feature: true do ...@@ -75,7 +75,7 @@ feature 'Projects > Members > Sorting', feature: true do
expect(page).to have_css('.member-sort-dropdown .dropdown-toggle-text', text: 'Recent sign in') expect(page).to have_css('.member-sort-dropdown .dropdown-toggle-text', text: 'Recent sign in')
end end
scenario 'sorts by oldest sign in', :redis do scenario 'sorts by oldest sign in', :clean_gitlab_redis_shared_state do
visit_members_list(sort: :oldest_sign_in) visit_members_list(sort: :oldest_sign_in)
expect(first_member).to include(developer.name) expect(first_member).to include(developer.name)
......
test:
url: <%= ENV['TEST_GITLAB_REDIS_CACHE_URL'] %>
# redis://[:password@]host[:port][/db-number][?option=value]
# more details: http://www.iana.org/assignments/uri-schemes/prov/redis
development:
url: redis://:mynewpassword@localhost:6380/10
sentinels:
-
host: localhost
port: 26380 # point to sentinel, not to redis port
-
host: slave2
port: 26380 # point to sentinel, not to redis port
test:
url: redis://:mynewpassword@localhost:6380/10
sentinels:
-
host: localhost
port: 26380 # point to sentinel, not to redis port
-
host: slave2
port: 26380 # point to sentinel, not to redis port
production:
url: redis://:mynewpassword@localhost:6380/10
sentinels:
-
host: slave1
port: 26380 # point to sentinel, not to redis port
-
host: slave2
port: 26380 # point to sentinel, not to redis port
development:
url: unix:/path/to/redis.cache.sock
test:
url: unix:/path/to/redis.cache.sock
production:
url: unix:/path/to/redis.cache.sock
# redis://[:password@]host[:port][/db-number][?option=value]
# more details: http://www.iana.org/assignments/uri-schemes/prov/redis
development: redis://:mypassword@localhost:6380/10
test: redis://:mypassword@localhost:6380/10
production: redis://:mypassword@localhost:6380/10
development: unix:/path/to/old/redis.cache.sock
test: unix:/path/to/old/redis.cache.sock
production: unix:/path/to/old/redis.cache.sock
...@@ -5,25 +5,25 @@ development: ...@@ -5,25 +5,25 @@ development:
sentinels: sentinels:
- -
host: localhost host: localhost
port: 26380 # point to sentinel, not to redis port port: 26379 # point to sentinel, not to redis port
- -
host: slave2 host: slave2
port: 26381 # point to sentinel, not to redis port port: 26379 # point to sentinel, not to redis port
test: test:
url: redis://:mynewpassword@localhost:6379/99 url: redis://:mynewpassword@localhost:6379/99
sentinels: sentinels:
- -
host: localhost host: localhost
port: 26380 # point to sentinel, not to redis port port: 26379 # point to sentinel, not to redis port
- -
host: slave2 host: slave2
port: 26381 # point to sentinel, not to redis port port: 26379 # point to sentinel, not to redis port
production: production:
url: redis://:mynewpassword@localhost:6379/99 url: redis://:mynewpassword@localhost:6379/99
sentinels: sentinels:
- -
host: slave1 host: slave1
port: 26380 # point to sentinel, not to redis port port: 26379 # point to sentinel, not to redis port
- -
host: slave2 host: slave2
port: 26381 # point to sentinel, not to redis port port: 26379 # point to sentinel, not to redis port
test:
url: <%= ENV['TEST_GITLAB_REDIS_QUEUES_URL'] %>
# redis://[:password@]host[:port][/db-number][?option=value]
# more details: http://www.iana.org/assignments/uri-schemes/prov/redis
development:
url: redis://:mynewpassword@localhost:6381/11
sentinels:
-
host: localhost
port: 26381 # point to sentinel, not to redis port
-
host: slave2
port: 26381 # point to sentinel, not to redis port
test:
url: redis://:mynewpassword@localhost:6381/11
sentinels:
-
host: localhost
port: 26381 # point to sentinel, not to redis port
-
host: slave2
port: 26381 # point to sentinel, not to redis port
production:
url: redis://:mynewpassword@localhost:6381/11
sentinels:
-
host: slave1
port: 26381 # point to sentinel, not to redis port
-
host: slave2
port: 26381 # point to sentinel, not to redis port
development:
url: unix:/path/to/redis.queues.sock
test:
url: unix:/path/to/redis.queues.sock
production:
url: unix:/path/to/redis.queues.sock
# redis://[:password@]host[:port][/db-number][?option=value]
# more details: http://www.iana.org/assignments/uri-schemes/prov/redis
development: redis://:mypassword@localhost:6381/11
test: redis://:mypassword@localhost:6381/11
production: redis://:mypassword@localhost:6381/11
development: unix:/path/to/old/redis.queues.sock
test: unix:/path/to/old/redis.queues.sock
production: unix:/path/to/old/redis.queues.sock
test:
url: <%= ENV['TEST_GITLAB_REDIS_SHARED_STATE_URL'] %>
# redis://[:password@]host[:port][/db-number][?option=value]
# more details: http://www.iana.org/assignments/uri-schemes/prov/redis
development:
url: redis://:mynewpassword@localhost:6382/12
sentinels:
-
host: localhost
port: 26382 # point to sentinel, not to redis port
-
host: slave2
port: 26382 # point to sentinel, not to redis port
test:
url: redis://:mynewpassword@localhost:6382/12
sentinels:
-
host: localhost
port: 26382 # point to sentinel, not to redis port
-
host: slave2
port: 26382 # point to sentinel, not to redis port
production:
url: redis://:mynewpassword@localhost:6382/12
sentinels:
-
host: slave1
port: 26382 # point to sentinel, not to redis port
-
host: slave2
port: 26382 # point to sentinel, not to redis port
development:
url: unix:/path/to/redis.shared_state.sock
test:
url: unix:/path/to/redis.shared_state.sock
production:
url: unix:/path/to/redis.shared_state.sock
# redis://[:password@]host[:port][/db-number][?option=value]
# more details: http://www.iana.org/assignments/uri-schemes/prov/redis
development: redis://:mypassword@localhost:6382/12
test: redis://:mypassword@localhost:6382/12
production: redis://:mypassword@localhost:6382/12
development: unix:/path/to/old/redis.shared_state.sock
test: unix:/path/to/old/redis.shared_state.sock
production: unix:/path/to/old/redis.shared_state.sock
...@@ -60,7 +60,7 @@ describe IssuablesHelper do ...@@ -60,7 +60,7 @@ describe IssuablesHelper do
end end
end end
describe 'counter caching based on issuable type and params', :caching do describe 'counter caching based on issuable type and params', :use_clean_rails_memory_store_caching do
let(:params) do let(:params) do
{ {
scope: 'created-by-me', scope: 'created-by-me',
......
...@@ -63,7 +63,7 @@ describe ProjectsHelper do ...@@ -63,7 +63,7 @@ describe ProjectsHelper do
end end
end end
describe "#project_list_cache_key", redis: true do describe "#project_list_cache_key", clean_gitlab_redis_shared_state: true do
let(:project) { create(:project) } let(:project) { create(:project) }
it "includes the route" do it "includes the route" do
......
require 'spec_helper' require 'spec_helper'
describe Gitlab::Auth::UniqueIpsLimiter, :redis, lib: true do describe Gitlab::Auth::UniqueIpsLimiter, :clean_gitlab_redis_shared_state, lib: true do
include_context 'unique ips sign in limit' include_context 'unique ips sign in limit'
let(:user) { create(:user) } let(:user) { create(:user) }
......
require 'spec_helper' require 'spec_helper'
describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do describe Gitlab::Cache::Ci::ProjectPipelineStatus, :clean_gitlab_redis_cache do
let!(:project) { create(:project) } let!(:project) { create(:project) }
let(:pipeline_status) { described_class.new(project) } let(:pipeline_status) { described_class.new(project) }
let(:cache_key) { "projects/#{project.id}/pipeline_status" } let(:cache_key) { "projects/#{project.id}/pipeline_status" }
...@@ -28,8 +28,8 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do ...@@ -28,8 +28,8 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do
expect(project.instance_variable_get('@pipeline_status')).to be_a(described_class) expect(project.instance_variable_get('@pipeline_status')).to be_a(described_class)
end end
describe 'without a status in redis' do describe 'without a status in redis_cache' do
it 'loads the status from a commit when it was not in redis' do it 'loads the status from a commit when it was not in redis_cache' do
empty_status = { sha: nil, status: nil, ref: nil } empty_status = { sha: nil, status: nil, ref: nil }
fake_pipeline = described_class.new( fake_pipeline = described_class.new(
project_without_status, project_without_status,
...@@ -48,9 +48,9 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do ...@@ -48,9 +48,9 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do
described_class.load_in_batch_for_projects([project_without_status]) described_class.load_in_batch_for_projects([project_without_status])
end end
it 'only connects to redis twice' do it 'only connects to redis_cache twice' do
# Once to load, once to store in the cache # Once to load, once to store in the cache
expect(Gitlab::Redis).to receive(:with).exactly(2).and_call_original expect(Gitlab::Redis::Cache).to receive(:with).exactly(2).and_call_original
described_class.load_in_batch_for_projects([project_without_status]) described_class.load_in_batch_for_projects([project_without_status])
...@@ -58,9 +58,9 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do ...@@ -58,9 +58,9 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do
end end
end end
describe 'when a status was cached in redis' do describe 'when a status was cached in redis_cache' do
before do before do
Gitlab::Redis.with do |redis| Gitlab::Redis::Cache.with do |redis|
redis.mapped_hmset(cache_key, redis.mapped_hmset(cache_key,
{ sha: sha, status: status, ref: ref }) { sha: sha, status: status, ref: ref })
end end
...@@ -76,8 +76,8 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do ...@@ -76,8 +76,8 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do
expect(pipeline_status.ref).to eq(ref) expect(pipeline_status.ref).to eq(ref)
end end
it 'only connects to redis once' do it 'only connects to redis_cache once' do
expect(Gitlab::Redis).to receive(:with).exactly(1).and_call_original expect(Gitlab::Redis::Cache).to receive(:with).exactly(1).and_call_original
described_class.load_in_batch_for_projects([project]) described_class.load_in_batch_for_projects([project])
...@@ -94,8 +94,8 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do ...@@ -94,8 +94,8 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do
end end
describe '.cached_results_for_projects' do describe '.cached_results_for_projects' do
it 'loads a status from redis for all projects' do it 'loads a status from caching for all projects' do
Gitlab::Redis.with do |redis| Gitlab::Redis::Cache.with do |redis|
redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref }) redis.mapped_hmset(cache_key, { sha: sha, status: status, ref: ref })
end end
...@@ -183,7 +183,7 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do ...@@ -183,7 +183,7 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do
end end
end end
describe "#load_from_project" do describe "#load_from_project", :clean_gitlab_redis_cache do
let!(:pipeline) { create(:ci_pipeline, :success, project: project, sha: project.commit.sha) } let!(:pipeline) { create(:ci_pipeline, :success, project: project, sha: project.commit.sha) }
it 'reads the status from the pipeline for the commit' do it 'reads the status from the pipeline for the commit' do
...@@ -203,40 +203,40 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do ...@@ -203,40 +203,40 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do
end end
end end
describe "#store_in_cache", :redis do describe "#store_in_cache", :clean_gitlab_redis_cache do
it "sets the object in redis" do it "sets the object in caching" do
pipeline_status.sha = '123456' pipeline_status.sha = '123456'
pipeline_status.status = 'failed' pipeline_status.status = 'failed'
pipeline_status.store_in_cache pipeline_status.store_in_cache
read_sha, read_status = Gitlab::Redis.with { |redis| redis.hmget(cache_key, :sha, :status) } read_sha, read_status = Gitlab::Redis::Cache.with { |redis| redis.hmget(cache_key, :sha, :status) }
expect(read_sha).to eq('123456') expect(read_sha).to eq('123456')
expect(read_status).to eq('failed') expect(read_status).to eq('failed')
end end
end end
describe '#store_in_cache_if_needed', :redis do describe '#store_in_cache_if_needed', :clean_gitlab_redis_cache do
it 'stores the state in the cache when the sha is the HEAD of the project' do it 'stores the state in the cache when the sha is the HEAD of the project' do
create(:ci_pipeline, :success, project: project, sha: project.commit.sha) create(:ci_pipeline, :success, project: project, sha: project.commit.sha)
pipeline_status = described_class.load_for_project(project) pipeline_status = described_class.load_for_project(project)
pipeline_status.store_in_cache_if_needed pipeline_status.store_in_cache_if_needed
sha, status, ref = Gitlab::Redis.with { |redis| redis.hmget(cache_key, :sha, :status, :ref) } sha, status, ref = Gitlab::Redis::Cache.with { |redis| redis.hmget(cache_key, :sha, :status, :ref) }
expect(sha).not_to be_nil expect(sha).not_to be_nil
expect(status).not_to be_nil expect(status).not_to be_nil
expect(ref).not_to be_nil expect(ref).not_to be_nil
end end
it "doesn't store the status in redis when the sha is not the head of the project" do it "doesn't store the status in redis_cache when the sha is not the head of the project" do
other_status = described_class.new( other_status = described_class.new(
project, project,
pipeline_info: { sha: "123456", status: "failed" } pipeline_info: { sha: "123456", status: "failed" }
) )
other_status.store_in_cache_if_needed other_status.store_in_cache_if_needed
sha, status = Gitlab::Redis.with { |redis| redis.hmget(cache_key, :sha, :status) } sha, status = Gitlab::Redis::Cache.with { |redis| redis.hmget(cache_key, :sha, :status) }
expect(sha).to be_nil expect(sha).to be_nil
expect(status).to be_nil expect(status).to be_nil
...@@ -244,7 +244,7 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do ...@@ -244,7 +244,7 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do
it "deletes the cache if the repository doesn't have a head commit" do it "deletes the cache if the repository doesn't have a head commit" do
empty_project = create(:empty_project) empty_project = create(:empty_project)
Gitlab::Redis.with do |redis| Gitlab::Redis::Cache.with do |redis|
redis.mapped_hmset(cache_key, redis.mapped_hmset(cache_key,
{ sha: 'sha', status: 'pending', ref: 'master' }) { sha: 'sha', status: 'pending', ref: 'master' })
end end
...@@ -255,7 +255,7 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do ...@@ -255,7 +255,7 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do
}) })
other_status.store_in_cache_if_needed other_status.store_in_cache_if_needed
sha, status, ref = Gitlab::Redis.with { |redis| redis.hmget("projects/#{empty_project.id}/pipeline_status", :sha, :status, :ref) } sha, status, ref = Gitlab::Redis::Cache.with { |redis| redis.hmget("projects/#{empty_project.id}/pipeline_status", :sha, :status, :ref) }
expect(sha).to be_nil expect(sha).to be_nil
expect(status).to be_nil expect(status).to be_nil
...@@ -263,20 +263,20 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do ...@@ -263,20 +263,20 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do
end end
end end
describe "with a status in redis", :redis do describe "with a status in caching", :clean_gitlab_redis_cache do
let(:status) { 'success' } let(:status) { 'success' }
let(:sha) { '424d1b73bc0d3cb726eb7dc4ce17a4d48552f8c6' } let(:sha) { '424d1b73bc0d3cb726eb7dc4ce17a4d48552f8c6' }
let(:ref) { 'master' } let(:ref) { 'master' }
before do before do
Gitlab::Redis.with do |redis| Gitlab::Redis::Cache.with do |redis|
redis.mapped_hmset(cache_key, redis.mapped_hmset(cache_key,
{ sha: sha, status: status, ref: ref }) { sha: sha, status: status, ref: ref })
end end
end end
describe '#load_from_cache' do describe '#load_from_cache' do
it 'reads the status from redis' do it 'reads the status from redis_cache' do
pipeline_status.load_from_cache pipeline_status.load_from_cache
expect(pipeline_status.sha).to eq(sha) expect(pipeline_status.sha).to eq(sha)
...@@ -292,10 +292,10 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do ...@@ -292,10 +292,10 @@ describe Gitlab::Cache::Ci::ProjectPipelineStatus, :redis do
end end
describe '#delete_from_cache' do describe '#delete_from_cache' do
it 'deletes values from redis' do it 'deletes values from redis_cache' do
pipeline_status.delete_from_cache pipeline_status.delete_from_cache
key_exists = Gitlab::Redis.with { |redis| redis.exists(cache_key) } key_exists = Gitlab::Redis::Cache.with { |redis| redis.exists(cache_key) }
expect(key_exists).to be_falsy expect(key_exists).to be_falsy
end end
......
...@@ -236,7 +236,7 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameBase, :trunca ...@@ -236,7 +236,7 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameBase, :trunca
subject.track_rename('namespace', 'path/to/namespace', 'path/to/renamed') subject.track_rename('namespace', 'path/to/namespace', 'path/to/renamed')
old_path, new_path = [nil, nil] old_path, new_path = [nil, nil]
Gitlab::Redis.with do |redis| Gitlab::Redis::SharedState.with do |redis|
rename_info = redis.lpop(key) rename_info = redis.lpop(key)
old_path, new_path = JSON.parse(rename_info) old_path, new_path = JSON.parse(rename_info)
end end
...@@ -268,7 +268,7 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameBase, :trunca ...@@ -268,7 +268,7 @@ describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameBase, :trunca
key = 'rename:FakeRenameReservedPathMigrationV1:project' key = 'rename:FakeRenameReservedPathMigrationV1:project'
stored_renames = nil stored_renames = nil
rename_count = 0 rename_count = 0
Gitlab::Redis.with do |redis| Gitlab::Redis::SharedState.with do |redis|
stored_renames = redis.lrange(key, 0, 1) stored_renames = redis.lrange(key, 0, 1)
rename_count = redis.llen(key) rename_count = redis.llen(key)
end end
......
require 'spec_helper' require 'spec_helper'
describe Gitlab::ExclusiveLease, type: :redis do describe Gitlab::ExclusiveLease, type: :clean_gitlab_redis_shared_state do
let(:unique_key) { SecureRandom.hex(10) } let(:unique_key) { SecureRandom.hex(10) }
describe '#try_obtain' do describe '#try_obtain' do
......
require 'spec_helper'
require_relative '../simple_check_shared'
describe Gitlab::HealthChecks::Redis::CacheCheck do
include_examples 'simple_check', 'redis_cache_ping', 'RedisCache', 'PONG'
end
require 'spec_helper'
require_relative '../simple_check_shared'
describe Gitlab::HealthChecks::Redis::QueuesCheck do
include_examples 'simple_check', 'redis_queues_ping', 'RedisQueues', 'PONG'
end
require 'spec_helper' require 'spec_helper'
require_relative './simple_check_shared' require_relative '../simple_check_shared'
describe Gitlab::HealthChecks::RedisCheck do describe Gitlab::HealthChecks::Redis::RedisCheck do
include_examples 'simple_check', 'redis_ping', 'Redis', 'PONG' include_examples 'simple_check', 'redis_ping', 'Redis', 'PONG'
end end
require 'spec_helper'
require_relative '../simple_check_shared'
describe Gitlab::HealthChecks::Redis::SharedStateCheck do
include_examples 'simple_check', 'redis_shared_state_ping', 'RedisSharedState', 'PONG'
end
...@@ -47,7 +47,7 @@ shared_context 'simple_check' do |metrics_prefix, check_name, success_result| ...@@ -47,7 +47,7 @@ shared_context 'simple_check' do |metrics_prefix, check_name, success_result|
allow(described_class).to receive(:check).and_return 'error!' allow(described_class).to receive(:check).and_return 'error!'
end end
it { is_expected.to have_attributes(success: false, message: "unexpected #{check_name} check result: error!") } it { is_expected.to have_attributes(success: false, message: "unexpected #{described_class.human_name} check result: error!") }
end end
context 'Check is timeouting' do context 'Check is timeouting' do
...@@ -55,7 +55,7 @@ shared_context 'simple_check' do |metrics_prefix, check_name, success_result| ...@@ -55,7 +55,7 @@ shared_context 'simple_check' do |metrics_prefix, check_name, success_result|
allow(described_class).to receive(:check ).and_return Timeout::Error.new allow(described_class).to receive(:check ).and_return Timeout::Error.new
end end
it { is_expected.to have_attributes(success: false, message: "#{check_name} check timed out") } it { is_expected.to have_attributes(success: false, message: "#{described_class.human_name} check timed out") }
end end
end end
......
...@@ -7,7 +7,7 @@ describe Gitlab::PerformanceBar do ...@@ -7,7 +7,7 @@ describe Gitlab::PerformanceBar do
described_class.enabled?(user) described_class.enabled?(user)
end end
it 'caches the allowed user IDs in cache', :caching do it 'caches the allowed user IDs in cache', :use_clean_rails_memory_store_caching do
expect do expect do
expect(described_class.enabled?(user)).to be_truthy expect(described_class.enabled?(user)).to be_truthy
end.not_to exceed_query_limit(0) end.not_to exceed_query_limit(0)
......
require 'spec_helper'
describe Gitlab::Redis::Cache do
let(:config_file_name) { "config/redis.cache.yml" }
let(:environment_config_file_name) { "GITLAB_REDIS_CACHE_CONFIG_FILE" }
let(:config_old_format_socket) { "spec/fixtures/config/redis_cache_old_format_socket.yml" }
let(:config_new_format_socket) { "spec/fixtures/config/redis_cache_new_format_socket.yml" }
let(:old_socket_path) {"/path/to/old/redis.cache.sock" }
let(:new_socket_path) {"/path/to/redis.cache.sock" }
let(:config_old_format_host) { "spec/fixtures/config/redis_cache_old_format_host.yml" }
let(:config_new_format_host) { "spec/fixtures/config/redis_cache_new_format_host.yml" }
let(:redis_port) { 6380 }
let(:redis_database) { 10 }
let(:sentinel_port) { redis_port + 20000 }
let(:config_with_environment_variable_inside) { "spec/fixtures/config/redis_cache_config_with_env.yml"}
let(:config_env_variable_url) {"TEST_GITLAB_REDIS_CACHE_URL"}
let(:class_redis_url) { Gitlab::Redis::Cache::DEFAULT_REDIS_CACHE_URL }
include_examples "redis_shared_examples"
end
require 'spec_helper'
describe Gitlab::Redis::Queues do
let(:config_file_name) { "config/redis.queues.yml" }
let(:environment_config_file_name) { "GITLAB_REDIS_QUEUES_CONFIG_FILE" }
let(:config_old_format_socket) { "spec/fixtures/config/redis_queues_old_format_socket.yml" }
let(:config_new_format_socket) { "spec/fixtures/config/redis_queues_new_format_socket.yml" }
let(:old_socket_path) {"/path/to/old/redis.queues.sock" }
let(:new_socket_path) {"/path/to/redis.queues.sock" }
let(:config_old_format_host) { "spec/fixtures/config/redis_queues_old_format_host.yml" }
let(:config_new_format_host) { "spec/fixtures/config/redis_queues_new_format_host.yml" }
let(:redis_port) { 6381 }
let(:redis_database) { 11 }
let(:sentinel_port) { redis_port + 20000 }
let(:config_with_environment_variable_inside) { "spec/fixtures/config/redis_queues_config_with_env.yml"}
let(:config_env_variable_url) {"TEST_GITLAB_REDIS_QUEUES_URL"}
let(:class_redis_url) { Gitlab::Redis::Queues::DEFAULT_REDIS_QUEUES_URL }
include_examples "redis_shared_examples"
end
require 'spec_helper'
describe Gitlab::Redis::SharedState do
let(:config_file_name) { "config/redis.shared_state.yml" }
let(:environment_config_file_name) { "GITLAB_REDIS_SHARED_STATE_CONFIG_FILE" }
let(:config_old_format_socket) { "spec/fixtures/config/redis_shared_state_old_format_socket.yml" }
let(:config_new_format_socket) { "spec/fixtures/config/redis_shared_state_new_format_socket.yml" }
let(:old_socket_path) {"/path/to/old/redis.shared_state.sock" }
let(:new_socket_path) {"/path/to/redis.shared_state.sock" }
let(:config_old_format_host) { "spec/fixtures/config/redis_shared_state_old_format_host.yml" }
let(:config_new_format_host) { "spec/fixtures/config/redis_shared_state_new_format_host.yml" }
let(:redis_port) { 6382 }
let(:redis_database) { 12 }
let(:sentinel_port) { redis_port + 20000 }
let(:config_with_environment_variable_inside) { "spec/fixtures/config/redis_shared_state_config_with_env.yml"}
let(:config_env_variable_url) {"TEST_GITLAB_REDIS_SHARED_STATE_URL"}
let(:class_redis_url) { Gitlab::Redis::SharedState::DEFAULT_REDIS_SHARED_STATE_URL }
include_examples "redis_shared_examples"
end
require 'spec_helper'
describe Gitlab::Redis::Wrapper do
let(:config_file_name) { "config/resque.yml" }
let(:environment_config_file_name) { "GITLAB_REDIS_CONFIG_FILE" }
let(:config_old_format_socket) { "spec/fixtures/config/redis_old_format_socket.yml" }
let(:config_new_format_socket) { "spec/fixtures/config/redis_new_format_socket.yml" }
let(:old_socket_path) {"/path/to/old/redis.sock" }
let(:new_socket_path) {"/path/to/redis.sock" }
let(:config_old_format_host) { "spec/fixtures/config/redis_old_format_host.yml" }
let(:config_new_format_host) { "spec/fixtures/config/redis_new_format_host.yml" }
let(:redis_port) { 6379 }
let(:redis_database) { 99 }
let(:sentinel_port) { redis_port + 20000 }
let(:config_with_environment_variable_inside) { "spec/fixtures/config/redis_config_with_env.yml"}
let(:config_env_variable_url) {"TEST_GITLAB_REDIS_URL"}
let(:class_redis_url) { Gitlab::Redis::Wrapper::DEFAULT_REDIS_URL }
include_examples "redis_shared_examples"
end
require 'spec_helper' require 'spec_helper'
describe Gitlab::SidekiqStatus do describe Gitlab::SidekiqStatus do
describe '.set', :redis do describe '.set', :clean_gitlab_redis_shared_state do
it 'stores the job ID' do it 'stores the job ID' do
described_class.set('123') described_class.set('123')
...@@ -14,7 +14,7 @@ describe Gitlab::SidekiqStatus do ...@@ -14,7 +14,7 @@ describe Gitlab::SidekiqStatus do
end end
end end
describe '.unset', :redis do describe '.unset', :clean_gitlab_redis_shared_state do
it 'removes the job ID' do it 'removes the job ID' do
described_class.set('123') described_class.set('123')
described_class.unset('123') described_class.unset('123')
...@@ -27,7 +27,7 @@ describe Gitlab::SidekiqStatus do ...@@ -27,7 +27,7 @@ describe Gitlab::SidekiqStatus do
end end
end end
describe '.all_completed?', :redis do describe '.all_completed?', :clean_gitlab_redis_shared_state do
it 'returns true if all jobs have been completed' do it 'returns true if all jobs have been completed' do
expect(described_class.all_completed?(%w(123))).to eq(true) expect(described_class.all_completed?(%w(123))).to eq(true)
end end
...@@ -39,7 +39,7 @@ describe Gitlab::SidekiqStatus do ...@@ -39,7 +39,7 @@ describe Gitlab::SidekiqStatus do
end end
end end
describe '.num_running', :redis do describe '.num_running', :clean_gitlab_redis_shared_state do
it 'returns 0 if all jobs have been completed' do it 'returns 0 if all jobs have been completed' do
expect(described_class.num_running(%w(123))).to eq(0) expect(described_class.num_running(%w(123))).to eq(0)
end end
...@@ -52,7 +52,7 @@ describe Gitlab::SidekiqStatus do ...@@ -52,7 +52,7 @@ describe Gitlab::SidekiqStatus do
end end
end end
describe '.num_completed', :redis do describe '.num_completed', :clean_gitlab_redis_shared_state do
it 'returns 1 if all jobs have been completed' do it 'returns 1 if all jobs have been completed' do
expect(described_class.num_completed(%w(123))).to eq(1) expect(described_class.num_completed(%w(123))).to eq(1)
end end
...@@ -74,7 +74,7 @@ describe Gitlab::SidekiqStatus do ...@@ -74,7 +74,7 @@ describe Gitlab::SidekiqStatus do
end end
end end
describe 'completed', :redis do describe 'completed', :clean_gitlab_redis_shared_state do
it 'returns the completed job' do it 'returns the completed job' do
expect(described_class.completed_jids(%w(123))).to eq(['123']) expect(described_class.completed_jids(%w(123))).to eq(['123'])
end end
......
require 'spec_helper' require 'spec_helper'
describe Gitlab::UserActivities, :redis, lib: true do describe Gitlab::UserActivities, :clean_gitlab_redis_shared_state, lib: true do
let(:now) { Time.now } let(:now) { Time.now }
describe '.record' do describe '.record' do
context 'with no time given' do context 'with no time given' do
it 'uses Time.now and records an activity in Redis' do it 'uses Time.now and records an activity in SharedState' do
Timecop.freeze do Timecop.freeze do
now # eager-load now now # eager-load now
described_class.record(42) described_class.record(42)
end end
Gitlab::Redis.with do |redis| Gitlab::Redis::SharedState.with do |redis|
expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]])
end end
end end
end end
context 'with a time given' do context 'with a time given' do
it 'uses the given time and records an activity in Redis' do it 'uses the given time and records an activity in SharedState' do
described_class.record(42, now) described_class.record(42, now)
Gitlab::Redis.with do |redis| Gitlab::Redis::SharedState.with do |redis|
expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]])
end end
end end
...@@ -31,30 +31,30 @@ describe Gitlab::UserActivities, :redis, lib: true do ...@@ -31,30 +31,30 @@ describe Gitlab::UserActivities, :redis, lib: true do
describe '.delete' do describe '.delete' do
context 'with a single key' do context 'with a single key' do
context 'and key exists' do context 'and key exists' do
it 'removes the pair from Redis' do it 'removes the pair from SharedState' do
described_class.record(42, now) described_class.record(42, now)
Gitlab::Redis.with do |redis| Gitlab::Redis::SharedState.with do |redis|
expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]])
end end
subject.delete(42) subject.delete(42)
Gitlab::Redis.with do |redis| Gitlab::Redis::SharedState.with do |redis|
expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []]) expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []])
end end
end end
end end
context 'and key does not exist' do context 'and key does not exist' do
it 'removes the pair from Redis' do it 'removes the pair from SharedState' do
Gitlab::Redis.with do |redis| Gitlab::Redis::SharedState.with do |redis|
expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []]) expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []])
end end
subject.delete(42) subject.delete(42)
Gitlab::Redis.with do |redis| Gitlab::Redis::SharedState.with do |redis|
expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []]) expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []])
end end
end end
...@@ -63,33 +63,33 @@ describe Gitlab::UserActivities, :redis, lib: true do ...@@ -63,33 +63,33 @@ describe Gitlab::UserActivities, :redis, lib: true do
context 'with multiple keys' do context 'with multiple keys' do
context 'and all keys exist' do context 'and all keys exist' do
it 'removes the pair from Redis' do it 'removes the pair from SharedState' do
described_class.record(41, now) described_class.record(41, now)
described_class.record(42, now) described_class.record(42, now)
Gitlab::Redis.with do |redis| Gitlab::Redis::SharedState.with do |redis|
expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['41', now.to_i.to_s], ['42', now.to_i.to_s]]]) expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['41', now.to_i.to_s], ['42', now.to_i.to_s]]])
end end
subject.delete(41, 42) subject.delete(41, 42)
Gitlab::Redis.with do |redis| Gitlab::Redis::SharedState.with do |redis|
expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []]) expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []])
end end
end end
end end
context 'and some keys does not exist' do context 'and some keys does not exist' do
it 'removes the existing pair from Redis' do it 'removes the existing pair from SharedState' do
described_class.record(42, now) described_class.record(42, now)
Gitlab::Redis.with do |redis| Gitlab::Redis::SharedState.with do |redis|
expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]]) expect(redis.hscan(described_class::KEY, 0)).to eq(['0', [['42', now.to_i.to_s]]])
end end
subject.delete(41, 42) subject.delete(41, 42)
Gitlab::Redis.with do |redis| Gitlab::Redis::SharedState.with do |redis|
expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []]) expect(redis.hscan(described_class::KEY, 0)).to eq(['0', []])
end end
end end
......
...@@ -276,7 +276,7 @@ describe Gitlab::Workhorse, lib: true do ...@@ -276,7 +276,7 @@ describe Gitlab::Workhorse, lib: true do
end end
it 'set and notify' do it 'set and notify' do
expect_any_instance_of(Redis).to receive(:publish) expect_any_instance_of(::Redis).to receive(:publish)
.with(described_class::NOTIFICATION_CHANNEL, "test-key=test-value") .with(described_class::NOTIFICATION_CHANNEL, "test-key=test-value")
subject subject
...@@ -310,7 +310,7 @@ describe Gitlab::Workhorse, lib: true do ...@@ -310,7 +310,7 @@ describe Gitlab::Workhorse, lib: true do
end end
it 'does not notify' do it 'does not notify' do
expect_any_instance_of(Redis).not_to receive(:publish) expect_any_instance_of(::Redis).not_to receive(:publish)
subject subject
end end
......
...@@ -54,7 +54,7 @@ describe MigrateProcessCommitWorkerJobs do ...@@ -54,7 +54,7 @@ describe MigrateProcessCommitWorkerJobs do
end end
end end
describe '#up', :redis do describe '#up', :clean_gitlab_redis_shared_state do
let(:migration) { described_class.new } let(:migration) { described_class.new }
def job_count def job_count
...@@ -172,7 +172,7 @@ describe MigrateProcessCommitWorkerJobs do ...@@ -172,7 +172,7 @@ describe MigrateProcessCommitWorkerJobs do
end end
end end
describe '#down', :redis do describe '#down', :clean_gitlab_redis_shared_state do
let(:migration) { described_class.new } let(:migration) { described_class.new }
def job_count def job_count
......
...@@ -3,13 +3,13 @@ ...@@ -3,13 +3,13 @@
require 'spec_helper' require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20170324160416_migrate_user_activities_to_users_last_activity_on.rb') require Rails.root.join('db', 'post_migrate', '20170324160416_migrate_user_activities_to_users_last_activity_on.rb')
describe MigrateUserActivitiesToUsersLastActivityOn, :redis, :truncate do describe MigrateUserActivitiesToUsersLastActivityOn, :clean_gitlab_redis_shared_state, :truncate do
let(:migration) { described_class.new } let(:migration) { described_class.new }
let!(:user_active_1) { create(:user) } let!(:user_active_1) { create(:user) }
let!(:user_active_2) { create(:user) } let!(:user_active_2) { create(:user) }
def record_activity(user, time) def record_activity(user, time)
Gitlab::Redis.with do |redis| Gitlab::Redis::SharedState.with do |redis|
redis.zadd(described_class::USER_ACTIVITY_SET_KEY, time.to_i, user.username) redis.zadd(described_class::USER_ACTIVITY_SET_KEY, time.to_i, user.username)
end end
end end
......
...@@ -276,14 +276,14 @@ describe Ci::Runner, models: true do ...@@ -276,14 +276,14 @@ describe Ci::Runner, models: true do
it 'sets a new last_update value when it is called the first time' do it 'sets a new last_update value when it is called the first time' do
last_update = runner.ensure_runner_queue_value last_update = runner.ensure_runner_queue_value
expect_value_in_redis.to eq(last_update) expect_value_in_queues.to eq(last_update)
end end
it 'does not change if it is not expired and called again' do it 'does not change if it is not expired and called again' do
last_update = runner.ensure_runner_queue_value last_update = runner.ensure_runner_queue_value
expect(runner.ensure_runner_queue_value).to eq(last_update) expect(runner.ensure_runner_queue_value).to eq(last_update)
expect_value_in_redis.to eq(last_update) expect_value_in_queues.to eq(last_update)
end end
context 'updates runner queue after changing editable value' do context 'updates runner queue after changing editable value' do
...@@ -294,7 +294,7 @@ describe Ci::Runner, models: true do ...@@ -294,7 +294,7 @@ describe Ci::Runner, models: true do
end end
it 'sets a new last_update value' do it 'sets a new last_update value' do
expect_value_in_redis.not_to eq(last_update) expect_value_in_queues.not_to eq(last_update)
end end
end end
...@@ -306,12 +306,12 @@ describe Ci::Runner, models: true do ...@@ -306,12 +306,12 @@ describe Ci::Runner, models: true do
end end
it 'has an old last_update value' do it 'has an old last_update value' do
expect_value_in_redis.to eq(last_update) expect_value_in_queues.to eq(last_update)
end end
end end
def expect_value_in_redis def expect_value_in_queues
Gitlab::Redis.with do |redis| Gitlab::Redis::Queues.with do |redis|
runner_queue_key = runner.send(:runner_queue_key) runner_queue_key = runner.send(:runner_queue_key)
expect(redis.get(runner_queue_key)) expect(redis.get(runner_queue_key))
end end
...@@ -330,7 +330,7 @@ describe Ci::Runner, models: true do ...@@ -330,7 +330,7 @@ describe Ci::Runner, models: true do
end end
it 'cleans up the queue' do it 'cleans up the queue' do
Gitlab::Redis.with do |redis| Gitlab::Redis::Queues.with do |redis|
expect(redis.get(queue_key)).to be_nil expect(redis.get(queue_key)).to be_nil
end end
end end
......
require 'spec_helper' require 'spec_helper'
describe ReactiveCaching, caching: true do describe ReactiveCaching, :use_clean_rails_memory_store_caching do
include ReactiveCachingHelpers include ReactiveCachingHelpers
class CacheTest class CacheTest
......
require 'spec_helper' require 'spec_helper'
describe BambooService, models: true, caching: true do describe BambooService, :use_clean_rails_memory_store_caching, models: true do
include ReactiveCachingHelpers include ReactiveCachingHelpers
let(:bamboo_url) { 'http://gitlab.com/bamboo' } let(:bamboo_url) { 'http://gitlab.com/bamboo' }
......
require 'spec_helper' require 'spec_helper'
describe BuildkiteService, models: true, caching: true do describe BuildkiteService, :use_clean_rails_memory_store_caching, models: true do
include ReactiveCachingHelpers include ReactiveCachingHelpers
let(:project) { create(:empty_project) } let(:project) { create(:empty_project) }
......
require 'spec_helper' require 'spec_helper'
describe DroneCiService, models: true, caching: true do describe DroneCiService, :use_clean_rails_memory_store_caching, models: true do
include ReactiveCachingHelpers include ReactiveCachingHelpers
describe 'associations' do describe 'associations' do
......
require 'spec_helper' require 'spec_helper'
describe KubernetesService, models: true, caching: true do describe KubernetesService, :use_clean_rails_memory_store_caching, models: true do
include KubernetesHelpers include KubernetesHelpers
include ReactiveCachingHelpers include ReactiveCachingHelpers
......
require 'spec_helper' require 'spec_helper'
describe PrometheusService, models: true, caching: true do describe PrometheusService, :use_clean_rails_memory_store_caching, models: true do
include PrometheusHelpers include PrometheusHelpers
include ReactiveCachingHelpers include ReactiveCachingHelpers
......
require 'spec_helper' require 'spec_helper'
describe TeamcityService, models: true, caching: true do describe TeamcityService, :use_clean_rails_memory_store_caching, models: true do
include ReactiveCachingHelpers include ReactiveCachingHelpers
let(:teamcity_url) { 'http://gitlab.com/teamcity' } let(:teamcity_url) { 'http://gitlab.com/teamcity' }
......
...@@ -899,7 +899,7 @@ describe Project, models: true do ...@@ -899,7 +899,7 @@ describe Project, models: true do
end end
end end
describe '.cached_count', caching: true do describe '.cached_count', :use_clean_rails_memory_store_caching do
let(:group) { create(:group, :public) } let(:group) { create(:group, :public) }
let!(:project1) { create(:empty_project, :public, group: group) } let!(:project1) { create(:empty_project, :public, group: group) }
let!(:project2) { create(:empty_project, :public, group: group) } let!(:project2) { create(:empty_project, :public, group: group) }
......
...@@ -561,7 +561,7 @@ describe Repository, models: true do ...@@ -561,7 +561,7 @@ describe Repository, models: true do
end end
end end
describe "#changelog", caching: true do describe "#changelog", :use_clean_rails_memory_store_caching do
it 'accepts changelog' do it 'accepts changelog' do
expect(repository.tree).to receive(:blobs).and_return([TestBlob.new('changelog')]) expect(repository.tree).to receive(:blobs).and_return([TestBlob.new('changelog')])
...@@ -593,7 +593,7 @@ describe Repository, models: true do ...@@ -593,7 +593,7 @@ describe Repository, models: true do
end end
end end
describe "#license_blob", caching: true do describe "#license_blob", :use_clean_rails_memory_store_caching do
before do before do
repository.delete_file( repository.delete_file(
user, 'LICENSE', message: 'Remove LICENSE', branch_name: 'master') user, 'LICENSE', message: 'Remove LICENSE', branch_name: 'master')
...@@ -638,7 +638,7 @@ describe Repository, models: true do ...@@ -638,7 +638,7 @@ describe Repository, models: true do
end end
end end
describe '#license_key', caching: true do describe '#license_key', :use_clean_rails_memory_store_caching do
before do before do
repository.delete_file(user, 'LICENSE', repository.delete_file(user, 'LICENSE',
message: 'Remove LICENSE', branch_name: 'master') message: 'Remove LICENSE', branch_name: 'master')
...@@ -703,7 +703,7 @@ describe Repository, models: true do ...@@ -703,7 +703,7 @@ describe Repository, models: true do
end end
end end
describe "#gitlab_ci_yml", caching: true do describe "#gitlab_ci_yml", :use_clean_rails_memory_store_caching do
it 'returns valid file' do it 'returns valid file' do
files = [TestBlob.new('file'), TestBlob.new('.gitlab-ci.yml'), TestBlob.new('copying')] files = [TestBlob.new('file'), TestBlob.new('.gitlab-ci.yml'), TestBlob.new('copying')]
expect(repository.tree).to receive(:blobs).and_return(files) expect(repository.tree).to receive(:blobs).and_return(files)
...@@ -1611,7 +1611,7 @@ describe Repository, models: true do ...@@ -1611,7 +1611,7 @@ describe Repository, models: true do
end end
end end
describe '#contribution_guide', caching: true do describe '#contribution_guide', :use_clean_rails_memory_store_caching do
it 'returns and caches the output' do it 'returns and caches the output' do
expect(repository).to receive(:file_on_head) expect(repository).to receive(:file_on_head)
.with(:contributing) .with(:contributing)
...@@ -1625,7 +1625,7 @@ describe Repository, models: true do ...@@ -1625,7 +1625,7 @@ describe Repository, models: true do
end end
end end
describe '#gitignore', caching: true do describe '#gitignore', :use_clean_rails_memory_store_caching do
it 'returns and caches the output' do it 'returns and caches the output' do
expect(repository).to receive(:file_on_head) expect(repository).to receive(:file_on_head)
.with(:gitignore) .with(:gitignore)
...@@ -1638,7 +1638,7 @@ describe Repository, models: true do ...@@ -1638,7 +1638,7 @@ describe Repository, models: true do
end end
end end
describe '#koding_yml', caching: true do describe '#koding_yml', :use_clean_rails_memory_store_caching do
it 'returns and caches the output' do it 'returns and caches the output' do
expect(repository).to receive(:file_on_head) expect(repository).to receive(:file_on_head)
.with(:koding) .with(:koding)
...@@ -1651,7 +1651,7 @@ describe Repository, models: true do ...@@ -1651,7 +1651,7 @@ describe Repository, models: true do
end end
end end
describe '#readme', caching: true do describe '#readme', :use_clean_rails_memory_store_caching do
context 'with a non-existing repository' do context 'with a non-existing repository' do
it 'returns nil' do it 'returns nil' do
allow(repository).to receive(:tree).with(:head).and_return(nil) allow(repository).to receive(:tree).with(:head).and_return(nil)
...@@ -1822,7 +1822,7 @@ describe Repository, models: true do ...@@ -1822,7 +1822,7 @@ describe Repository, models: true do
end end
end end
describe '#cache_method_output', caching: true do describe '#cache_method_output', :use_clean_rails_memory_store_caching do
context 'with a non-existing repository' do context 'with a non-existing repository' do
let(:value) do let(:value) do
repository.cache_method_output(:cats, fallback: 10) do repository.cache_method_output(:cats, fallback: 10) do
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment