Commit 4484de15 authored by Sean McGivern's avatar Sean McGivern

Move SidekiqConfig methods used by CLI to own module

These methods can't use any Rails dependencies, which also means the
SidekiqConfig module definition - i.e. constants - can't use Rails
dependencies either.

Move those methods to their own module to make this clearer. Also switch
to defining methods on the singleton class in both modules to make
private methods more convenient.
parent 4ac2a641
...@@ -4,7 +4,7 @@ ...@@ -4,7 +4,7 @@
require 'optparse' require 'optparse'
require_relative '../../lib/gitlab' require_relative '../../lib/gitlab'
require_relative '../../lib/gitlab/utils' require_relative '../../lib/gitlab/utils'
require_relative '../../lib/gitlab/sidekiq_config' require_relative '../../lib/gitlab/sidekiq_config/cli_methods'
require_relative '../lib/gitlab/sidekiq_cluster' require_relative '../lib/gitlab/sidekiq_cluster'
require_relative '../lib/gitlab/sidekiq_cluster/cli' require_relative '../lib/gitlab/sidekiq_cluster/cli'
......
...@@ -42,10 +42,10 @@ module Gitlab ...@@ -42,10 +42,10 @@ module Gitlab
queue_groups = SidekiqCluster.parse_queues(argv) queue_groups = SidekiqCluster.parse_queues(argv)
all_queues = SidekiqConfig.worker_queues(@rails_path) all_queues = SidekiqConfig::CliMethods.worker_queues(@rails_path)
queue_groups.map! do |queues| queue_groups.map! do |queues|
SidekiqConfig.expand_queues(queues, all_queues) SidekiqConfig::CliMethods.expand_queues(queues, all_queues)
end end
if @negate_queues if @negate_queues
......
...@@ -32,7 +32,7 @@ describe Gitlab::SidekiqCluster::CLI do ...@@ -32,7 +32,7 @@ describe Gitlab::SidekiqCluster::CLI do
context 'with --negate flag' do context 'with --negate flag' do
it 'starts Sidekiq workers for all queues in all_queues.yml except the ones in argv' do it 'starts Sidekiq workers for all queues in all_queues.yml except the ones in argv' do
expect(Gitlab::SidekiqConfig).to receive(:worker_queues).and_return(['baz']) expect(Gitlab::SidekiqConfig::CliMethods).to receive(:worker_queues).and_return(['baz'])
expect(Gitlab::SidekiqCluster).to receive(:start) expect(Gitlab::SidekiqCluster).to receive(:start)
.with([['baz']], default_options) .with([['baz']], default_options)
.and_return([]) .and_return([])
...@@ -43,7 +43,7 @@ describe Gitlab::SidekiqCluster::CLI do ...@@ -43,7 +43,7 @@ describe Gitlab::SidekiqCluster::CLI do
context 'with --max-concurrency flag' do context 'with --max-concurrency flag' do
it 'starts Sidekiq workers for specified queues with a max concurrency' do it 'starts Sidekiq workers for specified queues with a max concurrency' do
expect(Gitlab::SidekiqConfig).to receive(:worker_queues).and_return(%w(foo bar baz)) expect(Gitlab::SidekiqConfig::CliMethods).to receive(:worker_queues).and_return(%w(foo bar baz))
expect(Gitlab::SidekiqCluster).to receive(:start) expect(Gitlab::SidekiqCluster).to receive(:start)
.with([%w(foo bar baz), %w(solo)], default_options.merge(max_concurrency: 2)) .with([%w(foo bar baz), %w(solo)], default_options.merge(max_concurrency: 2))
.and_return([]) .and_return([])
...@@ -54,7 +54,7 @@ describe Gitlab::SidekiqCluster::CLI do ...@@ -54,7 +54,7 @@ describe Gitlab::SidekiqCluster::CLI do
context 'queue namespace expansion' do context 'queue namespace expansion' do
it 'starts Sidekiq workers for all queues in all_queues.yml with a namespace in argv' do it 'starts Sidekiq workers for all queues in all_queues.yml with a namespace in argv' do
expect(Gitlab::SidekiqConfig).to receive(:worker_queues).and_return(['cronjob:foo', 'cronjob:bar']) expect(Gitlab::SidekiqConfig::CliMethods).to receive(:worker_queues).and_return(['cronjob:foo', 'cronjob:bar'])
expect(Gitlab::SidekiqCluster).to receive(:start) expect(Gitlab::SidekiqCluster).to receive(:start)
.with([['cronjob', 'cronjob:foo', 'cronjob:bar']], default_options) .with([['cronjob', 'cronjob:foo', 'cronjob:bar']], default_options)
.and_return([]) .and_return([])
......
# frozen_string_literal: true # frozen_string_literal: true
require 'yaml' require 'yaml'
require 'set'
module Gitlab module Gitlab
module SidekiqConfig module SidekiqConfig
QUEUE_CONFIG_PATHS = begin class << self
result = %w[app/workers/all_queues.yml] include Gitlab::SidekiqConfig::CliMethods
result << 'ee/app/workers/all_queues.yml' if Gitlab.ee?
result
end.freeze
# This method is called by `ee/bin/sidekiq-cluster` in EE, which runs outside def redis_queues
# of bundler/Rails context, so we cannot use any gem or Rails methods. # Not memoized, because this can change during the life of the application
def self.worker_queues(rails_path = Rails.root.to_s) Sidekiq::Queue.all.map(&:name)
@worker_queues ||= {}
@worker_queues[rails_path] ||= QUEUE_CONFIG_PATHS.flat_map do |path|
full_path = File.join(rails_path, path)
File.exist?(full_path) ? YAML.load_file(full_path) : []
end end
end
# This method is called by `ee/bin/sidekiq-cluster` in EE, which runs outside
# of bundler/Rails context, so we cannot use any gem or Rails methods.
def self.expand_queues(queues, all_queues = self.worker_queues)
return [] if queues.empty?
queues_set = all_queues.to_set def config_queues
@config_queues ||= begin
queues.flat_map do |queue| config = YAML.load_file(Rails.root.join('config/sidekiq_queues.yml'))
[queue, *queues_set.grep(/\A#{queue}:/)] config[:queues].map(&:first)
end
end end
end
def self.redis_queues def cron_workers
# Not memoized, because this can change during the life of the application @cron_workers ||= Settings.cron_jobs.map { |job_name, options| options['job_class'].constantize }
Sidekiq::Queue.all.map(&:name) end
end
def self.config_queues def workers
@config_queues ||= begin @workers ||= begin
config = YAML.load_file(Rails.root.join('config/sidekiq_queues.yml')) result = find_workers(Rails.root.join('app', 'workers'))
config[:queues].map(&:first) result.concat(find_workers(Rails.root.join('ee', 'app', 'workers'))) if Gitlab.ee?
result
end
end end
end
def self.cron_workers private
@cron_workers ||= Settings.cron_jobs.map { |job_name, options| options['job_class'].constantize }
end
def self.workers def find_workers(root)
@workers ||= begin concerns = root.join('concerns').to_s
result = find_workers(Rails.root.join('app', 'workers'))
result.concat(find_workers(Rails.root.join('ee', 'app', 'workers'))) if Gitlab.ee?
result
end
end
def self.find_workers(root) workers = Dir[root.join('**', '*.rb')]
concerns = root.join('concerns').to_s .reject { |path| path.start_with?(concerns) }
workers = Dir[root.join('**', '*.rb')] workers.map! do |path|
.reject { |path| path.start_with?(concerns) } ns = Pathname.new(path).relative_path_from(root).to_s.gsub('.rb', '')
workers.map! do |path| ns.camelize.constantize
ns = Pathname.new(path).relative_path_from(root).to_s.gsub('.rb', '') end
ns.camelize.constantize # Skip things that aren't workers
workers.select { |w| w < Sidekiq::Worker }
end end
# Skip things that aren't workers
workers.select { |w| w < Sidekiq::Worker }
end end
end end
end end
# frozen_string_literal: true
require 'yaml'
require 'set'
# These methods are called by `sidekiq-cluster`, which runs outside of
# the bundler/Rails context, so we cannot use any gem or Rails methods.
module Gitlab
module SidekiqConfig
module CliMethods
# The methods in this module are used as module methods
# rubocop:disable Gitlab/ModuleWithInstanceVariables
extend self
QUEUE_CONFIG_PATHS = begin
result = %w[app/workers/all_queues.yml]
result << 'ee/app/workers/all_queues.yml' if Gitlab.ee?
result
end.freeze
def worker_queues(rails_path = Rails.root.to_s)
@worker_queues ||= {}
@worker_queues[rails_path] ||= QUEUE_CONFIG_PATHS.flat_map do |path|
full_path = File.join(rails_path, path)
File.exist?(full_path) ? YAML.load_file(full_path) : []
end
end
def expand_queues(queues, all_queues = self.worker_queues)
return [] if queues.empty?
queues_set = all_queues.to_set
queues.flat_map do |queue|
[queue, *queues_set.grep(/\A#{queue}:/)]
end
end
# rubocop:enable Gitlab/ModuleWithInstanceVariables
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment