Commit 4484de15 authored by Sean McGivern's avatar Sean McGivern

Move SidekiqConfig methods used by CLI to own module

These methods can't use any Rails dependencies, which also means the
SidekiqConfig module definition - i.e. constants - can't use Rails
dependencies either.

Move those methods to their own module to make this clearer. Also switch
to defining methods on the singleton class in both modules to make
private methods more convenient.
parent 4ac2a641
......@@ -4,7 +4,7 @@
require 'optparse'
require_relative '../../lib/gitlab'
require_relative '../../lib/gitlab/utils'
require_relative '../../lib/gitlab/sidekiq_config'
require_relative '../../lib/gitlab/sidekiq_config/cli_methods'
require_relative '../lib/gitlab/sidekiq_cluster'
require_relative '../lib/gitlab/sidekiq_cluster/cli'
......
......@@ -42,10 +42,10 @@ module Gitlab
queue_groups = SidekiqCluster.parse_queues(argv)
all_queues = SidekiqConfig.worker_queues(@rails_path)
all_queues = SidekiqConfig::CliMethods.worker_queues(@rails_path)
queue_groups.map! do |queues|
SidekiqConfig.expand_queues(queues, all_queues)
SidekiqConfig::CliMethods.expand_queues(queues, all_queues)
end
if @negate_queues
......
......@@ -32,7 +32,7 @@ describe Gitlab::SidekiqCluster::CLI do
context 'with --negate flag' do
it 'starts Sidekiq workers for all queues in all_queues.yml except the ones in argv' do
expect(Gitlab::SidekiqConfig).to receive(:worker_queues).and_return(['baz'])
expect(Gitlab::SidekiqConfig::CliMethods).to receive(:worker_queues).and_return(['baz'])
expect(Gitlab::SidekiqCluster).to receive(:start)
.with([['baz']], default_options)
.and_return([])
......@@ -43,7 +43,7 @@ describe Gitlab::SidekiqCluster::CLI do
context 'with --max-concurrency flag' do
it 'starts Sidekiq workers for specified queues with a max concurrency' do
expect(Gitlab::SidekiqConfig).to receive(:worker_queues).and_return(%w(foo bar baz))
expect(Gitlab::SidekiqConfig::CliMethods).to receive(:worker_queues).and_return(%w(foo bar baz))
expect(Gitlab::SidekiqCluster).to receive(:start)
.with([%w(foo bar baz), %w(solo)], default_options.merge(max_concurrency: 2))
.and_return([])
......@@ -54,7 +54,7 @@ describe Gitlab::SidekiqCluster::CLI do
context 'queue namespace expansion' do
it 'starts Sidekiq workers for all queues in all_queues.yml with a namespace in argv' do
expect(Gitlab::SidekiqConfig).to receive(:worker_queues).and_return(['cronjob:foo', 'cronjob:bar'])
expect(Gitlab::SidekiqConfig::CliMethods).to receive(:worker_queues).and_return(['cronjob:foo', 'cronjob:bar'])
expect(Gitlab::SidekiqCluster).to receive(:start)
.with([['cronjob', 'cronjob:foo', 'cronjob:bar']], default_options)
.and_return([])
......
# frozen_string_literal: true
require 'yaml'
require 'set'
module Gitlab
module SidekiqConfig
QUEUE_CONFIG_PATHS = begin
result = %w[app/workers/all_queues.yml]
result << 'ee/app/workers/all_queues.yml' if Gitlab.ee?
result
end.freeze
class << self
include Gitlab::SidekiqConfig::CliMethods
# This method is called by `ee/bin/sidekiq-cluster` in EE, which runs outside
# of bundler/Rails context, so we cannot use any gem or Rails methods.
def self.worker_queues(rails_path = Rails.root.to_s)
@worker_queues ||= {}
@worker_queues[rails_path] ||= QUEUE_CONFIG_PATHS.flat_map do |path|
full_path = File.join(rails_path, path)
File.exist?(full_path) ? YAML.load_file(full_path) : []
def redis_queues
# Not memoized, because this can change during the life of the application
Sidekiq::Queue.all.map(&:name)
end
end
# This method is called by `ee/bin/sidekiq-cluster` in EE, which runs outside
# of bundler/Rails context, so we cannot use any gem or Rails methods.
def self.expand_queues(queues, all_queues = self.worker_queues)
return [] if queues.empty?
queues_set = all_queues.to_set
queues.flat_map do |queue|
[queue, *queues_set.grep(/\A#{queue}:/)]
def config_queues
@config_queues ||= begin
config = YAML.load_file(Rails.root.join('config/sidekiq_queues.yml'))
config[:queues].map(&:first)
end
end
end
def self.redis_queues
# Not memoized, because this can change during the life of the application
Sidekiq::Queue.all.map(&:name)
end
def cron_workers
@cron_workers ||= Settings.cron_jobs.map { |job_name, options| options['job_class'].constantize }
end
def self.config_queues
@config_queues ||= begin
config = YAML.load_file(Rails.root.join('config/sidekiq_queues.yml'))
config[:queues].map(&:first)
def workers
@workers ||= begin
result = find_workers(Rails.root.join('app', 'workers'))
result.concat(find_workers(Rails.root.join('ee', 'app', 'workers'))) if Gitlab.ee?
result
end
end
end
def self.cron_workers
@cron_workers ||= Settings.cron_jobs.map { |job_name, options| options['job_class'].constantize }
end
private
def self.workers
@workers ||= begin
result = find_workers(Rails.root.join('app', 'workers'))
result.concat(find_workers(Rails.root.join('ee', 'app', 'workers'))) if Gitlab.ee?
result
end
end
def find_workers(root)
concerns = root.join('concerns').to_s
def self.find_workers(root)
concerns = root.join('concerns').to_s
workers = Dir[root.join('**', '*.rb')]
.reject { |path| path.start_with?(concerns) }
workers = Dir[root.join('**', '*.rb')]
.reject { |path| path.start_with?(concerns) }
workers.map! do |path|
ns = Pathname.new(path).relative_path_from(root).to_s.gsub('.rb', '')
workers.map! do |path|
ns = Pathname.new(path).relative_path_from(root).to_s.gsub('.rb', '')
ns.camelize.constantize
end
ns.camelize.constantize
# Skip things that aren't workers
workers.select { |w| w < Sidekiq::Worker }
end
# Skip things that aren't workers
workers.select { |w| w < Sidekiq::Worker }
end
end
end
# frozen_string_literal: true
require 'yaml'
require 'set'
# These methods are called by `sidekiq-cluster`, which runs outside of
# the bundler/Rails context, so we cannot use any gem or Rails methods.
module Gitlab
module SidekiqConfig
module CliMethods
# The methods in this module are used as module methods
# rubocop:disable Gitlab/ModuleWithInstanceVariables
extend self
QUEUE_CONFIG_PATHS = begin
result = %w[app/workers/all_queues.yml]
result << 'ee/app/workers/all_queues.yml' if Gitlab.ee?
result
end.freeze
def worker_queues(rails_path = Rails.root.to_s)
@worker_queues ||= {}
@worker_queues[rails_path] ||= QUEUE_CONFIG_PATHS.flat_map do |path|
full_path = File.join(rails_path, path)
File.exist?(full_path) ? YAML.load_file(full_path) : []
end
end
def expand_queues(queues, all_queues = self.worker_queues)
return [] if queues.empty?
queues_set = all_queues.to_set
queues.flat_map do |queue|
[queue, *queues_set.grep(/\A#{queue}:/)]
end
end
# rubocop:enable Gitlab/ModuleWithInstanceVariables
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment