Commit 334e93b1 authored by Dylan Griffith's avatar Dylan Griffith

Merge branch 'add-tags-to-queue-attributes' into 'master'

Add tags to queue selector attributes

See merge request gitlab-org/gitlab!32651
parents 7f1aa059 5bc653ce
This diff is collapsed.
......@@ -111,6 +111,14 @@ module WorkerAttributes
1
end
def tags(*values)
worker_attributes[:tags] = values
end
def get_tags
Array(worker_attributes[:tags])
end
protected
# Returns a worker attribute declared on this class or its parent class.
......
---
title: Add tags to experimental queue selector attributes
merge_request: 32651
author:
type: added
......@@ -126,6 +126,8 @@ in a more general way using the following components:
### Available attributes
- [Introduced](https://gitlab.com/gitlab-com/gl-infra/scalability/-/issues/261) in GitLab 13.1, `tags`.
From the [list of all available
attributes](https://gitlab.com/gitlab-org/gitlab/-/blob/master/app/workers/all_queues.yml),
`experimental_queue_selector` allows selecting of queues by the
......@@ -144,14 +146,21 @@ following attributes:
- `name` - the queue name. The other attributes are typically more useful as
they are more general, but this is available in case a particular queue needs
to be selected.
- `resource_boundary` - if the worker is bound by `cpu`, `memory`, or
- `resource_boundary` - if the queue is bound by `cpu`, `memory`, or
`unknown`. For example, the `project_export` queue is memory bound as it has
to load data in memory before saving it for export.
- `tags` - short-lived annotations for queues. These are expected to frequently
change from release to release, and may be removed entirely.
`has_external_dependencies` is a boolean attribute: only the exact
string `true` is considered true, and everything else is considered
false.
`tags` is a set, which means that `=` checks for intersecting sets, and
`!=` checks for disjoint sets. For example, `tags=a,b` selects queues
that have tags `a`, `b`, or both. `tags!=a,b` selects queues that have
neither of those tags.
### Available operators
`experimental_queue_selector` supports the following operators, listed
......
This diff is collapsed.
......@@ -14,8 +14,8 @@ module Gitlab
].compact.freeze
DEFAULT_WORKERS = [
DummyWorker.new('default', weight: 1),
DummyWorker.new('mailers', weight: 2)
DummyWorker.new('default', weight: 1, tags: []),
DummyWorker.new('mailers', weight: 2, tags: [])
].map { |worker| Gitlab::SidekiqConfig::Worker.new(worker, ee: false) }.freeze
class << self
......
......@@ -28,6 +28,7 @@ module Gitlab
has_external_dependencies: lambda { |value| value == 'true' },
name: :to_s,
resource_boundary: :to_sym,
tags: :to_sym,
urgency: :to_sym
}.freeze
......@@ -117,7 +118,11 @@ module Gitlab
raise UnknownPredicate.new("Unknown predicate: #{lhs}") unless values_block
lambda { |queue| values.map(&values_block).include?(queue[lhs.to_sym]) }
lambda do |queue|
comparator = Array(queue[lhs.to_sym]).to_set
values.map(&values_block).to_set.intersect?(comparator)
end
end
end
end
......
......@@ -12,7 +12,8 @@ module Gitlab
urgency: :get_urgency,
resource_boundary: :get_worker_resource_boundary,
idempotent: :idempotent?,
weight: :get_weight
weight: :get_weight,
tags: :get_tags
}.freeze
def initialize(queue, attributes = {})
......
......@@ -6,7 +6,7 @@ module Gitlab
include Comparable
attr_reader :klass
delegate :feature_category_not_owned?, :get_feature_category,
delegate :feature_category_not_owned?, :get_feature_category, :get_tags,
:get_urgency, :get_weight, :get_worker_resource_boundary,
:idempotent?, :queue, :queue_namespace,
:worker_has_external_dependencies?,
......@@ -52,7 +52,8 @@ module Gitlab
urgency: get_urgency,
resource_boundary: get_worker_resource_boundary,
weight: get_weight,
idempotent: idempotent?
idempotent: idempotent?,
tags: get_tags
}
end
......
......@@ -117,28 +117,32 @@ describe Gitlab::SidekiqConfig::CliMethods do
feature_category: :category_a,
has_external_dependencies: false,
urgency: :low,
resource_boundary: :cpu
resource_boundary: :cpu,
tags: [:no_disk_io, :git_access]
},
{
name: 'a:2',
feature_category: :category_a,
has_external_dependencies: false,
urgency: :high,
resource_boundary: :none
resource_boundary: :none,
tags: [:git_access]
},
{
name: 'b',
feature_category: :category_b,
has_external_dependencies: true,
urgency: :high,
resource_boundary: :memory
resource_boundary: :memory,
tags: [:no_disk_io]
},
{
name: 'c',
feature_category: :category_c,
has_external_dependencies: false,
urgency: :throttled,
resource_boundary: :memory
resource_boundary: :memory,
tags: []
}
]
end
......@@ -177,6 +181,18 @@ describe Gitlab::SidekiqConfig::CliMethods do
'resource_boundary=memory|resource_boundary=cpu' | %w(a b c)
'resource_boundary!=memory,cpu' | %w(a:2)
# tags
'tags=no_disk_io' | %w(a b)
'tags=no_disk_io,git_access' | %w(a a:2 b)
'tags=no_disk_io|tags=git_access' | %w(a a:2 b)
'tags=no_disk_io&tags=git_access' | %w(a)
'tags!=no_disk_io' | %w(a:2 c)
'tags!=no_disk_io,git_access' | %w(c)
'tags=unknown_tag' | []
'tags!=no_disk_io' | %w(a:2 c)
'tags!=no_disk_io,git_access' | %w(c)
'tags!=unknown_tag' | %w(a a:2 b c)
# combinations
'feature_category=category_a&urgency=high' | %w(a:2)
'feature_category=category_a&urgency=high|feature_category=category_c' | %w(a:2 c)
......
......@@ -13,7 +13,8 @@ describe Gitlab::SidekiqConfig::Worker do
get_worker_resource_boundary: attributes[:resource_boundary],
get_urgency: attributes[:urgency],
worker_has_external_dependencies?: attributes[:has_external_dependencies],
idempotent?: attributes[:idempotent]
idempotent?: attributes[:idempotent],
get_tags: attributes[:tags]
)
described_class.new(inner_worker, ee: false)
......@@ -91,7 +92,8 @@ describe Gitlab::SidekiqConfig::Worker do
urgency: :low,
resource_boundary: :memory,
weight: 2,
idempotent: true
idempotent: true,
tags: []
}
attributes_b = {
......@@ -100,7 +102,8 @@ describe Gitlab::SidekiqConfig::Worker do
urgency: :high,
resource_boundary: :unknown,
weight: 3,
idempotent: false
idempotent: false,
tags: [:no_disk_io]
}
worker_a = create_worker(queue: 'a', **attributes_a)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment