Commit e22e4a77 authored by Grzegorz Bizon's avatar Grzegorz Bizon

De-normalize tags used to retrieve builds queue

parent e690cc79
...@@ -12,7 +12,8 @@ module Ci ...@@ -12,7 +12,8 @@ module Ci
scope :queued_before, ->(time) { where(arel_table[:created_at].lt(time)) } scope :queued_before, ->(time) { where(arel_table[:created_at].lt(time)) }
scope :with_instance_runners, -> { where(instance_runners_enabled: true) } scope :with_instance_runners, -> { where(instance_runners_enabled: true) }
def self.upsert_from_build!(build) class << self
def upsert_from_build!(build)
entry = self.new(args_from_build(build)) entry = self.new(args_from_build(build))
entry.validate! entry.validate!
...@@ -20,7 +21,9 @@ module Ci ...@@ -20,7 +21,9 @@ module Ci
self.upsert(entry.attributes.compact, returning: %w[build_id], unique_by: :build_id) self.upsert(entry.attributes.compact, returning: %w[build_id], unique_by: :build_id)
end end
def self.args_from_build(build) private
def args_from_build(build)
args = { args = {
build: build, build: build,
project: build.project, project: build.project,
...@@ -28,35 +31,36 @@ module Ci ...@@ -28,35 +31,36 @@ module Ci
namespace: build.project.namespace namespace: build.project.namespace
} }
if Feature.enabled?(:ci_pending_builds_maintain_tags_data, type: :development, default_enabled: :yaml)
args.store(:tag_ids, build.tags_ids)
end
if Feature.enabled?(:ci_pending_builds_maintain_shared_runners_data, type: :development, default_enabled: :yaml) if Feature.enabled?(:ci_pending_builds_maintain_shared_runners_data, type: :development, default_enabled: :yaml)
args.merge(instance_runners_enabled: shareable?(build)) args.store(:instance_runners_enabled, shareable?(build))
else
args
end end
args
end end
private_class_method :args_from_build
def self.shareable?(build) def shareable?(build)
shared_runner_enabled?(build) && shared_runner_enabled?(build) &&
builds_access_level?(build) && builds_access_level?(build) &&
project_not_removed?(build) project_not_removed?(build)
end end
private_class_method :shareable?
def self.shared_runner_enabled?(build) def shared_runner_enabled?(build)
build.project.shared_runners.exists? build.project.shared_runners.exists?
end end
private_class_method :shared_runner_enabled?
def self.project_not_removed?(build) def project_not_removed?(build)
!build.project.pending_delete? !build.project.pending_delete?
end end
private_class_method :project_not_removed?
def self.builds_access_level?(build) def builds_access_level?(build)
build.project.project_feature.builds_access_level.nil? || build.project.project_feature.builds_access_level > 0 build.project.project_feature.builds_access_level.nil? ||
build.project.project_feature.builds_access_level > 0
end
end end
private_class_method :builds_access_level?
end end
end end
......
...@@ -3,6 +3,10 @@ ...@@ -3,6 +3,10 @@
module TaggableQueries module TaggableQueries
extend ActiveSupport::Concern extend ActiveSupport::Concern
MAX_TAGS_IDS = 50
TooManyTagsError = Class.new(StandardError)
class_methods do class_methods do
# context is a name `acts_as_taggable context` # context is a name `acts_as_taggable context`
def arel_tag_names_array(context = :tags) def arel_tag_names_array(context = :tags)
...@@ -34,4 +38,10 @@ module TaggableQueries ...@@ -34,4 +38,10 @@ module TaggableQueries
where("EXISTS (?)", matcher) where("EXISTS (?)", matcher)
end end
end end
def tags_ids
tags.limit(MAX_TAGS_IDS).order('id ASC').pluck(:id).tap do |ids|
raise TooManyTagsError if ids.size >= MAX_TAGS_IDS
end
end
end end
...@@ -17,12 +17,20 @@ module Ci ...@@ -17,12 +17,20 @@ module Ci
end end
def builds_matching_tag_ids(relation, ids) def builds_matching_tag_ids(relation, ids)
if ::Feature.enabled?(:ci_queueing_denormalize_tags_information, runner, default_enabled: :yaml)
relation.where('tag_ids <@ ARRAY[?]::int[]', runner.tags_ids)
else
relation.merge(CommitStatus.matches_tag_ids(ids, table: 'ci_pending_builds', column: 'build_id')) relation.merge(CommitStatus.matches_tag_ids(ids, table: 'ci_pending_builds', column: 'build_id'))
end end
end
def builds_with_any_tags(relation) def builds_with_any_tags(relation)
if ::Feature.enabled?(:ci_queueing_denormalize_tags_information, runner, default_enabled: :yaml)
relation.where('cardinality(tag_ids) > 0')
else
relation.merge(CommitStatus.with_any_tags(table: 'ci_pending_builds', column: 'build_id')) relation.merge(CommitStatus.with_any_tags(table: 'ci_pending_builds', column: 'build_id'))
end end
end
def order(relation) def order(relation)
relation.order('build_id ASC') relation.order('build_id ASC')
......
---
name: ci_pending_builds_maintain_tags_data
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/65648
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/338363
milestone: '14.2'
type: development
group: group::pipeline execution
default_enabled: false
---
name: ci_queueing_denormalize_tags_information
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/65648
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/338366
milestone: '14.1'
type: development
group: group::pipeline execution
default_enabled: false
# frozen_string_literal: true
class AddTagsArrayToCiPendingBuilds < ActiveRecord::Migration[6.1]
include Gitlab::Database::MigrationHelpers
def up
with_lock_retries do
add_column :ci_pending_builds, :tag_ids, :integer, array: true, default: []
end
end
def down
with_lock_retries do
remove_column :ci_pending_builds, :tag_ids
end
end
end
# frozen_string_literal: true
class AddTagIdsIndexToCiPendingBuild < ActiveRecord::Migration[6.1]
include Gitlab::Database::MigrationHelpers
disable_ddl_transaction!
INDEX_NAME = 'index_ci_pending_builds_on_tag_ids'
def up
add_concurrent_index(:ci_pending_builds, :tag_ids, name: INDEX_NAME, where: 'cardinality(tag_ids) > 0')
end
def down
remove_concurrent_index_by_name(:ci_pending_builds, name: INDEX_NAME)
end
end
837b9a56114c63064379cf276a3c7e2bbe845af9022a542c4fcec94a25062017
\ No newline at end of file
88ca485c8513df96b1f1aec1585c385223dc53889e547db42b509b0cd1bea9b7
\ No newline at end of file
...@@ -10903,7 +10903,8 @@ CREATE TABLE ci_pending_builds ( ...@@ -10903,7 +10903,8 @@ CREATE TABLE ci_pending_builds (
protected boolean DEFAULT false NOT NULL, protected boolean DEFAULT false NOT NULL,
instance_runners_enabled boolean DEFAULT false NOT NULL, instance_runners_enabled boolean DEFAULT false NOT NULL,
namespace_id bigint, namespace_id bigint,
minutes_exceeded boolean DEFAULT false NOT NULL minutes_exceeded boolean DEFAULT false NOT NULL,
tag_ids integer[] DEFAULT '{}'::integer[]
); );
CREATE SEQUENCE ci_pending_builds_id_seq CREATE SEQUENCE ci_pending_builds_id_seq
...@@ -23439,6 +23440,8 @@ CREATE INDEX index_ci_pending_builds_on_namespace_id ON ci_pending_builds USING ...@@ -23439,6 +23440,8 @@ CREATE INDEX index_ci_pending_builds_on_namespace_id ON ci_pending_builds USING
CREATE INDEX index_ci_pending_builds_on_project_id ON ci_pending_builds USING btree (project_id); CREATE INDEX index_ci_pending_builds_on_project_id ON ci_pending_builds USING btree (project_id);
CREATE INDEX index_ci_pending_builds_on_tag_ids ON ci_pending_builds USING btree (tag_ids) WHERE (cardinality(tag_ids) > 0);
CREATE INDEX index_ci_pipeline_artifacts_failed_verification ON ci_pipeline_artifacts USING btree (verification_retry_at NULLS FIRST) WHERE (verification_state = 3); CREATE INDEX index_ci_pipeline_artifacts_failed_verification ON ci_pipeline_artifacts USING btree (verification_retry_at NULLS FIRST) WHERE (verification_state = 3);
CREATE INDEX index_ci_pipeline_artifacts_needs_verification ON ci_pipeline_artifacts USING btree (verification_state) WHERE ((verification_state = 0) OR (verification_state = 3)); CREATE INDEX index_ci_pipeline_artifacts_needs_verification ON ci_pipeline_artifacts USING btree (verification_state) WHERE ((verification_state = 0) OR (verification_state = 3));
...@@ -113,5 +113,31 @@ RSpec.describe Ci::PendingBuild do ...@@ -113,5 +113,31 @@ RSpec.describe Ci::PendingBuild do
end end
end end
end end
context 'when build has tags' do
let!(:build) { create(:ci_build, :tags) }
subject(:ci_pending_build) { described_class.last }
context 'when ci_pending_builds_maintain_tags_data is enabled' do
it 'sets tag_ids' do
described_class.upsert_from_build!(build)
expect(ci_pending_build.tag_ids).to eq(build.tags_ids)
end
end
context 'when ci_pending_builds_maintain_tags_data is disabled' do
before do
stub_feature_flags(ci_pending_builds_maintain_tags_data: false)
end
it 'does not set tag_ids' do
described_class.upsert_from_build!(build)
expect(ci_pending_build.tag_ids).to be_empty
end
end
end
end end
end end
...@@ -739,6 +739,22 @@ module Ci ...@@ -739,6 +739,22 @@ module Ci
include_examples 'handles runner assignment' include_examples 'handles runner assignment'
end end
context 'with ci_queueing_denormalize_tags_information enabled' do
before do
stub_feature_flags(ci_queueing_denormalize_tags_information: true)
end
include_examples 'handles runner assignment'
end
context 'with ci_queueing_denormalize_tags_information disabled' do
before do
stub_feature_flags(ci_queueing_denormalize_tags_information: false)
end
include_examples 'handles runner assignment'
end
end end
context 'when not using pending builds table' do context 'when not using pending builds table' do
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment