Introduce a limit to the runners builds queue depth

This commit reduces the maximum builds queue depth that can be iterated
to find a build that can be picked. This should result in making this
service much more efficient and should reduce the strain on the
database.

app/services/ci/register_job_service.rb

@@ -10,6 +10,8 @@ module Ci
 
     Result = Struct.new(:build, :build_json, :valid?)
 
+    MAX_QUEUE_DEPTH = 50
+
     def initialize(runner)
       @runner = runner
       @metrics = ::Gitlab::Ci::Queue::Metrics.new(runner)
@@ -33,6 +35,14 @@ module Ci
         depth += 1
         @metrics.increment_queue_operation(:queue_iteration)
 
+        if depth > max_queue_depth
+          @metrics.increment_queue_operation(:queue_depth_limit)
+
+          valid = false
+
+          break
+        end
+
         # We read builds from replicas
         # It is likely that some other concurrent connection is processing
         # a given build at a given moment. To avoid an expensive compute
@@ -44,6 +54,7 @@ module Ci
           # - our queue is not complete as some resources are locked temporarily
           # - we need to re-process it again to ensure that all builds are handled
           valid = false
+
           next
         end
 
@@ -158,6 +169,16 @@ module Ci
       nil
     end
 
+    def max_queue_depth
+      @max_queue_depth ||= begin
+        if Feature.enabled?(:gitlab_ci_builds_queue_limit, runner, default_enabled: false)
+          MAX_QUEUE_DEPTH
+        else
+          ::Gitlab::Database::MAX_INT_VALUE
+        end
+      end
+    end
+
     # Force variables evaluation to occur now
     def present_build!(build)
       # We need to use the presenter here because Gitaly calls in the presenter

changelogs/unreleased/feature-gb-improve-builds-queueing.yml

+---
+title: Improve performance of builds queuing by introducing a limit on the queue depth
+merge_request: 54579
+author:
+type: performance

config/feature_flags/development/gitlab_ci_builds_queue_limit.yml

+---
+name: gitlab_ci_builds_queue_limit
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/54579
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/323201
+milestone: '13.10'
+type: development
+group: group::continuous integration
+default_enabled: false

lib/gitlab/ci/queue/metrics.rb

@@ -27,6 +27,7 @@ module Gitlab
           :queue_attempt,
           :queue_conflict,
           :queue_iteration,
+          :queue_depth_limit,
           :queue_replication_lag,
           :runner_pre_assign_checks_failed,
           :runner_pre_assign_checks_success,

spec/services/ci/register_job_service_spec.rb

@@ -790,6 +790,46 @@ module Ci
       end
     end
 
+    context 'when max queue depth is reached' do
+      let!(:pending_job) { create(:ci_build, :pending, :degenerated, pipeline: pipeline) }
+      let!(:pending_job_2) { create(:ci_build, :pending, :degenerated, pipeline: pipeline) }
+      let!(:pending_job_3) { create(:ci_build, :pending, pipeline: pipeline) }
+
+      before do
+        stub_const("#{described_class}::MAX_QUEUE_DEPTH", 2)
+      end
+
+      context 'when feature is enabled' do
+        before do
+          stub_feature_flags(gitlab_ci_builds_queue_limit: true)
+        end
+
+        it 'returns 409 conflict' do
+          expect(Ci::Build.pending.unstarted.count).to eq 3
+
+          result = described_class.new(specific_runner).execute
+
+          expect(result).not_to be_valid
+          expect(result.build).to be_nil
+        end
+      end
+
+      context 'when feature is disabled' do
+        before do
+          stub_feature_flags(gitlab_ci_builds_queue_limit: false)
+        end
+
+        it 'returns a valid result' do
+          expect(Ci::Build.pending.unstarted.count).to eq 3
+
+          result = described_class.new(specific_runner).execute
+
+          expect(result).to be_valid
+          expect(result.build).to eq pending_job_3
+        end
+      end
+    end
+
     def execute(runner, params = {})
       described_class.new(runner).execute(params).build
     end