Merge branch 'feature/gb/builds-queue-on-replicas' into 'master'

Ensure that the builds queuing query is executed on replicas [RUN ALL RSPEC] [RUN AS-IF-FOSS]

See merge request gitlab-org/gitlab!56849

app/services/ci/register_job_service.rb

@@ -24,7 +24,7 @@ module Ci
     def execute(params = {})
       @metrics.increment_queue_operation(:queue_attempt)
 
-      @metrics.observe_queue_time do
+      @metrics.observe_queue_time(:process) do
         process_queue(params)
       end
     end
@@ -110,7 +110,7 @@ module Ci
       end
 
       if Feature.enabled?(:ci_register_job_service_one_by_one, runner, default_enabled: true)
-        build_ids = builds.pluck(:id)
+        build_ids = retrieve_queue(-> { builds.pluck(:id) })
 
         @metrics.observe_queue_size(-> { build_ids.size })
 
@@ -118,13 +118,21 @@ module Ci
           yield Ci::Build.find(build_id)
         end
       else
-        @metrics.observe_queue_size(-> { builds.to_a.size })
+        builds_array = retrieve_queue(-> { builds.to_a })
 
-        builds.each(&blk)
+        @metrics.observe_queue_size(-> { builds_array.size })
+
+        builds_array.each(&blk)
       end
     end
     # rubocop: enable CodeReuse/ActiveRecord
 
+    def retrieve_queue(queue_query_proc)
+      @metrics.observe_queue_time(:retrieve) do
+        queue_query_proc.call
+      end
+    end
+
     def process_build(build, params)
       unless build.pending?
         @metrics.increment_queue_operation(:build_not_pending)

config/feature_flags/development/ci_runner_builds_queue_on_replicas.yml

+---
+name: ci_runner_builds_queue_on_replicas
+introduced_by_url: 
+rollout_issue_url: 
+milestone: '13.10'
+type: development
+group: group::continuous integration
+default_enabled: false

ee/app/models/ee/ci/runner.rb

@@ -6,11 +6,28 @@ module EE
       extend ActiveSupport::Concern
 
       def tick_runner_queue
+        ##
+        # We only stick a runner to primary database to be able to detect the
+        # replication lag in `EE::Ci::RegisterJobService#execute`. The
+        # intention here is not to execute `Ci::RegisterJobService#execute` on
+        # the primary database.
+        #
         ::Gitlab::Database::LoadBalancing::Sticking.stick(:runner, id)
 
         super
       end
 
+      def heartbeat(values)
+        return super unless ::Feature.enabled?(:ci_runner_builds_queue_on_replicas, self, default_enabled: :yaml)
+
+        ##
+        # We can safely ignore writes performed by a runner heartbeat. We do
+        # not want to upgrade database connection proxy to use the primary
+        # database after heartbeat write happens.
+        #
+        ::Gitlab::Database::LoadBalancing::Session.without_sticky_writes { super }
+      end
+
       def minutes_cost_factor(access_level)
         return 0.0 unless instance_type?
 

ee/app/services/ee/ci/register_job_service.rb

@@ -28,6 +28,18 @@ module EE
         end
       end
 
+      def retrieve_queue(queue_query_proc)
+        if ::Feature.enabled?(:ci_runner_builds_queue_on_replicas, runner, default_enabled: :yaml)
+          ##
+          # We want to reset a load balancing session to discard the side
+          # effects of writes that could have happened prior to this moment.
+          #
+          ::Gitlab::Database::LoadBalancing::Session.clear_session
+        end
+
+        super
+      end
+
       def builds_for_shared_runner
         return super unless shared_runner_build_limits_feature_enabled?
 

ee/lib/gitlab/database/load_balancing/session.rb

@@ -19,9 +19,14 @@ module Gitlab
           RequestStore.delete(CACHE_KEY)
         end
 
+        def self.without_sticky_writes(&block)
+          current.ignore_writes(&block)
+        end
+
         def initialize
           @use_primary = false
           @performed_write = false
+          @ignore_writes = false
         end
 
         def use_primary?
@@ -42,8 +47,19 @@ module Gitlab
           @use_primary = used_primary || @performed_write
         end
 
+        def ignore_writes(&block)
+          @ignore_writes = true
+
+          yield
+        ensure
+          @ignore_writes = false
+        end
+
         def write!
           @performed_write = true
+
+          return if @ignore_writes
+
           use_primary!
         end
 

ee/spec/lib/gitlab/database/load_balancing/session_spec.rb

@@ -22,6 +22,28 @@ RSpec.describe Gitlab::Database::LoadBalancing::Session do
     end
   end
 
+  describe '.without_sticky_writes' do
+    it 'ignores sticky write events sent by a connection proxy' do
+      described_class.without_sticky_writes do
+        described_class.current.write!
+      end
+
+      session = described_class.current
+
+      expect(session).not_to be_using_primary
+    end
+
+    it 'still is aware of write that happened' do
+      described_class.without_sticky_writes do
+        described_class.current.write!
+      end
+
+      session = described_class.current
+
+      expect(session.performed_write?).to be true
+    end
+  end
+
   describe '#use_primary?' do
     it 'returns true when the primary should be used' do
       instance = described_class.new
@@ -96,4 +118,24 @@ RSpec.describe Gitlab::Database::LoadBalancing::Session do
       expect(instance.performed_write?).to eq(true)
     end
   end
+
+  describe '#ignore_writes' do
+    it 'ignores write events' do
+      instance = described_class.new
+
+      instance.ignore_writes { instance.write! }
+
+      expect(instance).not_to be_using_primary
+      expect(instance.performed_write?).to eq true
+    end
+
+    it 'does not prevent using primary if an exception is raised' do
+      instance = described_class.new
+
+      instance.ignore_writes { raise ArgumentError } rescue ArgumentError
+      instance.write!
+
+      expect(instance).to be_using_primary
+    end
+  end
 end

lib/gitlab/ci/queue/metrics.rb

@@ -10,7 +10,7 @@ module Gitlab
         QUEUE_ACTIVE_RUNNERS_BUCKETS = [1, 3, 10, 30, 60, 300, 900, 1800, 3600].freeze
         QUEUE_DEPTH_TOTAL_BUCKETS = [1, 2, 3, 5, 8, 16, 32, 50, 100, 250, 500, 1000, 2000, 5000].freeze
         QUEUE_SIZE_TOTAL_BUCKETS = [1, 5, 10, 50, 100, 500, 1000, 2000, 5000].freeze
-        QUEUE_ITERATION_DURATION_SECONDS_BUCKETS = [0.1, 0.3, 0.5, 1, 5, 10, 30, 60, 180, 300].freeze
+        QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS = [0.01, 0.05, 0.1, 0.3, 0.5, 1, 5, 10, 30, 60, 180, 300].freeze
 
         METRICS_SHARD_TAG_PREFIX = 'metrics_shard::'
         DEFAULT_METRICS_SHARD = 'default'
@@ -100,7 +100,7 @@ module Gitlab
           self.class.queue_size_total.observe({}, size_proc.call.to_f)
         end
 
-        def observe_queue_time
+        def observe_queue_time(metric)
           start_time = ::Gitlab::Metrics::System.monotonic_time
 
           result = yield
@@ -108,7 +108,15 @@ module Gitlab
           return result unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false)
 
           seconds = ::Gitlab::Metrics::System.monotonic_time - start_time
+
+          case metric
+          when :process
             self.class.queue_iteration_duration_seconds.observe({}, seconds.to_f)
+          when :retrieve
+            self.class.queue_retrieval_duration_seconds.observe({}, seconds.to_f)
+          else
+            raise ArgumentError unless Rails.env.production?
+          end
 
           result
         end
@@ -187,7 +195,18 @@ module Gitlab
           strong_memoize(:queue_iteration_duration_seconds) do
             name = :gitlab_ci_queue_iteration_duration_seconds
             comment = 'Time it takes to find a build in CI/CD queue'
-            buckets = QUEUE_ITERATION_DURATION_SECONDS_BUCKETS
+            buckets = QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS
+            labels = {}
+
+            Gitlab::Metrics.histogram(name, comment, labels, buckets)
+          end
+        end
+
+        def self.queue_retrieval_duration_seconds
+          strong_memoize(:queue_retrieval_duration_seconds) do
+            name = :gitlab_ci_queue_retrieval_duration_seconds
+            comment = 'Time it takes to execute a SQL query to retrieve builds queue'
+            buckets = QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS
             labels = {}
 
             Gitlab::Metrics.histogram(name, comment, labels, buckets)