Improve gitlab_ci_queue_size_total metric

Improving the metric in two ways:

1. By adding few more buckets. At GitLab.com we've found that  the
   maximum bucket of 5000 is too small in some important cases with the
   scale that we're working there.

1. Labels the metric with the `runner_type` label. This will help
   distinguish queues handled for different runner types.

app/services/ci/register_job_service.rb

@@ -112,7 +112,7 @@ module Ci
       if Feature.enabled?(:ci_register_job_service_one_by_one, runner, default_enabled: true)
         build_ids = retrieve_queue(-> { builds.pluck(:id) })
 
-        @metrics.observe_queue_size(-> { build_ids.size })
+        @metrics.observe_queue_size(-> { build_ids.size }, @runner.runner_type)
 
         build_ids.each do |build_id|
           yield Ci::Build.find(build_id)
@@ -120,7 +120,7 @@ module Ci
       else
         builds_array = retrieve_queue(-> { builds.to_a })
 
-        @metrics.observe_queue_size(-> { builds_array.size })
+        @metrics.observe_queue_size(-> { builds_array.size }, @runner.runner_type)
 
         builds_array.each(&blk)
       end

lib/gitlab/ci/queue/metrics.rb

@@ -9,7 +9,7 @@ module Gitlab
         QUEUE_DURATION_SECONDS_BUCKETS = [1, 3, 10, 30, 60, 300, 900, 1800, 3600].freeze
         QUEUE_ACTIVE_RUNNERS_BUCKETS = [1, 3, 10, 30, 60, 300, 900, 1800, 3600].freeze
         QUEUE_DEPTH_TOTAL_BUCKETS = [1, 2, 3, 5, 8, 16, 32, 50, 100, 250, 500, 1000, 2000, 5000].freeze
-        QUEUE_SIZE_TOTAL_BUCKETS = [1, 5, 10, 50, 100, 500, 1000, 2000, 5000].freeze
+        QUEUE_SIZE_TOTAL_BUCKETS = [1, 5, 10, 50, 100, 500, 1000, 2000, 5000, 7500, 10000, 15000, 20000].freeze
         QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS = [0.01, 0.05, 0.1, 0.3, 0.5, 1, 5, 10, 30, 60, 180, 300].freeze
 
         METRICS_SHARD_TAG_PREFIX = 'metrics_shard::'
@@ -94,10 +94,10 @@ module Gitlab
           self.class.queue_depth_total.observe({ queue: queue }, size.to_f)
         end
 
-        def observe_queue_size(size_proc)
+        def observe_queue_size(size_proc, runner_type)
           return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false)
 
-          self.class.queue_size_total.observe({}, size_proc.call.to_f)
+          self.class.queue_size_total.observe({ runner_type: runner_type }, size_proc.call.to_f)
         end
 
         def observe_queue_time(metric)

spec/services/ci/register_job_service_spec.rb

@@ -617,7 +617,8 @@ module Ci
 
           it "observes queue size of only matching jobs" do
             # pending_job + 2 x matching ones
-            expect(Gitlab::Ci::Queue::Metrics.queue_size_total).to receive(:observe).with({}, 3)
+            expect(Gitlab::Ci::Queue::Metrics.queue_size_total).to receive(:observe)
+              .with({ runner_type: specific_runner.runner_type }, 3)
 
             expect(execute(specific_runner)).to eq(pending_job)
           end