Merge branch 'skr-pipeline-build' into 'master'

Add log count of active jobs

See merge request gitlab-org/gitlab!64623

app/models/ci/pipeline.rb

@@ -393,6 +393,10 @@ module Ci
       newest_first(ref: ref).failed.take
     end
 
+    def self.jobs_count_in_alive_pipelines
+      created_after(24.hours.ago).alive.joins(:builds).count
+    end
+
     # Returns a Hash containing the latest pipeline for every given
     # commit.
     #

doc/administration/monitoring/prometheus/gitlab_metrics.md

@@ -45,6 +45,7 @@ The following metrics are available:
 | `gitlab_ci_pipeline_size_builds`                                 | Histogram   | 13.1    | Total number of builds within a pipeline grouped by a pipeline source                                                 | `source`                                                  |
 | `job_waiter_started_total`                                       | Counter     | 12.9    | Number of batches of jobs started where a web request is waiting for the jobs to complete                             | `worker`                                                  |
 | `job_waiter_timeouts_total`                                      | Counter     | 12.9    | Number of batches of jobs that timed out where a web request is waiting for the jobs to complete                      | `worker`                                                  |
+| `gitlab_ci_active_jobs`                                          | Histogram   | 14.2    | Count of active jobs when pipeline is created                                                                         |                                                           |
 | `gitlab_database_transaction_seconds`                            | Histogram   | 12.1    | Time spent in database transactions, in seconds                                                                       |                                                           |
 | `gitlab_method_call_duration_seconds`                            | Histogram   | 10.2    | Method calls real duration                                                                                            | `controller`, `action`, `module`, `method`                |
 | `gitlab_page_out_of_bounds`                                      | Counter     | 12.8    | Counter for the PageLimiter pagination limit being hit                                                                | `controller`, `action`, `bot`                             |

ee/lib/ee/gitlab/ci/pipeline/quota/job_activity.rb

@@ -38,13 +38,11 @@ module EE
               @excessive ||= jobs_in_alive_pipelines_count - ci_active_jobs_limit
             end
 
-            # rubocop: disable CodeReuse/ActiveRecord
             def jobs_in_alive_pipelines_count
               strong_memoize(:jobs_in_alive_pipelines_count) do
-                @project.all_pipelines.created_after(24.hours.ago).alive.joins(:builds).count
+                @project.all_pipelines.jobs_count_in_alive_pipelines
               end
             end
-            # rubocop: enable CodeReuse/ActiveRecord
 
             def ci_active_jobs_limit
               strong_memoize(:ci_active_jobs_limit) do

lib/gitlab/ci/pipeline/chain/command.rb

@@ -97,6 +97,11 @@ module Gitlab
               .observe({ source: pipeline.source.to_s }, pipeline.total_size)
           end
 
+          def observe_jobs_count_in_alive_pipelines
+            metrics.active_jobs_histogram
+              .observe({ plan: project.actual_plan_name }, project.all_pipelines.jobs_count_in_alive_pipelines)
+          end
+
           def increment_pipeline_failure_reason_counter(reason)
             metrics.pipeline_failure_reason_counter
               .increment(reason: (reason || :unknown_failure).to_s)

lib/gitlab/ci/pipeline/chain/sequence.rb

@@ -22,6 +22,7 @@ module Gitlab
 
             @command.observe_creation_duration(Time.now - @start)
             @command.observe_pipeline_size(@pipeline)
+            @command.observe_jobs_count_in_alive_pipelines
 
             @pipeline
           end

lib/gitlab/ci/pipeline/metrics.rb

@@ -29,6 +29,15 @@ module Gitlab
           ::Gitlab::Metrics.histogram(name, comment, labels, buckets)
         end
 
+        def self.active_jobs_histogram
+          name = :gitlab_ci_active_jobs
+          comment = 'Total amount of active jobs'
+          labels = { plan: nil }
+          buckets = [0, 200, 500, 1_000, 2_000, 5_000, 10_000]
+
+          ::Gitlab::Metrics.histogram(name, comment, labels, buckets)
+        end
+
         def self.pipeline_processing_events_counter
           name = :gitlab_ci_pipeline_processing_events_total
           comment = 'Total amount of pipeline processing events'

spec/lib/gitlab/ci/pipeline/chain/sequence_spec.rb

@@ -7,7 +7,7 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Sequence do
   let_it_be(:user) { create(:user) }
 
   let(:pipeline) { build_stubbed(:ci_pipeline) }
-  let(:command) { Gitlab::Ci::Pipeline::Chain::Command.new }
+  let(:command) { Gitlab::Ci::Pipeline::Chain::Command.new(project: project) }
   let(:first_step) { spy('first step') }
   let(:second_step) { spy('second step') }
   let(:sequence) { [first_step, second_step] }
@@ -71,5 +71,20 @@ RSpec.describe Gitlab::Ci::Pipeline::Chain::Sequence do
       expect(histogram).to have_received(:observe)
         .with({ source: 'push' }, 0)
     end
+
+    it 'records active jobs by pipeline plan in a histogram' do
+      allow(command.metrics)
+        .to receive(:active_jobs_histogram)
+        .and_return(histogram)
+
+      pipeline = create(:ci_pipeline, project: project, status: :running)
+      create(:ci_build, :finished, project: project, pipeline: pipeline)
+      create(:ci_build, :failed, project: project, pipeline: pipeline)
+      create(:ci_build, :running, project: project, pipeline: pipeline)
+      subject.build!
+
+      expect(histogram).to have_received(:observe)
+        .with(hash_including(plan: project.actual_plan_name), 3)
+    end
   end
 end