Optimize StuckCiJobsWorker running builds query

Optimize the query to select running builds since it times out

Changelog: performance
MR: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68891

app/models/commit_status.rb

@@ -58,8 +58,8 @@ class CommitStatus < Ci::ApplicationRecord
   scope :in_pipelines, ->(pipelines) { where(pipeline: pipelines) }
   scope :eager_load_pipeline, -> { eager_load(:pipeline, project: { namespace: :route }) }
   scope :with_pipeline, -> { joins(:pipeline) }
-  scope :updated_at_before, ->(date) { where('ci_builds.updated_at < ?', date) }
   scope :created_at_before, ->(date) { where('ci_builds.created_at < ?', date) }
+  scope :updated_at_before, ->(date) { where('updated_at < ?', date) }
   scope :updated_before, ->(lookback:, timeout:) {
     where('(ci_builds.created_at BETWEEN ? AND ?) AND (ci_builds.updated_at BETWEEN ? AND ?)', lookback, timeout, lookback, timeout)
   }

app/workers/stuck_ci_jobs_worker.rb

@@ -63,10 +63,15 @@ class StuckCiJobsWorker # rubocop:disable Scalability/IdempotentWorker
   end
 
   def running_timed_out_builds
-    Ci::Build.running.where( # rubocop: disable CodeReuse/ActiveRecord
-      'ci_builds.updated_at < ?',
-      BUILD_RUNNING_OUTDATED_TIMEOUT.ago
-    )
+    if Feature.enabled?(:ci_new_query_for_running_stuck_jobs)
+      running_builds = Ci::Build.running.created_at_before(BUILD_RUNNING_OUTDATED_TIMEOUT.ago)
+      Ci::Build.id_in(running_builds).updated_at_before(BUILD_RUNNING_OUTDATED_TIMEOUT.ago)
+    else
+      Ci::Build.running.where( # rubocop: disable CodeReuse/ActiveRecord
+        'ci_builds.updated_at < ?',
+        BUILD_RUNNING_OUTDATED_TIMEOUT.ago
+      )
+    end
   end
 
   def try_obtain_lease

config/feature_flags/development/ci_new_query_for_running_stuck_jobs.yml

+---
+name: ci_new_query_for_running_stuck_jobs
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68891
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/339264
+milestone: '14.3'
+type: development
+group: group::pipeline execution
+default_enabled: false

spec/workers/stuck_ci_jobs_worker_spec.rb

@@ -186,20 +186,32 @@ RSpec.describe StuckCiJobsWorker do
     end
   end
 
-  context 'when job is running' do
-    let(:status) { 'running' }
+  shared_examples 'job is running' do
+    context 'when job is running' do
+      let(:status) { 'running' }
+
+      context 'when job was updated_at more than an hour ago' do
+        let(:updated_at) { 2.hours.ago }
 
-    context 'when job was updated_at more than an hour ago' do
-      let(:updated_at) { 2.hours.ago }
+        it_behaves_like 'job is dropped'
+      end
+
+      context 'when job was updated in less than 1 hour ago' do
+        let(:updated_at) { 30.minutes.ago }
 
-      it_behaves_like 'job is dropped'
+        it_behaves_like 'job is unchanged'
+      end
     end
+  end
 
-    context 'when job was updated in less than 1 hour ago' do
-      let(:updated_at) { 30.minutes.ago }
+  it_behaves_like 'job is running'
 
-      it_behaves_like 'job is unchanged'
+  context 'when ci_new_query_for_running_stuck_jobs feature flag is disabled' do
+    before do
+      stub_feature_flags(ci_new_query_for_running_stuck_jobs: false)
     end
+
+    it_behaves_like 'job is running'
   end
 
   %w(success skipped failed canceled).each do |status|