Update the query that looks for stuck pending jobs

We have around 500 CI jobs that are in the pending state but can't be
picked by the runners.

app/models/commit_status.rb

@@ -58,7 +58,8 @@ class CommitStatus < Ci::ApplicationRecord
   scope :in_pipelines, ->(pipelines) { where(pipeline: pipelines) }
   scope :eager_load_pipeline, -> { eager_load(:pipeline, project: { namespace: :route }) }
   scope :with_pipeline, -> { joins(:pipeline) }
-  scope :updated_at_before, ->(date) { where('updated_at < ?', date) }
+  scope :updated_at_before, ->(date) { where('ci_builds.updated_at < ?', date) }
+  scope :created_at_before, ->(date) { where('ci_builds.created_at < ?', date) }
   scope :updated_before, ->(lookback:, timeout:) {
     where('(ci_builds.created_at BETWEEN ? AND ?) AND (ci_builds.updated_at BETWEEN ? AND ?)', lookback, timeout, lookback, timeout)
   }

app/workers/stuck_ci_jobs_worker.rb

@@ -26,14 +26,14 @@ def perform
     drop(running_timed_out_builds, failure_reason: :stuck_or_timeout_failure)
 
     drop(
-      Ci::Build.pending.updated_before(lookback: BUILD_LOOKBACK.ago, timeout: BUILD_PENDING_OUTDATED_TIMEOUT.ago),
+      pending_builds(BUILD_PENDING_OUTDATED_TIMEOUT.ago),
       failure_reason: :stuck_or_timeout_failure
     )
 
     drop(scheduled_timed_out_builds, failure_reason: :stale_schedule)
 
     drop_stuck(
-      Ci::Build.pending.updated_before(lookback: BUILD_LOOKBACK.ago, timeout: BUILD_PENDING_STUCK_TIMEOUT.ago),
+      pending_builds(BUILD_PENDING_STUCK_TIMEOUT.ago),
       failure_reason: :stuck_or_timeout_failure
     )
 
@@ -42,6 +42,16 @@ def perform
 
   private
 
+  # rubocop: disable CodeReuse/ActiveRecord
+  def pending_builds(timeout)
+    if Feature.enabled?(:ci_new_query_for_pending_stuck_jobs)
+      Ci::Build.pending.created_at_before(timeout).updated_at_before(timeout).order(created_at: :asc, project_id: :asc)
+    else
+      Ci::Build.pending.updated_before(lookback: BUILD_LOOKBACK.ago, timeout: timeout)
+    end
+  end
+  # rubocop: enable CodeReuse/ActiveRecord
+
   def scheduled_timed_out_builds
     Ci::Build.where(status: :scheduled).where( # rubocop: disable CodeReuse/ActiveRecord
       'ci_builds.scheduled_at IS NOT NULL AND ci_builds.scheduled_at < ?',

config/feature_flags/development/ci_new_query_for_pending_stuck_jobs.yml

+---
+name: ci_new_query_for_pending_stuck_jobs
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68880
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/339322
+milestone: '14.3'
+type: development
+group: group::pipeline execution
+default_enabled: false

spec/models/commit_status_spec.rb

@@ -88,6 +88,15 @@ def create_status(**opts)
     end
   end
 
+  describe '.created_at_before' do
+    it 'finds the relevant records' do
+      status = create(:commit_status, created_at: 1.day.ago, project: project)
+      create(:commit_status, created_at: 1.day.since, project: project)
+
+      expect(described_class.created_at_before(Time.current)).to eq([status])
+    end
+  end
+
   describe '.updated_before' do
     let!(:lookback) { 5.days.ago }
     let!(:timeout) { 1.day.ago }