From a7d75d0b601437a9fd96d879ebc060280b71d0a0 Mon Sep 17 00:00:00 2001
From: Marius Bobin <mbobin@gitlab.com>
Date: Tue, 24 Aug 2021 19:41:02 +0300
Subject: [PATCH] Update the query that looks for stuck pending jobs
We have around 500 CI jobs that are in the pending state but can't be
picked by the runners.
---
app/models/commit_status.rb | 3 ++-
app/workers/stuck_ci_jobs_worker.rb | 14 ++++++++++++--
.../ci_new_query_for_pending_stuck_jobs.yml | 8 ++++++++
spec/models/commit_status_spec.rb | 9 +++++++++
4 files changed, 31 insertions(+), 3 deletions(-)
create mode 100644 config/feature_flags/development/ci_new_query_for_pending_stuck_jobs.yml
diff --git a/app/models/commit_status.rb b/app/models/commit_status.rb
index 8ff5a6f38e211..8dd650751bf0b 100644
--- a/app/models/commit_status.rb
+++ b/app/models/commit_status.rb
@@ -58,7 +58,8 @@ class CommitStatus < Ci::ApplicationRecord
scope :in_pipelines, ->(pipelines) { where(pipeline: pipelines) }
scope :eager_load_pipeline, -> { eager_load(:pipeline, project: { namespace: :route }) }
scope :with_pipeline, -> { joins(:pipeline) }
- scope :updated_at_before, ->(date) { where('updated_at < ?', date) }
+ scope :updated_at_before, ->(date) { where('ci_builds.updated_at < ?', date) }
+ scope :created_at_before, ->(date) { where('ci_builds.created_at < ?', date) }
scope :updated_before, ->(lookback:, timeout:) {
where('(ci_builds.created_at BETWEEN ? AND ?) AND (ci_builds.updated_at BETWEEN ? AND ?)', lookback, timeout, lookback, timeout)
}
diff --git a/app/workers/stuck_ci_jobs_worker.rb b/app/workers/stuck_ci_jobs_worker.rb
index 5723380a3f32e..04bef83040f7a 100644
--- a/app/workers/stuck_ci_jobs_worker.rb
+++ b/app/workers/stuck_ci_jobs_worker.rb
@@ -26,14 +26,14 @@ def perform
drop(running_timed_out_builds, failure_reason: :stuck_or_timeout_failure)
drop(
- Ci::Build.pending.updated_before(lookback: BUILD_LOOKBACK.ago, timeout: BUILD_PENDING_OUTDATED_TIMEOUT.ago),
+ pending_builds(BUILD_PENDING_OUTDATED_TIMEOUT.ago),
failure_reason: :stuck_or_timeout_failure
)
drop(scheduled_timed_out_builds, failure_reason: :stale_schedule)
drop_stuck(
- Ci::Build.pending.updated_before(lookback: BUILD_LOOKBACK.ago, timeout: BUILD_PENDING_STUCK_TIMEOUT.ago),
+ pending_builds(BUILD_PENDING_STUCK_TIMEOUT.ago),
failure_reason: :stuck_or_timeout_failure
)
@@ -42,6 +42,16 @@ def perform
private
+ # rubocop: disable CodeReuse/ActiveRecord
+ def pending_builds(timeout)
+ if Feature.enabled?(:ci_new_query_for_pending_stuck_jobs)
+ Ci::Build.pending.created_at_before(timeout).updated_at_before(timeout).order(created_at: :asc, project_id: :asc)
+ else
+ Ci::Build.pending.updated_before(lookback: BUILD_LOOKBACK.ago, timeout: timeout)
+ end
+ end
+ # rubocop: enable CodeReuse/ActiveRecord
+
def scheduled_timed_out_builds
Ci::Build.where(status: :scheduled).where( # rubocop: disable CodeReuse/ActiveRecord
'ci_builds.scheduled_at IS NOT NULL AND ci_builds.scheduled_at < ?',
diff --git a/config/feature_flags/development/ci_new_query_for_pending_stuck_jobs.yml b/config/feature_flags/development/ci_new_query_for_pending_stuck_jobs.yml
new file mode 100644
index 0000000000000..5e63330d01d4d
--- /dev/null
+++ b/config/feature_flags/development/ci_new_query_for_pending_stuck_jobs.yml
@@ -0,0 +1,8 @@
+---
+name: ci_new_query_for_pending_stuck_jobs
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/68880
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/339322
+milestone: '14.3'
+type: development
+group: group::pipeline execution
+default_enabled: false
diff --git a/spec/models/commit_status_spec.rb b/spec/models/commit_status_spec.rb
index a951af4cc4f61..7134a387e6518 100644
--- a/spec/models/commit_status_spec.rb
+++ b/spec/models/commit_status_spec.rb
@@ -88,6 +88,15 @@ def create_status(**opts)
end
end
+ describe '.created_at_before' do
+ it 'finds the relevant records' do
+ status = create(:commit_status, created_at: 1.day.ago, project: project)
+ create(:commit_status, created_at: 1.day.since, project: project)
+
+ expect(described_class.created_at_before(Time.current)).to eq([status])
+ end
+ end
+
describe '.updated_before' do
let!(:lookback) { 5.days.ago }
let!(:timeout) { 1.day.ago }
--
GitLab