From 79df54fb898e02a5d129edd13a0bfc794ec00abd Mon Sep 17 00:00:00 2001
From: Stan Hu <stanhu@gmail.com>
Date: Wed, 13 Nov 2024 20:34:45 -0800
Subject: [PATCH] Avoid division by zero in Elasticsearch rate calculator
https://gitlab.com/gitlab-org/gitlab/-/merge_requests/101693
introduced a bytes per second timing calculation for the indexer
that relied on `Time.current`. However, on some hosts `Time.current`
may only have second precision, which leads to a division by 0 error.
Fix this by using `Process.clock_gettime(Process::CLOCK_MONOTONIC)`
to get a more accurate timestamp.
Relates to
https://gitlab.com/gitlab-org/quality/engineering-productivity/master-broken-incidents/-/issues/9336
---
ee/app/services/elastic/process_bookkeeping_service.rb | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
diff --git a/ee/app/services/elastic/process_bookkeeping_service.rb b/ee/app/services/elastic/process_bookkeeping_service.rb
index 11d407e9a51a..da7cf2ae6a2b 100644
--- a/ee/app/services/elastic/process_bookkeeping_service.rb
+++ b/ee/app/services/elastic/process_bookkeeping_service.rb
@@ -148,8 +148,12 @@ def execute(shards: SHARDS)
private
+ def current_time
+ Process.clock_gettime(Process::CLOCK_MONOTONIC)
+ end
+
def execute_with_redis(redis, shards:) # rubocop:disable Metrics/AbcSize
- start_time = Time.current
+ start_time = current_time
specs_buffer = []
scores = {}
@@ -192,7 +196,7 @@ def execute_with_redis(redis, shards:) # rubocop:disable Metrics/AbcSize
@failures = bulk_indexer.flush
end
- indexed_bytes_per_second = (total_bytes / (Time.current - start_time)).ceil
+ indexed_bytes_per_second = (total_bytes / (current_time - start_time)).ceil
logger.info(
'class' => self.class.name,
@@ -220,7 +224,7 @@ def execute_with_redis(redis, shards:) # rubocop:disable Metrics/AbcSize
'meta.indexing.first_score' => first_score,
'meta.indexing.last_score' => last_score,
'meta.indexing.failures_count' => @failures.count,
- 'meta.indexing.bulk_execution_duration_s' => Time.current - start_time
+ 'meta.indexing.bulk_execution_duration_s' => current_time - start_time
)
end
--
GitLab