From 52edbc6e312c826c5b47f5cf65d60adee770b031 Mon Sep 17 00:00:00 2001
From: Matthias Kaeppler <mkaeppler@gitlab.com>
Date: Tue, 20 Jul 2021 10:36:35 +0200
Subject: [PATCH] Fix: Sidekiq workers delete each other's metrics
When we moved the logic that wipes the Prometheus metrics
dir out of the Rackup file and into the initializer, all
Sidekiq workers would call this and potentially enter a
race condition where they deleted each other's database
files.
Changelog: fixed
---
config.ru | 6 ++++++
config/initializers/7_prometheus_metrics.rb | 3 ---
2 files changed, 6 insertions(+), 3 deletions(-)
diff --git a/config.ru b/config.ru
index e9964ddc96ebe..c74a49cd0e2c6 100644
--- a/config.ru
+++ b/config.ru
@@ -9,6 +9,12 @@ def master_process?
end
warmup do |app|
+ # The following is necessary to ensure stale Prometheus metrics don't accumulate over time.
+ # It needs to be done as early as here to ensure metrics files aren't deleted.
+ # After we hit our app in `warmup`, first metrics and corresponding files already being created,
+ # for example in `lib/gitlab/metrics/requests_rack_middleware.rb`.
+ Prometheus::CleanupMultiprocDirService.new.execute if master_process?
+
client = Rack::MockRequest.new(app)
client.get('/')
end
diff --git a/config/initializers/7_prometheus_metrics.rb b/config/initializers/7_prometheus_metrics.rb
index fc69ae7b0777d..d2d546a54386d 100644
--- a/config/initializers/7_prometheus_metrics.rb
+++ b/config/initializers/7_prometheus_metrics.rb
@@ -40,9 +40,6 @@ def prometheus_default_multiproc_dir
# When running Puma in a Single mode, `on_master_start` and `on_worker_start` are the same.
# Thus, we order these events to run `reinitialize_on_pid_change` with `force: true` first.
Gitlab::Cluster::LifecycleEvents.on_master_start do
- # Ensure that stale Prometheus metrics don't accumulate over time
- ::Prometheus::CleanupMultiprocDirService.new.execute
-
::Prometheus::Client.reinitialize_on_pid_change(force: true)
if Gitlab::Runtime.puma?
--
GitLab