From b30ae67d140973dcee5d292c13119d82f80182ad Mon Sep 17 00:00:00 2001
From: Aleksei Lipniagov <alipniagov@gitlab.com>
Date: Wed, 19 May 2021 14:34:59 +0300
Subject: [PATCH] Remove Unicorn Sampler and its dependencies
With removing Unicorn support in 14.0, UnicornSampler is no longer
needed.
Raindrops was only used with Unicorn, so it could be removed too.
Update the docs related to the change.
Changelog: removed
---
Gemfile | 1 -
Gemfile.lock | 1 -
...-remove-unicorn-gitlab-unicorn-sampler.yml | 5 +
config/gitlab.yml.example | 8 +-
config/initializers/1_settings.rb | 1 -
config/initializers/7_prometheus_metrics.rb | 6 +-
.../monitoring/performance/index.md | 1 -
.../monitoring/prometheus/gitlab_metrics.md | 16 +-
.../gitlab_rails_cheat_sheet.md | 49 ------
.../metrics/samplers/unicorn_sampler.rb | 73 ---------
.../metrics/samplers/unicorn_sampler_spec.rb | 141 ------------------
11 files changed, 11 insertions(+), 291 deletions(-)
create mode 100644 changelogs/unreleased/330402-remove-unicorn-gitlab-unicorn-sampler.yml
delete mode 100644 lib/gitlab/metrics/samplers/unicorn_sampler.rb
delete mode 100644 spec/lib/gitlab/metrics/samplers/unicorn_sampler_spec.rb
diff --git a/Gemfile b/Gemfile
index cc7d59ef73245..724c7143119ff 100644
--- a/Gemfile
+++ b/Gemfile
@@ -341,7 +341,6 @@ group :metrics do
# Prometheus
gem 'prometheus-client-mmap', '~> 0.12.0'
- gem 'raindrops', '~> 0.18'
end
group :development do
diff --git a/Gemfile.lock b/Gemfile.lock
index fcf5acf6c29eb..6b0c9222858af 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1592,7 +1592,6 @@ DEPENDENCIES
rails-controller-testing
rails-i18n (~> 6.0)
rainbow (~> 3.0)
- raindrops (~> 0.18)
rblineprof (~> 0.3.6)
rbtrace (~> 0.4)
rdoc (~> 6.1.2)
diff --git a/changelogs/unreleased/330402-remove-unicorn-gitlab-unicorn-sampler.yml b/changelogs/unreleased/330402-remove-unicorn-gitlab-unicorn-sampler.yml
new file mode 100644
index 0000000000000..1773ea98cab3d
--- /dev/null
+++ b/changelogs/unreleased/330402-remove-unicorn-gitlab-unicorn-sampler.yml
@@ -0,0 +1,5 @@
+---
+title: Remove Unicorn Sampler
+merge_request: 62090
+author:
+type: removed
diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example
index aee9dd455af1c..81eb98783766c 100644
--- a/config/gitlab.yml.example
+++ b/config/gitlab.yml.example
@@ -33,7 +33,7 @@ production: &base
host: localhost
port: 80 # Set to 443 if using HTTPS, see installation.md#using-https for additional HTTPS configuration details
https: false # Set to true if using HTTPS, see installation.md#using-https for additional HTTPS configuration details
- # The maximum time unicorn/puma can spend on the request. This needs to be smaller than the worker timeout.
+ # The maximum time Puma can spend on the request. This needs to be smaller than the worker timeout.
# Default is 95% of the worker timeout
max_request_duration_seconds: 57
@@ -153,7 +153,7 @@ production: &base
### GraphQL Settings
# Tells the rails application how long it has to complete a GraphQL request.
# We suggest this value to be higher than the database timeout value
- # and lower than the worker timeout set in unicorn/puma. (default: 30)
+ # and lower than the worker timeout set in Puma. (default: 30)
# graphql_timeout: 30
## Repository downloads directory
@@ -1212,8 +1212,6 @@ production: &base
## Monitoring
# Built in monitoring settings
monitoring:
- # Time between sampling of unicorn socket metrics, in seconds
- # unicorn_sampler_interval: 10
# IP whitelist to access monitoring endpoints
ip_whitelist:
- 127.0.0.0/8
@@ -1225,7 +1223,7 @@ production: &base
# address: localhost
# port: 8082
- # Web exporter is webserver built in to Unicorn/Puma to expose Prometheus metrics
+ # Web exporter is a dedicated Rack server running alongside Puma to expose Prometheus metrics
# It runs alongside the `/metrics` endpoints to ease the publish of metrics
web_exporter:
# enabled: true
diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb
index 8d5e19afc20c3..c00d0c04e5f36 100644
--- a/config/initializers/1_settings.rb
+++ b/config/initializers/1_settings.rb
@@ -902,7 +902,6 @@
#
Settings['monitoring'] ||= Settingslogic.new({})
Settings.monitoring['ip_whitelist'] ||= ['127.0.0.1/8']
-Settings.monitoring['unicorn_sampler_interval'] ||= 10
Settings.monitoring['sidekiq_exporter'] ||= Settingslogic.new({})
Settings.monitoring.sidekiq_exporter['enabled'] ||= false
Settings.monitoring.sidekiq_exporter['log_enabled'] ||= false
diff --git a/config/initializers/7_prometheus_metrics.rb b/config/initializers/7_prometheus_metrics.rb
index a304f861db8fb..35acf26c796b4 100644
--- a/config/initializers/7_prometheus_metrics.rb
+++ b/config/initializers/7_prometheus_metrics.rb
@@ -8,8 +8,6 @@ def prometheus_default_multiproc_dir
if Gitlab::Runtime.sidekiq?
Rails.root.join('tmp/prometheus_multiproc_dir/sidekiq')
- elsif Gitlab::Runtime.unicorn?
- Rails.root.join('tmp/prometheus_multiproc_dir/unicorn')
elsif Gitlab::Runtime.puma?
Rails.root.join('tmp/prometheus_multiproc_dir/puma')
else
@@ -49,9 +47,7 @@ def prometheus_default_multiproc_dir
::Prometheus::Client.reinitialize_on_pid_change(force: true)
- if Gitlab::Runtime.unicorn?
- Gitlab::Metrics::Samplers::UnicornSampler.instance(Settings.monitoring.unicorn_sampler_interval).start
- elsif Gitlab::Runtime.puma?
+ if Gitlab::Runtime.puma?
Gitlab::Metrics::Samplers::PumaSampler.instance.start
end
diff --git a/doc/administration/monitoring/performance/index.md b/doc/administration/monitoring/performance/index.md
index 15c54a36f6c98..f3db6ac9f032f 100644
--- a/doc/administration/monitoring/performance/index.md
+++ b/doc/administration/monitoring/performance/index.md
@@ -69,6 +69,5 @@ The following environment variables are recognized:
- `DATABASE_SAMPLER_INTERVAL_SECONDS`
- `ACTION_CABLE_SAMPLER_INTERVAL_SECONDS`
- `PUMA_SAMPLER_INTERVAL_SECONDS`
-- `UNICORN_SAMPLER_INTERVAL_SECONDS`
- `THREADS_SAMPLER_INTERVAL_SECONDS`
- `GLOBAL_SEARCH_SAMPLER_INTERVAL_SECONDS`
diff --git a/doc/administration/monitoring/prometheus/gitlab_metrics.md b/doc/administration/monitoring/prometheus/gitlab_metrics.md
index f29db9ead389d..faa24e76a0e1f 100644
--- a/doc/administration/monitoring/prometheus/gitlab_metrics.md
+++ b/doc/administration/monitoring/prometheus/gitlab_metrics.md
@@ -308,20 +308,8 @@ Some basic Ruby runtime metrics are available:
| `ruby_process_proportional_memory_bytes` | Gauge | 13.0 | Memory usage by process (PSS/Proportional Set Size) |
| `ruby_process_start_time_seconds` | Gauge | 12.0 | UNIX timestamp of process start time |
-## Unicorn Metrics
-
-Unicorn specific metrics, when Unicorn is used.
-
-| Metric | Type | Since | Description |
-|:-----------------------------|:------|:------|:---------------------------------------------------|
-| `unicorn_active_connections` | Gauge | 11.0 | The number of active Unicorn connections (workers) |
-| `unicorn_queued_connections` | Gauge | 11.0 | The number of queued Unicorn connections |
-| `unicorn_workers` | Gauge | 12.0 | The number of Unicorn workers |
-
## Puma Metrics
-When Puma is used instead of Unicorn, the following metrics are available:
-
| Metric | Type | Since | Description |
|:--------------------------------- |:------- |:----- |:----------- |
| `puma_workers` | Gauge | 12.0 | Total number of workers |
@@ -352,8 +340,8 @@ instance (`cache`, `shared_state` etc.).
## Metrics shared directory
The GitLab Prometheus client requires a directory to store metrics data shared between multi-process services.
-Those files are shared among all instances running under Unicorn server.
-The directory must be accessible to all running Unicorn's processes, or
+Those files are shared among all instances running under Puma server.
+The directory must be accessible to all running Puma's processes, or
metrics can't function correctly.
This directory's location is configured using environment variable `prometheus_multiproc_dir`.
diff --git a/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md b/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md
index 588be73e78664..dfa17b4e85d62 100644
--- a/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md
+++ b/doc/administration/troubleshooting/gitlab_rails_cheat_sheet.md
@@ -863,55 +863,6 @@ license.save
License.current # check to make sure it applied
```
-## Unicorn
-
-From [Zendesk ticket #91083](https://gitlab.zendesk.com/agent/tickets/91083) (internal)
-
-### Poll Unicorn requests by seconds
-
-```ruby
-require 'rubygems'
-require 'unicorn'
-
-# Usage for this program
-def usage
- puts "ruby unicorn_status.rb <path to unix socket> <poll interval in seconds>"
- puts "Polls the given Unix socket every interval in seconds. Will not allow you to drop below 3 second poll intervals."
- puts "Example: /opt/gitlab/embedded/bin/ruby poll_unicorn.rb /var/opt/gitlab/gitlab-rails/sockets/gitlab.socket 10"
-end
-
-# Look for required args. Throw usage and exit if they don't exist.
-if ARGV.count < 2
- usage
- exit 1
-end
-
-# Get the socket and threshold values.
-socket = ARGV[0]
-threshold = (ARGV[1]).to_i
-
-# Check threshold - is it less than 3? If so, set to 3 seconds. Safety first!
-if threshold.to_i < 3
- threshold = 3
-end
-
-# Check - does that socket exist?
-unless File.exist?(socket)
- puts "Socket file not found: #{socket}"
- exit 1
-end
-
-# Poll the given socket every THRESHOLD seconds as specified above.
-puts "Running infinite loop. Use CTRL+C to exit."
-puts "------------------------------------------"
-loop do
- Raindrops::Linux.unix_listener_stats([socket]).each do |addr, stats|
- puts DateTime.now.to_s + " Active: " + stats.active.to_s + " Queued: " + stats.queued.to_s
- end
- sleep threshold
-end
-```
-
## Registry
### Registry Disk Space Usage by Project
diff --git a/lib/gitlab/metrics/samplers/unicorn_sampler.rb b/lib/gitlab/metrics/samplers/unicorn_sampler.rb
deleted file mode 100644
index 2fa324f3feab5..0000000000000
--- a/lib/gitlab/metrics/samplers/unicorn_sampler.rb
+++ /dev/null
@@ -1,73 +0,0 @@
-# frozen_string_literal: true
-
-module Gitlab
- module Metrics
- module Samplers
- class UnicornSampler < BaseSampler
- DEFAULT_SAMPLING_INTERVAL_SECONDS = 5
-
- def metrics
- @metrics ||= init_metrics
- end
-
- def init_metrics
- {
- unicorn_active_connections: ::Gitlab::Metrics.gauge(:unicorn_active_connections, 'Unicorn active connections', {}, :max),
- unicorn_queued_connections: ::Gitlab::Metrics.gauge(:unicorn_queued_connections, 'Unicorn queued connections', {}, :max),
- unicorn_workers: ::Gitlab::Metrics.gauge(:unicorn_workers, 'Unicorn workers')
- }
- end
-
- def enabled?
- # Raindrops::Linux.tcp_listener_stats is only present on Linux
- unicorn_with_listeners? && Raindrops::Linux.respond_to?(:tcp_listener_stats)
- end
-
- def sample
- Raindrops::Linux.tcp_listener_stats(tcp_listeners).each do |addr, stats|
- set_unicorn_connection_metrics('tcp', addr, stats)
- end
- Raindrops::Linux.unix_listener_stats(unix_listeners).each do |addr, stats|
- set_unicorn_connection_metrics('unix', addr, stats)
- end
-
- metrics[:unicorn_workers].set({}, unicorn_workers_count)
- end
-
- private
-
- def tcp_listeners
- @tcp_listeners ||= Unicorn.listener_names.grep(%r{\A[^/]+:\d+\z})
- end
-
- def set_unicorn_connection_metrics(type, addr, stats)
- labels = { socket_type: type, socket_address: addr }
-
- metrics[:unicorn_active_connections].set(labels, stats.active)
- metrics[:unicorn_queued_connections].set(labels, stats.queued)
- end
-
- def unix_listeners
- @unix_listeners ||= Unicorn.listener_names - tcp_listeners
- end
-
- def unicorn_with_listeners?
- defined?(Unicorn) && Unicorn.listener_names.any?
- end
-
- def unicorn_workers_count
- http_servers.sum(&:worker_processes)
- end
-
- # Traversal of ObjectSpace is expensive, on fully loaded application
- # it takes around 80ms. The instances of HttpServers are not a subject
- # to change so we can cache the list of servers.
- def http_servers
- return [] unless Gitlab::Runtime.unicorn?
-
- @http_servers ||= ObjectSpace.each_object(::Unicorn::HttpServer).to_a
- end
- end
- end
- end
-end
diff --git a/spec/lib/gitlab/metrics/samplers/unicorn_sampler_spec.rb b/spec/lib/gitlab/metrics/samplers/unicorn_sampler_spec.rb
deleted file mode 100644
index 7971a7cabd531..0000000000000
--- a/spec/lib/gitlab/metrics/samplers/unicorn_sampler_spec.rb
+++ /dev/null
@@ -1,141 +0,0 @@
-# frozen_string_literal: true
-
-require 'spec_helper'
-
-RSpec.describe Gitlab::Metrics::Samplers::UnicornSampler do
- subject { described_class.new(1.second) }
-
- it_behaves_like 'metrics sampler', 'UNICORN_SAMPLER'
-
- describe '#sample' do
- let(:unicorn) { Module.new }
- let(:raindrops) { double('raindrops') }
- let(:stats) { double('stats') }
-
- before do
- stub_const('Unicorn', unicorn)
- stub_const('Raindrops::Linux', raindrops)
- allow(raindrops).to receive(:unix_listener_stats).and_return({})
- allow(raindrops).to receive(:tcp_listener_stats).and_return({})
- end
-
- context 'unicorn listens on unix sockets' do
- let(:socket_address) { '/some/sock' }
- let(:sockets) { [socket_address] }
-
- before do
- allow(unicorn).to receive(:listener_names).and_return(sockets)
- end
-
- it 'samples socket data' do
- expect(raindrops).to receive(:unix_listener_stats).with(sockets)
-
- subject.sample
- end
-
- context 'stats collected' do
- before do
- allow(stats).to receive(:active).and_return('active')
- allow(stats).to receive(:queued).and_return('queued')
- allow(raindrops).to receive(:unix_listener_stats).and_return({ socket_address => stats })
- end
-
- it 'updates metrics type unix and with addr' do
- labels = { socket_type: 'unix', socket_address: socket_address }
-
- expect(subject.metrics[:unicorn_active_connections]).to receive(:set).with(labels, 'active')
- expect(subject.metrics[:unicorn_queued_connections]).to receive(:set).with(labels, 'queued')
-
- subject.sample
- end
- end
- end
-
- context 'unicorn listens on tcp sockets' do
- let(:tcp_socket_address) { '0.0.0.0:8080' }
- let(:tcp_sockets) { [tcp_socket_address] }
-
- before do
- allow(unicorn).to receive(:listener_names).and_return(tcp_sockets)
- end
-
- it 'samples socket data' do
- expect(raindrops).to receive(:tcp_listener_stats).with(tcp_sockets)
-
- subject.sample
- end
-
- context 'stats collected' do
- before do
- allow(stats).to receive(:active).and_return('active')
- allow(stats).to receive(:queued).and_return('queued')
- allow(raindrops).to receive(:tcp_listener_stats).and_return({ tcp_socket_address => stats })
- end
-
- it 'updates metrics type unix and with addr' do
- labels = { socket_type: 'tcp', socket_address: tcp_socket_address }
-
- expect(subject.metrics[:unicorn_active_connections]).to receive(:set).with(labels, 'active')
- expect(subject.metrics[:unicorn_queued_connections]).to receive(:set).with(labels, 'queued')
-
- subject.sample
- end
- end
- end
-
- context 'unicorn workers' do
- before do
- allow(unicorn).to receive(:listener_names).and_return([])
- end
-
- context 'without http server' do
- it "does set unicorn_workers to 0" do
- expect(subject.metrics[:unicorn_workers]).to receive(:set).with({}, 0)
-
- subject.sample
- end
- end
-
- context 'with http server' do
- let(:http_server_class) { Struct.new(:worker_processes) }
- let!(:http_server) { http_server_class.new(5) }
-
- before do
- stub_const('Unicorn::HttpServer', http_server_class)
- end
-
- it "sets additional metrics" do
- expect(subject.metrics[:unicorn_workers]).to receive(:set).with({}, 5)
-
- subject.sample
- end
- end
- end
- end
-
- describe '#start' do
- context 'when enabled' do
- before do
- allow(subject).to receive(:enabled?).and_return(true)
- end
-
- it 'creates new thread' do
- expect(Thread).to receive(:new)
-
- subject.start
- end
- end
-
- context 'when disabled' do
- before do
- allow(subject).to receive(:enabled?).and_return(false)
- end
-
- it "doesn't create new thread" do
- expect(Thread).not_to receive(:new)
-
- subject.start
- end
- end
- end
-end
--
GitLab