diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index 841e7670e454ad30c72541332b15aeefbf08440d..91b197afd325db2d7c4d0e9871d23a637326fd30 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -1266,24 +1266,29 @@ production: &base ip_whitelist: - 127.0.0.0/8 - # Sidekiq exporter is webserver built in to Sidekiq to expose Prometheus metrics + # Sidekiq exporter is a dedicated Prometheus metrics server optionally running alongside Sidekiq. sidekiq_exporter: # enabled: true # log_enabled: false # address: localhost # port: 8082 + # tls_enabled: false + # tls_cert_path: /path/to/cert.pem + # tls_key_path: /path/to/key.pem sidekiq_health_checks: # enabled: true # address: localhost # port: 8092 - # Web exporter is a dedicated Rack server running alongside Puma to expose Prometheus metrics - # It runs alongside the `/metrics` endpoints to ease the publish of metrics + # Web exporter is a dedicated Prometheus metrics server optionally running alongside Puma. web_exporter: # enabled: true # address: localhost # port: 8083 + # tls_enabled: false + # tls_cert_path: /path/to/cert.pem + # tls_key_path: /path/to/key.pem ## Prometheus settings # Do not modify these settings here. They should be modified in /etc/gitlab/gitlab.rb diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 8de514e9455eaf7036fc614d6bf753df1a3cfe2f..c94f0009678b35426228bdb9d14324ed35af1297 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -971,6 +971,9 @@ Settings.monitoring.sidekiq_exporter['log_enabled'] ||= false Settings.monitoring.sidekiq_exporter['address'] ||= 'localhost' Settings.monitoring.sidekiq_exporter['port'] ||= 8082 +Settings.monitoring.sidekiq_exporter['tls_enabled'] ||= false +Settings.monitoring.sidekiq_exporter['tls_cert_path'] ||= nil +Settings.monitoring.sidekiq_exporter['tls_key_path'] ||= nil Settings.monitoring['sidekiq_health_checks'] ||= Settingslogic.new({}) Settings.monitoring.sidekiq_health_checks['enabled'] ||= false @@ -981,6 +984,9 @@ Settings.monitoring.web_exporter['enabled'] ||= false Settings.monitoring.web_exporter['address'] ||= 'localhost' Settings.monitoring.web_exporter['port'] ||= 8083 +Settings.monitoring.web_exporter['tls_enabled'] ||= false +Settings.monitoring.web_exporter['tls_cert_path'] ||= nil +Settings.monitoring.web_exporter['tls_key_path'] ||= nil # # Prometheus settings diff --git a/doc/administration/monitoring/prometheus/web_exporter.md b/doc/administration/monitoring/prometheus/web_exporter.md index 4b449a1d74ec34c3d8620d9b0d2cb4b066bc5d85..f3ad826565c2a235c9ae73a491e6ebbd06e05746 100644 --- a/doc/administration/monitoring/prometheus/web_exporter.md +++ b/doc/administration/monitoring/prometheus/web_exporter.md @@ -51,3 +51,21 @@ To enable the dedicated server: for the changes to take effect. Metrics can now be served and scraped from `localhost:8083/metrics`. + +## Enable HTTPS + +To serve metrics via HTTPS instead of HTTP, enable TLS in the exporter settings: + +1. Edit `/etc/gitlab/gitlab.rb` to add (or find and uncomment) the following lines: + + ```ruby + puma['exporter_tls_enabled'] = true + puma['exporter_tls_cert_path'] = "/path/to/certificate.pem" + puma['exporter_tls_key_path'] = "/path/to/private-key.pem" + ``` + +1. Save the file and [reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure) + for the changes to take effect. + +When TLS is enabled, the same `port` and `address` will be used as described above. +The metrics server cannot serve both HTTP and HTTPS at the same time. diff --git a/doc/administration/sidekiq.md b/doc/administration/sidekiq.md index 528ecf12df9f22f6170a20ee75bcf0af70a16434..c8f20e819e100f75c0ecc3a4e35fe41dbae63ead 100644 --- a/doc/administration/sidekiq.md +++ b/doc/administration/sidekiq.md @@ -191,6 +191,24 @@ To configure the metrics server: sudo gitlab-ctl reconfigure ``` +### Enable HTTPS + +To serve metrics via HTTPS instead of HTTP, enable TLS in the exporter settings: + +1. Edit `/etc/gitlab/gitlab.rb` to add (or find and uncomment) the following lines: + + ```ruby + sidekiq['exporter_tls_enabled'] = true + sidekiq['exporter_tls_cert_path'] = "/path/to/certificate.pem" + sidekiq['exporter_tls_key_path'] = "/path/to/private-key.pem" + ``` + +1. Save the file and [reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure) + for the changes to take effect. + +When TLS is enabled, the same `port` and `address` will be used as described above. +The metrics server cannot serve both HTTP and HTTPS at the same time. + ## Configure health checks If you use health check probes to observe Sidekiq, enable the Sidekiq health check server. diff --git a/lib/gitlab/metrics/exporter/base_exporter.rb b/lib/gitlab/metrics/exporter/base_exporter.rb index ba2eb729d7b84d9f4bcf30ea9dce8ea48493a0af..fc271a24cf24afc2bdb5934122067f805d90f8cf 100644 --- a/lib/gitlab/metrics/exporter/base_exporter.rb +++ b/lib/gitlab/metrics/exporter/base_exporter.rb @@ -38,10 +38,28 @@ def start_working [logger, WEBrick::AccessLog::COMBINED_LOG_FORMAT] ] - @server = ::WEBrick::HTTPServer.new( - Port: settings.port, BindAddress: settings.address, - Logger: logger, AccessLog: access_log - ) + server_config = { + Port: settings.port, + BindAddress: settings.address, + Logger: logger, + AccessLog: access_log + } + + if settings['tls_enabled'] + # This monkey-patches WEBrick::GenericServer, so never require this unless TLS is enabled. + require 'webrick/ssl' + + server_config.merge!({ + SSLEnable: true, + SSLCertificate: OpenSSL::X509::Certificate.new(File.binread(settings['tls_cert_path'])), + SSLPrivateKey: OpenSSL::PKey.read(File.binread(settings['tls_key_path'])), + # SSLStartImmediately is true by default according to the docs, but when WEBrick creates the + # SSLServer internally, the switch was always nil for some reason. Setting this explicitly fixes this. + SSLStartImmediately: true + }) + end + + @server = ::WEBrick::HTTPServer.new(server_config) server.mount '/', Rack::Handler::WEBrick, rack_app true diff --git a/metrics_server/metrics_server.rb b/metrics_server/metrics_server.rb index a5c448b51055f6950395fae1f6e35ee2548b4f80..d1a64aa5b7924862beead5914d4b04776568e503 100644 --- a/metrics_server/metrics_server.rb +++ b/metrics_server/metrics_server.rb @@ -56,6 +56,11 @@ def spawn(target, metrics_dir:, **options) env['GME_LOG_LEVEL'] = 'quiet' end + if settings['tls_enabled'] + env['GME_CERT_FILE'] = settings['tls_cert_path'] + env['GME_CERT_KEY'] = settings['tls_key_path'] + end + Process.spawn(env, cmd, err: $stderr, out: $stdout, pgroup: true).tap do |pid| Process.detach(pid) end diff --git a/spec/lib/gitlab/metrics/exporter/base_exporter_spec.rb b/spec/lib/gitlab/metrics/exporter/base_exporter_spec.rb index 66fba7ab6839db3f70d39195d2c7ceb86dfeb3eb..8175d60c2c96ce6a6df18b1a4f2b6957ba2501e8 100644 --- a/spec/lib/gitlab/metrics/exporter/base_exporter_spec.rb +++ b/spec/lib/gitlab/metrics/exporter/base_exporter_spec.rb @@ -19,6 +19,7 @@ allow(settings).to receive(:enabled).and_return(true) allow(settings).to receive(:port).and_return(0) allow(settings).to receive(:address).and_return('127.0.0.1') + allow(settings).to receive(:[]).with('tls_enabled').and_return(false) end after do @@ -88,6 +89,29 @@ exporter end end + + context 'with TLS enabled' do + let(:test_cert) { Rails.root.join('spec/fixtures/x509_certificate.crt').to_s } + let(:test_key) { Rails.root.join('spec/fixtures/x509_certificate_pk.key').to_s } + + before do + allow(settings).to receive(:[]).with('tls_enabled').and_return(true) + allow(settings).to receive(:[]).with('tls_cert_path').and_return(test_cert) + allow(settings).to receive(:[]).with('tls_key_path').and_return(test_key) + end + + it 'injects the necessary OpenSSL config for WEBrick' do + expect(::WEBrick::HTTPServer).to receive(:new).with( + a_hash_including( + SSLEnable: true, + SSLCertificate: an_instance_of(OpenSSL::X509::Certificate), + SSLPrivateKey: an_instance_of(OpenSSL::PKey::RSA), + SSLStartImmediately: true + )) + + exporter.start + end + end end describe 'when thread is not alive' do @@ -159,6 +183,7 @@ def call(env) allow(settings).to receive(:enabled).and_return(true) allow(settings).to receive(:port).and_return(0) allow(settings).to receive(:address).and_return('127.0.0.1') + allow(settings).to receive(:[]).with('tls_enabled').and_return(false) stub_const('Gitlab::Metrics::Exporter::MetricsMiddleware', fake_collector) diff --git a/spec/metrics_server/metrics_server_spec.rb b/spec/metrics_server/metrics_server_spec.rb index 4c188a6ba299b3ed3b3623df5001d89a27dc63da..c7716184d4865a8f4d938afc19238a7cfb12a20d 100644 --- a/spec/metrics_server/metrics_server_spec.rb +++ b/spec/metrics_server/metrics_server_spec.rb @@ -171,6 +171,29 @@ described_class.spawn(target, metrics_dir: metrics_dir) end end + + context 'when TLS settings are present' do + before do + %w(web_exporter sidekiq_exporter).each do |key| + settings[key]['tls_enabled'] = true + settings[key]['tls_cert_path'] = '/path/to/cert.pem' + settings[key]['tls_key_path'] = '/path/to/key.pem' + end + end + + it 'sets the correct environment variables' do + expect(Process).to receive(:spawn).with( + expected_env.merge( + 'GME_CERT_FILE' => '/path/to/cert.pem', + 'GME_CERT_KEY' => '/path/to/key.pem' + ), + '/path/to/gme/gitlab-metrics-exporter', + hash_including(pgroup: true) + ).and_return(99) + + described_class.spawn(target, metrics_dir: metrics_dir, path: '/path/to/gme/') + end + end end end end