Support TLS in dedicated metrics servers

This is required for FIPS compliance.

Changelog: added

config/gitlab.yml.example

@@ -1266,24 +1266,29 @@ production: &base
     ip_whitelist:
       - 127.0.0.0/8
 
-    # Sidekiq exporter is webserver built in to Sidekiq to expose Prometheus metrics
+    # Sidekiq exporter is a dedicated Prometheus metrics server optionally running alongside Sidekiq.
     sidekiq_exporter:
     #  enabled: true
     #  log_enabled: false
     #  address: localhost
     #  port: 8082
+    #  tls_enabled: false
+    #  tls_cert_path: /path/to/cert.pem
+    #  tls_key_path: /path/to/key.pem
 
     sidekiq_health_checks:
     #  enabled: true
     #  address: localhost
     #  port: 8092
 
-    # Web exporter is a dedicated Rack server running alongside Puma to expose Prometheus metrics
-    # It runs alongside the `/metrics` endpoints to ease the publish of metrics
+    # Web exporter is a dedicated Prometheus metrics server optionally running alongside Puma.
     web_exporter:
     #  enabled: true
     #  address: localhost
     #  port: 8083
+    #  tls_enabled: false
+    #  tls_cert_path: /path/to/cert.pem
+    #  tls_key_path: /path/to/key.pem
 
   ## Prometheus settings
   # Do not modify these settings here. They should be modified in /etc/gitlab/gitlab.rb

config/initializers/1_settings.rb

@@ -971,6 +971,9 @@
 Settings.monitoring.sidekiq_exporter['log_enabled'] ||= false
 Settings.monitoring.sidekiq_exporter['address'] ||= 'localhost'
 Settings.monitoring.sidekiq_exporter['port'] ||= 8082
+Settings.monitoring.sidekiq_exporter['tls_enabled'] ||= false
+Settings.monitoring.sidekiq_exporter['tls_cert_path'] ||= nil
+Settings.monitoring.sidekiq_exporter['tls_key_path'] ||= nil
 
 Settings.monitoring['sidekiq_health_checks'] ||= Settingslogic.new({})
 Settings.monitoring.sidekiq_health_checks['enabled'] ||= false
@@ -981,6 +984,9 @@
 Settings.monitoring.web_exporter['enabled'] ||= false
 Settings.monitoring.web_exporter['address'] ||= 'localhost'
 Settings.monitoring.web_exporter['port'] ||= 8083
+Settings.monitoring.web_exporter['tls_enabled'] ||= false
+Settings.monitoring.web_exporter['tls_cert_path'] ||= nil
+Settings.monitoring.web_exporter['tls_key_path'] ||= nil
 
 #
 # Prometheus settings

doc/administration/monitoring/prometheus/web_exporter.md

@@ -51,3 +51,21 @@ To enable the dedicated server:
    for the changes to take effect.
 
 Metrics can now be served and scraped from `localhost:8083/metrics`.
+
+## Enable HTTPS
+
+To serve metrics via HTTPS instead of HTTP, enable TLS in the exporter settings:
+
+1. Edit `/etc/gitlab/gitlab.rb` to add (or find and uncomment) the following lines:
+
+   ```ruby
+   puma['exporter_tls_enabled'] = true
+   puma['exporter_tls_cert_path'] = "/path/to/certificate.pem"
+   puma['exporter_tls_key_path'] = "/path/to/private-key.pem"
+   ```
+
+1. Save the file and [reconfigure GitLab](../../restart_gitlab.md#omnibus-gitlab-reconfigure)
+   for the changes to take effect.
+
+When TLS is enabled, the same `port` and `address` will be used as described above.
+The metrics server cannot serve both HTTP and HTTPS at the same time.

doc/administration/sidekiq.md

@@ -191,6 +191,24 @@ To configure the metrics server:
    sudo gitlab-ctl reconfigure
    ```
 
+### Enable HTTPS
+
+To serve metrics via HTTPS instead of HTTP, enable TLS in the exporter settings:
+
+1. Edit `/etc/gitlab/gitlab.rb` to add (or find and uncomment) the following lines:
+
+   ```ruby
+   sidekiq['exporter_tls_enabled'] = true
+   sidekiq['exporter_tls_cert_path'] = "/path/to/certificate.pem"
+   sidekiq['exporter_tls_key_path'] = "/path/to/private-key.pem"
+   ```
+
+1. Save the file and [reconfigure GitLab](restart_gitlab.md#omnibus-gitlab-reconfigure)
+   for the changes to take effect.
+
+When TLS is enabled, the same `port` and `address` will be used as described above.
+The metrics server cannot serve both HTTP and HTTPS at the same time.
+
 ## Configure health checks
 
 If you use health check probes to observe Sidekiq, enable the Sidekiq health check server.

lib/gitlab/metrics/exporter/base_exporter.rb

@@ -38,10 +38,28 @@ def start_working
             [logger, WEBrick::AccessLog::COMBINED_LOG_FORMAT]
           ]
 
-          @server = ::WEBrick::HTTPServer.new(
-            Port: settings.port, BindAddress: settings.address,
-            Logger: logger, AccessLog: access_log
-          )
+          server_config = {
+            Port: settings.port,
+            BindAddress: settings.address,
+            Logger: logger,
+            AccessLog: access_log
+          }
+
+          if settings['tls_enabled']
+            # This monkey-patches WEBrick::GenericServer, so never require this unless TLS is enabled.
+            require 'webrick/ssl'
+
+            server_config.merge!({
+              SSLEnable: true,
+              SSLCertificate: OpenSSL::X509::Certificate.new(File.binread(settings['tls_cert_path'])),
+              SSLPrivateKey: OpenSSL::PKey.read(File.binread(settings['tls_key_path'])),
+              # SSLStartImmediately is true by default according to the docs, but when WEBrick creates the
+              # SSLServer internally, the switch was always nil for some reason. Setting this explicitly fixes this.
+              SSLStartImmediately: true
+            })
+          end
+
+          @server = ::WEBrick::HTTPServer.new(server_config)
           server.mount '/', Rack::Handler::WEBrick, rack_app
 
           true

metrics_server/metrics_server.rb

@@ -56,6 +56,11 @@ def spawn(target, metrics_dir:, **options)
         env['GME_LOG_LEVEL'] = 'quiet'
       end
 
+      if settings['tls_enabled']
+        env['GME_CERT_FILE'] = settings['tls_cert_path']
+        env['GME_CERT_KEY'] = settings['tls_key_path']
+      end
+
       Process.spawn(env, cmd, err: $stderr, out: $stdout, pgroup: true).tap do |pid|
         Process.detach(pid)
       end

spec/lib/gitlab/metrics/exporter/base_exporter_spec.rb

@@ -19,6 +19,7 @@
       allow(settings).to receive(:enabled).and_return(true)
       allow(settings).to receive(:port).and_return(0)
       allow(settings).to receive(:address).and_return('127.0.0.1')
+      allow(settings).to receive(:[]).with('tls_enabled').and_return(false)
     end
 
     after do
@@ -88,6 +89,29 @@
               exporter
             end
           end
+
+          context 'with TLS enabled' do
+            let(:test_cert) { Rails.root.join('spec/fixtures/x509_certificate.crt').to_s }
+            let(:test_key) { Rails.root.join('spec/fixtures/x509_certificate_pk.key').to_s }
+
+            before do
+              allow(settings).to receive(:[]).with('tls_enabled').and_return(true)
+              allow(settings).to receive(:[]).with('tls_cert_path').and_return(test_cert)
+              allow(settings).to receive(:[]).with('tls_key_path').and_return(test_key)
+            end
+
+            it 'injects the necessary OpenSSL config for WEBrick' do
+              expect(::WEBrick::HTTPServer).to receive(:new).with(
+                a_hash_including(
+                  SSLEnable: true,
+                  SSLCertificate: an_instance_of(OpenSSL::X509::Certificate),
+                  SSLPrivateKey: an_instance_of(OpenSSL::PKey::RSA),
+                  SSLStartImmediately: true
+                ))
+
+              exporter.start
+            end
+          end
         end
 
         describe 'when thread is not alive' do
@@ -159,6 +183,7 @@ def call(env)
       allow(settings).to receive(:enabled).and_return(true)
       allow(settings).to receive(:port).and_return(0)
       allow(settings).to receive(:address).and_return('127.0.0.1')
+      allow(settings).to receive(:[]).with('tls_enabled').and_return(false)
 
       stub_const('Gitlab::Metrics::Exporter::MetricsMiddleware', fake_collector)
 

spec/metrics_server/metrics_server_spec.rb

@@ -171,6 +171,29 @@
               described_class.spawn(target, metrics_dir: metrics_dir)
             end
           end
+
+          context 'when TLS settings are present' do
+            before do
+              %w(web_exporter sidekiq_exporter).each do |key|
+                settings[key]['tls_enabled'] = true
+                settings[key]['tls_cert_path'] = '/path/to/cert.pem'
+                settings[key]['tls_key_path'] = '/path/to/key.pem'
+              end
+            end
+
+            it 'sets the correct environment variables' do
+              expect(Process).to receive(:spawn).with(
+                expected_env.merge(
+                  'GME_CERT_FILE' => '/path/to/cert.pem',
+                  'GME_CERT_KEY' => '/path/to/key.pem'
+                ),
+                '/path/to/gme/gitlab-metrics-exporter',
+                hash_including(pgroup: true)
+              ).and_return(99)
+
+              described_class.spawn(target, metrics_dir: metrics_dir, path: '/path/to/gme/')
+            end
+          end
         end
       end
     end