diff --git a/app/controllers/health_controller.rb b/app/controllers/health_controller.rb
index 125746d04264bcf6ca87d9671aa1fa3e530500bb..03cc896fce9512b7b4f8c8bc672388436416ec9c 100644
--- a/app/controllers/health_controller.rb
+++ b/app/controllers/health_controller.rb
@@ -1,3 +1,5 @@
+require 'prometheus/client/formats/text'
+
class HealthController < ActionController::Base
protect_from_forgery with: :exception
include RequiresHealthToken
@@ -24,7 +26,7 @@ def metrics
results = CHECKS.flat_map(&:metrics)
response = results.map(&method(:metric_to_prom_line)).join("\n")
-
+ response = ::Prometheus::Client::Formats::Text.marshal_multiprocess
render text: response, content_type: 'text/plain; version=0.0.4'
end
diff --git a/app/services/prom_service.rb b/app/services/prom_service.rb
index e9ccce758e96d50fbf9532747c2921a34893b44c..93d52fd2c79e96d4f8d2911ce53187be3d520f71 100644
--- a/app/services/prom_service.rb
+++ b/app/services/prom_service.rb
@@ -11,6 +11,5 @@ def initialize
@login = Prometheus::Client::Counter.new(:login, 'Login counter')
@prometheus.register(@login)
-
end
end
diff --git a/config.ru b/config.ru
index 82af814341b91ca0dd5034ee30878bf01bdd6a75..cba44122da93ee63db4b2cc3340d4619a74b14a9 100644
--- a/config.ru
+++ b/config.ru
@@ -15,11 +15,7 @@ if defined?(Unicorn)
end
# TODO(lyda): Needs to be set externally.
- ENV['prometheus_multiproc_dir'] = '/tmp'
-
- require 'prometheus/client/rack/exporter'
-
- use Prometheus::Client::Rack::Exporter, path: '/admin/metrics'
+ ENV['prometheus_multiproc_dir'] = '/tmp/somestuff'
end
require ::File.expand_path('../config/environment', __FILE__)
diff --git a/lib/gitlab/metrics.rb b/lib/gitlab/metrics.rb
index cb8db2f1e9fa6aabfa5dba62102c91e398c49dab..e784ca785f046f109baad36359a044e838fde0d3 100644
--- a/lib/gitlab/metrics.rb
+++ b/lib/gitlab/metrics.rb
@@ -1,3 +1,5 @@
+require 'prometheus/client'
+
module Gitlab
module Metrics
extend Gitlab::CurrentSettings
@@ -9,6 +11,7 @@ module Metrics
def self.settings
@settings ||= {
enabled: current_application_settings[:metrics_enabled],
+ prometheus_metrics_enabled: true,
pool_size: current_application_settings[:metrics_pool_size],
timeout: current_application_settings[:metrics_timeout],
method_call_threshold: current_application_settings[:metrics_method_call_threshold],
@@ -19,10 +22,18 @@ def self.settings
}
end
- def self.enabled?
+ def self.prometheus_metrics_enabled?
+ settings[:prometheus_metrics_enabled] || false
+ end
+
+ def self.influx_metrics_enabled?
settings[:enabled] || false
end
+ def self.enabled?
+ influx_metrics_enabled? || prometheus_metrics_enabled? || false
+ end
+
def self.mri?
RUBY_ENGINE == 'ruby'
end
@@ -38,10 +49,58 @@ def self.pool
@pool
end
+ def self.registry
+ @registry ||= ::Prometheus::Client.registry
+ end
+
+ def self.counter(name, docstring, base_labels = {})
+ dummy_metric || registry.get(name) || registry.counter(name, docstring, base_labels)
+ end
+
+ def self.summary(name, docstring, base_labels = {})
+ dummy_metric || registry.get(name) || registry.summary(name, docstring, base_labels)
+ end
+
+ def self.gauge(name, docstring, base_labels = {})
+ dummy_metric || registry.get(name) || registry.gauge(name, docstring, base_labels)
+ end
+
+ def self.histogram(name, docstring, base_labels = {}, buckets = Histogram::DEFAULT_BUCKETS)
+ dummy_metric || registry.get(name) || registry.histogram(name, docstring, base_labels, buckets)
+ end
+
+ def self.dummy_metric
+ unless prometheus_metrics_enabled?
+ DummyMetric.new
+ end
+ end
+
def self.submit_metrics(metrics)
prepared = prepare_metrics(metrics)
- pool.with do |connection|
+ if prometheus_metrics_enabled?
+ metrics.map do |metric|
+ known = [:series, :tags,:values, :timestamp]
+ value = metric&.[](:values)&.[](:value)
+ handled= [:rails_gc_statistics]
+ if handled.include? metric[:series].to_sym
+ next
+ end
+
+ if metric.keys.any? {|k| !known.include?(k)} || value.nil?
+ print metric
+ print "\n"
+
+ {:series=>"rails_gc_statistics", :tags=>{}, :values=>{:count=>0, :heap_allocated_pages=>4245, :heap_sorted_length=>4426, :heap_allocatable_pages=>0, :heap_available_slots=>1730264, :heap_live_slots=>1729935, :heap_free_slots=>329, :heap_final_slots=>0, :heap_marked_slots=>1184216, :heap_swept_slots=>361843, :heap_eden_pages=>4245, :heap_tomb_pages=>0, :total_allocated_pages=>4245, :total_freed_pages=>0, :total_allocated_objects=>15670757, :total_freed_objects=>13940822, :malloc_increase_bytes=>4842256, :malloc_increase_bytes_limit=>29129457, :minor_gc_count=>0, :major_gc_count=>0, :remembered_wb_unprotected_objects=>39905, :remembered_wb_unprotected_objects_limit=>74474, :old_objects=>1078731, :old_objects_limit=>1975860, :oldmalloc_increase_bytes=>4842640, :oldmalloc_increase_bytes_limit=>31509677, :total_time=>0.0}, :timestamp=>1494356175592659968}
+
+ next
+ end
+ metric_value = gauge(metric[:series].to_sym, metric[:series])
+ metric_value.set(metric[:tags], value)
+ end
+ end
+
+ pool&.with do |connection|
prepared.each_slice(settings[:packet_size]) do |slice|
begin
connection.write_points(slice)
@@ -148,7 +207,7 @@ def self.current_transaction
# When enabled this should be set before being used as the usual pattern
# "@foo ||= bar" is _not_ thread-safe.
- if enabled?
+ if influx_metrics_enabled?
@pool = ConnectionPool.new(size: settings[:pool_size], timeout: settings[:timeout]) do
host = settings[:host]
port = settings[:port]
diff --git a/lib/gitlab/metrics/dummy_metric.rb b/lib/gitlab/metrics/dummy_metric.rb
new file mode 100644
index 0000000000000000000000000000000000000000..d27bb83854ae90efe4a617c80f51d57bd44bde59
--- /dev/null
+++ b/lib/gitlab/metrics/dummy_metric.rb
@@ -0,0 +1,29 @@
+module Gitlab
+ module Metrics
+ # Mocks ::Prometheus::Client::Metric and all derived metrics
+ class DummyMetric
+ def get(*args)
+ raise NotImplementedError
+ end
+
+ def values(*args)
+ raise NotImplementedError
+ end
+
+ # counter
+ def increment(*args)
+ # noop
+ end
+
+ # gauge
+ def set(*args)
+ # noop
+ end
+
+ # histogram / summary
+ def observe(*args)
+ # noop
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/metrics/prometheus_sampler.rb b/lib/gitlab/metrics/prometheus_sampler.rb
new file mode 100644
index 0000000000000000000000000000000000000000..5f90d4f0b66f4d6106495c0598b62ca8f2cbe618
--- /dev/null
+++ b/lib/gitlab/metrics/prometheus_sampler.rb
@@ -0,0 +1,51 @@
+module Gitlab
+ module Metrics
+ # Class that sends certain metrics to InfluxDB at a specific interval.
+ #
+ # This class is used to gather statistics that can't be directly associated
+ # with a transaction such as system memory usage, garbage collection
+ # statistics, etc.
+ class PrometheusSamples
+ # interval - The sampling interval in seconds.
+ def initialize(interval = Metrics.settings[:sample_interval])
+ interval_half = interval.to_f / 2
+
+ @interval = interval
+ @interval_steps = (-interval_half..interval_half).step(0.1).to_a
+ end
+
+ def start
+ Thread.new do
+ Thread.current.abort_on_exception = true
+
+ loop do
+ sleep(sleep_interval)
+
+ sample
+ end
+ end
+ end
+
+ def sidekiq?
+ Sidekiq.server?
+ end
+
+ # Returns the sleep interval with a random adjustment.
+ #
+ # The random adjustment is put in place to ensure we:
+ #
+ # 1. Don't generate samples at the exact same interval every time (thus
+ # potentially missing anything that happens in between samples).
+ # 2. Don't sample data at the same interval two times in a row.
+ def sleep_interval
+ while step = @interval_steps.sample
+ if step != @last_step
+ @last_step = step
+
+ return @interval + @last_step
+ end
+ end
+ end
+ end
+ end
+end