From 552d38861ad77bf6a64b1e61a91e36fcd26d057c Mon Sep 17 00:00:00 2001
From: Peter Leitzen <pleitzen@gitlab.com>
Date: Thu, 4 Apr 2019 15:38:37 +0000
Subject: [PATCH] Automatically set Prometheus step interval
By computing the step interval passed to the query_range Prometheus API
call we improve the performance on the Prometheus server and GitLab by
reducing the amount of data points sent back and prevent Prometheus
from sending errors when requesting longer intervals.
---
...matically-set-prometheus-step-interval.yml | 5 +++
lib/gitlab/prometheus_client.rb | 32 ++++++++++++++++---
spec/lib/gitlab/prometheus_client_spec.rb | 28 ++++++++++++++++
spec/support/helpers/prometheus_helpers.rb | 10 ++++--
4 files changed, 67 insertions(+), 8 deletions(-)
create mode 100644 changelogs/unreleased/58839-automatically-set-prometheus-step-interval.yml
diff --git a/changelogs/unreleased/58839-automatically-set-prometheus-step-interval.yml b/changelogs/unreleased/58839-automatically-set-prometheus-step-interval.yml
new file mode 100644
index 0000000000000..2c6edf45ae217
--- /dev/null
+++ b/changelogs/unreleased/58839-automatically-set-prometheus-step-interval.yml
@@ -0,0 +1,5 @@
+---
+title: Automatically set Prometheus step interval
+merge_request: 26441
+author:
+type: changed
diff --git a/lib/gitlab/prometheus_client.rb b/lib/gitlab/prometheus_client.rb
index 45828c77a33b5..b4de7cd2bce3e 100644
--- a/lib/gitlab/prometheus_client.rb
+++ b/lib/gitlab/prometheus_client.rb
@@ -6,6 +6,14 @@ class PrometheusClient
Error = Class.new(StandardError)
QueryError = Class.new(Gitlab::PrometheusClient::Error)
+ # Target number of data points for `query_range`.
+ # Please don't exceed the limit of 11000 data points
+ # See https://github.com/prometheus/prometheus/blob/91306bdf24f5395e2601773316945a478b4b263d/web/api/v1/api.go#L347
+ QUERY_RANGE_DATA_POINTS = 600
+
+ # Minimal value of the `step` parameter for `query_range` in seconds.
+ QUERY_RANGE_MIN_STEP = 60
+
attr_reader :rest_client, :headers
def initialize(rest_client)
@@ -23,12 +31,18 @@ def query(query, time: Time.now)
end
def query_range(query, start: 8.hours.ago, stop: Time.now)
+ start = start.to_f
+ stop = stop.to_f
+ step = self.class.compute_step(start, stop)
+
get_result('matrix') do
- json_api_get('query_range',
- query: query,
- start: start.to_f,
- end: stop.to_f,
- step: 1.minute.to_i)
+ json_api_get(
+ 'query_range',
+ query: query,
+ start: start,
+ end: stop,
+ step: step
+ )
end
end
@@ -40,6 +54,14 @@ def series(*matches, start: 8.hours.ago, stop: Time.now)
json_api_get('series', 'match': matches, start: start.to_f, end: stop.to_f)
end
+ def self.compute_step(start, stop)
+ diff = stop - start
+
+ step = (diff / QUERY_RANGE_DATA_POINTS).ceil
+
+ [QUERY_RANGE_MIN_STEP, step].max
+ end
+
private
def json_api_get(type, args = {})
diff --git a/spec/lib/gitlab/prometheus_client_spec.rb b/spec/lib/gitlab/prometheus_client_spec.rb
index 4c3b8deefb913..2517ee71f24b3 100644
--- a/spec/lib/gitlab/prometheus_client_spec.rb
+++ b/spec/lib/gitlab/prometheus_client_spec.rb
@@ -230,4 +230,32 @@
let(:execute_query) { subject.query_range(prometheus_query) }
end
end
+
+ describe '.compute_step' do
+ using RSpec::Parameterized::TableSyntax
+
+ let(:now) { Time.now.utc }
+
+ subject { described_class.compute_step(start, stop) }
+
+ where(:time_interval_in_seconds, :step) do
+ 0 | 60
+ 10.hours | 60
+ 10.hours + 1 | 61
+ # frontend options
+ 30.minutes | 60
+ 3.hours | 60
+ 8.hours | 60
+ 1.day | 144
+ 3.days | 432
+ 1.week | 1008
+ end
+
+ with_them do
+ let(:start) { now - time_interval_in_seconds }
+ let(:stop) { now }
+
+ it { is_expected.to eq(step) }
+ end
+ end
end
diff --git a/spec/support/helpers/prometheus_helpers.rb b/spec/support/helpers/prometheus_helpers.rb
index ce1f9fce10dd5..08d1d7a605915 100644
--- a/spec/support/helpers/prometheus_helpers.rb
+++ b/spec/support/helpers/prometheus_helpers.rb
@@ -25,12 +25,16 @@ def prometheus_query_with_time_url(prometheus_query, time)
"https://prometheus.example.com/api/v1/query?#{query}"
end
- def prometheus_query_range_url(prometheus_query, start: 8.hours.ago, stop: Time.now.to_f)
+ def prometheus_query_range_url(prometheus_query, start: 8.hours.ago, stop: Time.now, step: nil)
+ start = start.to_f
+ stop = stop.to_f
+ step ||= Gitlab::PrometheusClient.compute_step(start, stop)
+
query = {
query: prometheus_query,
- start: start.to_f,
+ start: start,
end: stop,
- step: 1.minute.to_i
+ step: step
}.to_query
"https://prometheus.example.com/api/v1/query_range?#{query}"
--
GitLab