From 65fe7827072d005ae6f44ce62bcf154f03007cb6 Mon Sep 17 00:00:00 2001
From: Andrejs Cunskis <acunskis@gitlab.com>
Date: Fri, 25 Oct 2024 17:04:41 +0000
Subject: [PATCH] Create optimised resource config for parallel tests
Remove maxReplicas values
Rename constant
Update class doc comment
Fix spec name
---
.../gitlab/cng/lib/deployment/installation.rb | 1 +
.../cng/lib/deployment/resource_presets.rb | 153 +++++++++++++++++
.../lib/gitlab/cng/lib/kind/cluster.rb | 1 +
.../cng/deployment/installation_spec.rb | 8 +-
.../cng/deployment/resource_presets_spec.rb | 154 ++++++++++++++++++
.../spec/unit/gitlab/cng/kind/cluster_spec.rb | 4 +-
6 files changed, 318 insertions(+), 3 deletions(-)
create mode 100644 qa/gems/gitlab-cng/lib/gitlab/cng/lib/deployment/resource_presets.rb
create mode 100644 qa/gems/gitlab-cng/spec/unit/gitlab/cng/deployment/resource_presets_spec.rb
diff --git a/qa/gems/gitlab-cng/lib/gitlab/cng/lib/deployment/installation.rb b/qa/gems/gitlab-cng/lib/gitlab/cng/lib/deployment/installation.rb
index ac3e09e985291..79ecf04898630 100644
--- a/qa/gems/gitlab-cng/lib/gitlab/cng/lib/deployment/installation.rb
+++ b/qa/gems/gitlab-cng/lib/gitlab/cng/lib/deployment/installation.rb
@@ -189,6 +189,7 @@ def run_deploy(chart_reference)
.deep_merge(license_values)
.deep_merge(env_values)
.deep_merge(configuration.values)
+ .deep_merge(ResourcePresets.resource_values(ci ? ResourcePresets::HIGH : ResourcePresets::DEFAULT))
.deep_stringify_keys
.to_yaml
diff --git a/qa/gems/gitlab-cng/lib/gitlab/cng/lib/deployment/resource_presets.rb b/qa/gems/gitlab-cng/lib/gitlab/cng/lib/deployment/resource_presets.rb
new file mode 100644
index 0000000000000..15a4f326074b1
--- /dev/null
+++ b/qa/gems/gitlab-cng/lib/gitlab/cng/lib/deployment/resource_presets.rb
@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module Cng
+ module Deployment
+ # Kubernetes resource request/limit presets optimised for different usecases
+ #
+ class ResourcePresets
+ DEFAULT = "default"
+ HIGH = "high"
+
+ class << self
+ # Kubernetes resources values for given preset
+ #
+ # @param [String] preset_name
+ # @return [Hash]
+ def resource_values(preset_name)
+ presets.fetch(preset_name)
+ end
+
+ private
+
+ # Different resources presets and replicas count
+ #
+ # Prefer vertical scaling over hpa for test stability
+ # Waiting for new pods to scale will lead to test flakiness and makes log reading harder
+ #
+ # @return [Hash]
+ def presets
+ @presets ||= {
+ # Default preset for local deployments
+ DEFAULT => {
+ gitlab: {
+ webservice: {
+ workerProcesses: 2,
+ minReplicas: 1,
+ resources: resources("1500m", "3Gi")
+ },
+ sidekiq: {
+ concurrency: 20,
+ minReplicas: 1,
+ resources: resources("900m", "1.6Gi"),
+ hpa: {
+ cpu: { targetAverageValue: "800m" }
+ }
+ },
+ kas: {
+ minReplicas: 1,
+ resources: resources("10m", "45Mi")
+ },
+ gitlab_shell: {
+ minReplicas: 1,
+ resources: resources("80m", "16Mi")
+ },
+ gitaly: {
+ resources: resources("300m", "300Mi")
+ }
+ },
+ registry: {
+ resources: resources("40m", "20Mi"),
+ hpa: {
+ minReplicas: 1,
+ **cpu_utilization
+ }
+ },
+ minio: {
+ resources: resources("9m", "128Mi")
+ }
+ },
+ # This preset is optimised for running e2e tests in parallel
+ HIGH => {
+ gitlab: {
+ webservice: {
+ workerProcesses: 4,
+ minReplicas: 1,
+ resources: resources(3, "4.5Gi"),
+ hpa: cpu_utilization
+ },
+ sidekiq: {
+ concurrency: 30,
+ minReplicas: 1,
+ resources: resources("1200m", "2Gi"),
+ hpa: cpu_utilization
+ },
+ kas: {
+ minReplicas: 1,
+ resources: resources("40m", "64Mi"),
+ hpa: cpu_utilization
+ },
+ gitlab_shell: {
+ minReplicas: 1,
+ resources: resources("24m", "32Mi"),
+ hpa: cpu_utilization
+ },
+ gitaly: {
+ resources: resources("450m", "450Mi")
+ }
+ },
+ registry: {
+ resources: resources("50m", "32Mi"),
+ hpa: {
+ minReplicas: 1,
+ **cpu_utilization
+ }
+ },
+ minio: {
+ resources: resources("15m", "256Mi")
+ }
+ }
+ }
+ end
+
+ # Kubernetes resources configuration
+ #
+ # Set limits equal to requests by default for simplicity
+ #
+ # @param [<String, Integer>] cpu_r
+ # @param [String] memory_r
+ # @param [<String, Integer>] cpu_l
+ # @param [String] memory_l
+ # @return [Hash]
+ def resources(cpu_r, memory_r, cpu_l = nil, memory_l = nil)
+ cpu_l ||= cpu_r
+ memory_l ||= memory_r
+
+ {
+ requests: {
+ cpu: cpu_r,
+ memory: memory_r
+ },
+ limits: {
+ cpu: cpu_l,
+ memory: memory_l
+ }
+ }
+ end
+
+ # Common hpa cpu utilization config
+ #
+ # @return [Hash]
+ def cpu_utilization
+ @cpu_utilization ||= {
+ cpu: {
+ targetType: "Utilization",
+ targetAverageUtilization: 90
+ }
+ }
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/qa/gems/gitlab-cng/lib/gitlab/cng/lib/kind/cluster.rb b/qa/gems/gitlab-cng/lib/gitlab/cng/lib/kind/cluster.rb
index 563adef31b041..ea8062f8d9cfd 100644
--- a/qa/gems/gitlab-cng/lib/gitlab/cng/lib/kind/cluster.rb
+++ b/qa/gems/gitlab-cng/lib/gitlab/cng/lib/kind/cluster.rb
@@ -48,6 +48,7 @@ def create
create_cluster
update_server_url
+ install_metrics_server
log("Cluster '#{name}' created", :success)
rescue Helpers::Shell::CommandFailure
# Exit cleanly without stacktrace if shell command fails
diff --git a/qa/gems/gitlab-cng/spec/unit/gitlab/cng/deployment/installation_spec.rb b/qa/gems/gitlab-cng/spec/unit/gitlab/cng/deployment/installation_spec.rb
index b1ffb7e90bfb4..56235a93450ef 100644
--- a/qa/gems/gitlab-cng/spec/unit/gitlab/cng/deployment/installation_spec.rb
+++ b/qa/gems/gitlab-cng/spec/unit/gitlab/cng/deployment/installation_spec.rb
@@ -41,6 +41,12 @@
)
end
+ let(:resources_values) do
+ Gitlab::Cng::Deployment::ResourcePresets.resource_values(
+ ci ? Gitlab::Cng::Deployment::ResourcePresets::HIGH : Gitlab::Cng::Deployment::ResourcePresets::DEFAULT
+ )
+ end
+
let(:expected_values_yml) do
{
global: {
@@ -55,7 +61,7 @@
license: { secret: "gitlab-license" }
},
**config_values
- }.deep_stringify_keys.to_yaml
+ }.deep_merge(resources_values).deep_stringify_keys.to_yaml
end
before do
diff --git a/qa/gems/gitlab-cng/spec/unit/gitlab/cng/deployment/resource_presets_spec.rb b/qa/gems/gitlab-cng/spec/unit/gitlab/cng/deployment/resource_presets_spec.rb
new file mode 100644
index 0000000000000..9b53282dcaff7
--- /dev/null
+++ b/qa/gems/gitlab-cng/spec/unit/gitlab/cng/deployment/resource_presets_spec.rb
@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+RSpec.describe Gitlab::Cng::Deployment::ResourcePresets do
+ it "returns default resources values preset" do
+ expect(described_class.resource_values(described_class::DEFAULT)).to eq({
+ gitlab: {
+ webservice: {
+ workerProcesses: 2,
+ minReplicas: 1,
+ resources: {
+ requests: { cpu: "1500m", memory: "3Gi" },
+ limits: { cpu: "1500m", memory: "3Gi" }
+ }
+ },
+ sidekiq: {
+ concurrency: 20,
+ minReplicas: 1,
+ resources: {
+ requests: { cpu: "900m", memory: "1.6Gi" },
+ limits: { cpu: "900m", memory: "1.6Gi" }
+ },
+ hpa: {
+ cpu: { targetAverageValue: "800m" }
+ }
+ },
+ kas: {
+ minReplicas: 1,
+ resources: {
+ requests: { cpu: "10m", memory: "45Mi" },
+ limits: { cpu: "10m", memory: "45Mi" }
+ }
+ },
+ gitlab_shell: {
+ minReplicas: 1,
+ resources: {
+ requests: { cpu: "80m", memory: "16Mi" },
+ limits: { cpu: "80m", memory: "16Mi" }
+ }
+ },
+ gitaly: {
+ resources: {
+ requests: { cpu: "300m", memory: "300Mi" },
+ limits: { cpu: "300m", memory: "300Mi" }
+ }
+ }
+ },
+ registry: {
+ resources: {
+ requests: { cpu: "40m", memory: "20Mi" },
+ limits: { cpu: "40m", memory: "20Mi" }
+ },
+ hpa: {
+ minReplicas: 1,
+ cpu: {
+ targetType: "Utilization",
+ targetAverageUtilization: 90
+ }
+ }
+ },
+ minio: {
+ resources: {
+ requests: { cpu: "9m", memory: "128Mi" },
+ limits: { cpu: "9m", memory: "128Mi" }
+ }
+ }
+ })
+ end
+
+ it "returns high resources values preset" do
+ expect(described_class.resource_values(described_class::HIGH)).to eq({
+ gitlab: {
+ webservice: {
+ workerProcesses: 4,
+ minReplicas: 1,
+ resources: {
+ requests: { cpu: 3, memory: "4.5Gi" },
+ limits: { cpu: 3, memory: "4.5Gi" }
+ },
+ hpa: {
+ cpu: {
+ targetType: "Utilization",
+ targetAverageUtilization: 90
+ }
+ }
+ },
+ sidekiq: {
+ concurrency: 30,
+ minReplicas: 1,
+ resources: {
+ requests: { cpu: "1200m", memory: "2Gi" },
+ limits: { cpu: "1200m", memory: "2Gi" }
+ },
+ hpa: {
+ cpu: {
+ targetType: "Utilization",
+ targetAverageUtilization: 90
+ }
+ }
+ },
+ kas: {
+ minReplicas: 1,
+ resources: {
+ requests: { cpu: "40m", memory: "64Mi" },
+ limits: { cpu: "40m", memory: "64Mi" }
+ },
+ hpa: {
+ cpu: {
+ targetType: "Utilization",
+ targetAverageUtilization: 90
+ }
+ }
+ },
+ gitlab_shell: {
+ minReplicas: 1,
+ resources: {
+ requests: { cpu: "24m", memory: "32Mi" },
+ limits: { cpu: "24m", memory: "32Mi" }
+ },
+ hpa: {
+ cpu: {
+ targetType: "Utilization",
+ targetAverageUtilization: 90
+ }
+ }
+ },
+ gitaly: {
+ resources: {
+ requests: { cpu: "450m", memory: "450Mi" },
+ limits: { cpu: "450m", memory: "450Mi" }
+ }
+ }
+ },
+ registry: {
+ resources: {
+ requests: { cpu: "50m", memory: "32Mi" },
+ limits: { cpu: "50m", memory: "32Mi" }
+ },
+ hpa: {
+ minReplicas: 1,
+ cpu: {
+ targetType: "Utilization",
+ targetAverageUtilization: 90
+ }
+ }
+ },
+ minio: {
+ resources: {
+ requests: { cpu: "15m", memory: "256Mi" },
+ limits: { cpu: "15m", memory: "256Mi" }
+ }
+ }
+ })
+ end
+end
diff --git a/qa/gems/gitlab-cng/spec/unit/gitlab/cng/kind/cluster_spec.rb b/qa/gems/gitlab-cng/spec/unit/gitlab/cng/kind/cluster_spec.rb
index 1430fc1368203..d365febd9138b 100644
--- a/qa/gems/gitlab-cng/spec/unit/gitlab/cng/kind/cluster_spec.rb
+++ b/qa/gems/gitlab-cng/spec/unit/gitlab/cng/kind/cluster_spec.rb
@@ -84,11 +84,11 @@
it "creates cluster with ci specific configuration", :aggregate_failures do
expect { cluster.create }.to output(/Cluster '#{name}' created/).to_stdout
- expect(helm).not_to have_received(:add_helm_chart).with(
+ expect(helm).to have_received(:add_helm_chart).with(
"metrics-server",
"https://kubernetes-sigs.github.io/metrics-server/"
)
- expect(helm).not_to have_received(:upgrade).with(
+ expect(helm).to have_received(:upgrade).with(
"metrics-server",
"metrics-server/metrics-server",
namespace: "kube-system",
--
GitLab