diff --git a/Gemfile b/Gemfile
index 23350e3a90707b7b252f17140cd43625216493f9..9c8c5e8b30d7d6f10c63331645c74deae7fc868d 100644
--- a/Gemfile
+++ b/Gemfile
@@ -318,9 +318,6 @@ gem 'peek', '~> 1.1'
 # Snowplow events tracking
 gem 'snowplow-tracker', '~> 0.6.1'
 
-# Memory benchmarks
-gem 'derailed_benchmarks', require: false
-
 # Metrics
 group :metrics do
   gem 'method_source', '~> 0.8', require: false
@@ -413,6 +410,9 @@ group :test do
   gem 'test-prof', '~> 0.10.0'
   gem 'rspec_junit_formatter'
   gem 'guard-rspec'
+
+  # Moved in `test` because https://gitlab.com/gitlab-org/gitlab/-/issues/217527
+  gem 'derailed_benchmarks', require: false
 end
 
 gem 'octokit', '~> 4.15'