From 2ed9bedebc725ad8e755de9a4f001e91f7507b9a Mon Sep 17 00:00:00 2001
From: Kev Kloss <kkloss@gitlab.com>
Date: Mon, 27 Jan 2025 18:37:58 +0100
Subject: [PATCH] Cache database once per pipeline
Every RSpec CI job uses the `setup_db` to set up a Postgres test
database. This takes about 1m 10s because the database has to be created
and then all migrations have to be applied.
In a pipeline that runs 150 RSpec jobs, this amounts to 3 hours of
work that is needlessly ran 150 times instead of once.
Therefore, this adds a CI job `db:setup pg14` that sets up a database
and makes it available as dump (using `pg_dumpall`) to all RSpec jobs
that test using Postgres 14.
By doing this, and applying the dump in each job with `psql`, we'll
reliably and (hopefully) permanently save about 50 seconds per RSpec
job, or about 3% of total job duration.
---
.gitlab/ci/database.gitlab-ci.yml | 15 +++++++++++++++
.gitlab/ci/rails/shared.gitlab-ci.yml | 4 ++++
scripts/utils.sh | 10 ++++++++++
3 files changed, 29 insertions(+)
diff --git a/.gitlab/ci/database.gitlab-ci.yml b/.gitlab/ci/database.gitlab-ci.yml
index 39065d1c9db1e..7806cc15b1837 100644
--- a/.gitlab/ci/database.gitlab-ci.yml
+++ b/.gitlab/ci/database.gitlab-ci.yml
@@ -1,6 +1,21 @@
include:
- local: .gitlab/ci/rails/shared.gitlab-ci.yml
+db:setup pg14:
+ stage: prepare
+ needs: []
+ extends:
+ - .use-pg14
+ - .default-before_script
+ - .ruby-cache
+ - .rails:rules:setup-test-env
+ script:
+ - source scripts/utils.sh
+ - run_timed_command "pg_dumpall -h postgres -U postgres > pg_dumpall.sql"
+ artifacts:
+ paths:
+ - pg_dumpall.sql
+
db:rollback single-db-ci-connection:
extends:
- db:rollback
diff --git a/.gitlab/ci/rails/shared.gitlab-ci.yml b/.gitlab/ci/rails/shared.gitlab-ci.yml
index b7cb464675fb5..ddbbbac9b00e3 100644
--- a/.gitlab/ci/rails/shared.gitlab-ci.yml
+++ b/.gitlab/ci/rails/shared.gitlab-ci.yml
@@ -210,6 +210,10 @@ include:
- !reference [.rspec-base, after_script]
.rspec-base-pg14:
+ needs:
+ - !reference [.rspec-base, needs]
+ - job: "db:setup pg14"
+ optional: true
extends:
- .rspec-base
- .use-pg14
diff --git a/scripts/utils.sh b/scripts/utils.sh
index e93fdb96b4db0..d8b2ff466dd0f 100644
--- a/scripts/utils.sh
+++ b/scripts/utils.sh
@@ -198,6 +198,16 @@ function setup_db_praefect() {
function setup_db() {
section_start "setup-db" "Setting up DBs"
+ if [ -f pg_dumpall.sql ]; then
+ echo "Found pg_dumpall.sql, applying!"
+
+ psql -h postgres -U postgres -q < pg_dumpall.sql > /dev/null
+ rm pg_dumpall.sql
+
+ section_end "setup-db"
+ return 0
+ fi
+
setup_db_user_only
run_timed_command_with_metric "bundle exec rake db:drop db:create db:schema:load db:migrate gitlab:db:lock_writes" "setup_db"
setup_db_praefect
--
GitLab