From d860ca3f70b2fed4484242e8b3c463eb811cb73d Mon Sep 17 00:00:00 2001
From: Stan Hu <stanhu@gmail.com>
Date: Tue, 13 Feb 2024 05:05:01 -0800
Subject: [PATCH] Add Ruby gem and Gitaly binary cache CI jobs for FIPS
This will allow the FIPS jobs to run without having to reinstall
Ruby gems and recompile Gitaly and Workhorse.
Replace `DEBIAN_VERSION` and hard-coded `debian` values in favor
of `BUILD_OS` and `OS_VERSION` variables.
Simplify the `setup-test-env-fips` CI step now that `libpq-devel` is
now in the UBI image
(https://gitlab.com/gitlab-org/gitlab-build-images/-/merge_requests/775).
Relates to https://gitlab.com/gitlab-org/gitlab/-/issues/441519
---
.gitlab-ci.yml | 5 +++--
.gitlab/ci/as-if-jh.gitlab-ci.yml | 3 ++-
.gitlab/ci/database.gitlab-ci.yml | 2 +-
.gitlab/ci/frontend.gitlab-ci.yml | 2 +-
.gitlab/ci/global.gitlab-ci.yml | 20 +++++++++----------
.gitlab/ci/preflight.gitlab-ci.yml | 2 +-
.gitlab/ci/qa-common/main.gitlab-ci.yml | 2 +-
.gitlab/ci/qa.gitlab-ci.yml | 5 +++--
.gitlab/ci/rails.gitlab-ci.yml | 24 ++++++++++++++++-------
.gitlab/ci/review-apps/qa.gitlab-ci.yml | 2 +-
.gitlab/ci/review.gitlab-ci.yml | 3 ++-
.gitlab/ci/test-on-gdk/main.gitlab-ci.yml | 2 +-
.gitlab/ci/workhorse.gitlab-ci.yml | 7 ++++---
13 files changed, 47 insertions(+), 32 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index d618fa54ae2b..c5259a30075a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -163,7 +163,7 @@ workflow:
variables:
PG_VERSION: "14"
- DEFAULT_CI_IMAGE: "${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}.patched-golang-${GO_VERSION}-rust-${RUST_VERSION}-node-18.17-postgresql-${PG_VERSION}:rubygems-${RUBYGEMS_VERSION}-git-2.36-lfs-2.9-chrome-${CHROME_VERSION}-yarn-1.22-graphicsmagick-1.3.36"
+ DEFAULT_CI_IMAGE: "${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/${BUILD_OS}-${OS_VERSION}-ruby-${RUBY_VERSION}.patched-golang-${GO_VERSION}-rust-${RUST_VERSION}-node-18.17-postgresql-${PG_VERSION}:rubygems-${RUBYGEMS_VERSION}-git-2.36-lfs-2.9-chrome-${CHROME_VERSION}-yarn-1.22-graphicsmagick-1.3.36"
# We set $GITLAB_DEPENDENCY_PROXY to another variable (since it's set at the group level and has higher precedence than .gitlab-ci.yml)
# so that we can override $GITLAB_DEPENDENCY_PROXY_ADDRESS in workflow rules.
GITLAB_DEPENDENCY_PROXY_ADDRESS: "${GITLAB_DEPENDENCY_PROXY}"
@@ -181,7 +181,8 @@ variables:
GET_SOURCES_ATTEMPTS: "3"
# CI_FETCH_REPO_GIT_STRATEGY: "none" is from artifacts. "clone" is from cloning
CI_FETCH_REPO_GIT_STRATEGY: "none"
- DEBIAN_VERSION: "bookworm"
+ BUILD_OS: "debian"
+ OS_VERSION: "bookworm"
UBI_VERSION: "8.6"
CHROME_VERSION: "120"
DOCKER_VERSION: "24.0.5"
diff --git a/.gitlab/ci/as-if-jh.gitlab-ci.yml b/.gitlab/ci/as-if-jh.gitlab-ci.yml
index 6c1c3357089b..823c7b211489 100644
--- a/.gitlab/ci/as-if-jh.gitlab-ci.yml
+++ b/.gitlab/ci/as-if-jh.gitlab-ci.yml
@@ -77,7 +77,8 @@ sync-as-if-jh-branch:
- DEFAULT_CI_IMAGE
- REGISTRY_HOST
- REGISTRY_GROUP
- - DEBIAN_VERSION
+ - BUILD_OS
+ - OS_VERSION
- RUBY_VERSION
- GO_VERSION
- RUST_VERSION
diff --git a/.gitlab/ci/database.gitlab-ci.yml b/.gitlab/ci/database.gitlab-ci.yml
index f54865c2967a..577e0f4fdc08 100644
--- a/.gitlab/ci/database.gitlab-ci.yml
+++ b/.gitlab/ci/database.gitlab-ci.yml
@@ -47,7 +47,7 @@ db:rollback single-db:
db:migrate:multi-version-upgrade-1:
stage: test
- image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}:bundler-${BUNDLER_VERSION}-docker-${DOCKER_VERSION}
+ image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/${BUILD_OS}-${OS_VERSION}-ruby-${RUBY_VERSION}:bundler-${BUNDLER_VERSION}-docker-${DOCKER_VERSION}
extends:
- .db-job-base
- .use-docker-in-docker
diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml
index 22a6b3d4842b..1be57b9374b8 100644
--- a/.gitlab/ci/frontend.gitlab-ci.yml
+++ b/.gitlab/ci/frontend.gitlab-ci.yml
@@ -3,7 +3,7 @@
- .default-retry
- .default-before_script
- .assets-compile-cache
- image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}-node-18.17:rubygems-${RUBYGEMS_VERSION}-git-2.33-lfs-2.9-yarn-1.22-graphicsmagick-1.3.36
+ image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/${BUILD_OS}-${OS_VERSION}-ruby-${RUBY_VERSION}-node-18.17:rubygems-${RUBYGEMS_VERSION}-git-2.33-lfs-2.9-yarn-1.22-graphicsmagick-1.3.36
variables:
SETUP_DB: "false"
WEBPACK_VENDOR_DLL: "true"
diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml
index 09c909627a41..6455de5d2e35 100644
--- a/.gitlab/ci/global.gitlab-ci.yml
+++ b/.gitlab/ci/global.gitlab-ci.yml
@@ -37,7 +37,7 @@
GITLAB_ALLOW_SEPARATE_CI_DATABASE: "true"
.ruby-gems-cache: &ruby-gems-cache
- key: "ruby-gems-debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}"
+ key: "ruby-gems-${BUILD_OS}-${OS_VERSION}-ruby-${RUBY_VERSION}"
paths:
- vendor/ruby/
policy: pull
@@ -47,7 +47,7 @@
policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
.ruby-coverage-gems-cache: &ruby-coverage-gems-cache
- key: "ruby-coverage-gems-debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}"
+ key: "ruby-coverage-gems-${BUILD_OS}-${OS_VERSION}-ruby-${RUBY_VERSION}"
paths:
- vendor/ruby/
policy: pull
@@ -61,7 +61,7 @@
files:
- GITALY_SERVER_VERSION
- lib/gitlab/setup_helper.rb
- prefix: "gitaly-binaries-debian-${DEBIAN_VERSION}"
+ prefix: "gitaly-binaries-${BUILD_OS}-${OS_VERSION}"
paths:
- ${TMP_TEST_FOLDER}/gitaly/_build/bin/
- ${TMP_TEST_FOLDER}/gitaly/_build/deps/git/install/
@@ -76,7 +76,7 @@
policy: pull
.go-pkg-cache: &go-pkg-cache
- key: "go-pkg-${DEBIAN_VERSION}"
+ key: "go-pkg-${BUILD_OS}-${OS_VERSION}"
paths:
- .go/pkg/mod/
policy: pull
@@ -86,7 +86,7 @@
policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
.node-modules-cache: &node-modules-cache
- key: "node-modules-${DEBIAN_VERSION}-${NODE_ENV}"
+ key: "node-modules-${BUILD_OS}-${OS_VERSION}-${NODE_ENV}"
paths:
- node_modules/
- tmp/cache/webpack-dlls/
@@ -97,7 +97,7 @@
policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
.assets-tmp-cache: &assets-tmp-cache
- key: "assets-tmp-debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}-node-${NODE_ENV}-v1"
+ key: "assets-tmp-${BUILD_OS}-${OS_VERSION}-ruby-${RUBY_VERSION}-node-${NODE_ENV}-v1"
paths:
- tmp/cache/assets/sprockets/
- tmp/cache/babel-loader/
@@ -109,7 +109,7 @@
policy: push # We want to rebuild the cache from scratch to ensure we don't pile up outdated cache files.
.storybook-node-modules-cache: &storybook-node-modules-cache
- key: "storybook-node-modules-${DEBIAN_VERSION}-${NODE_ENV}"
+ key: "storybook-node-modules-${BUILD_OS}-${OS_VERSION}-${NODE_ENV}"
paths:
- storybook/node_modules/
policy: pull
@@ -119,7 +119,7 @@
policy: push # We want to rebuild the cache from scratch to ensure stale dependencies are cleaned up.
.rubocop-cache: &rubocop-cache
- key: "rubocop-debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}"
+ key: "rubocop-${BUILD_OS}-${OS_VERSION}-ruby-${RUBY_VERSION}"
paths:
- tmp/rubocop_cache/
policy: pull
@@ -132,7 +132,7 @@
.qa-ruby-gems-cache: &qa-ruby-gems-cache
key:
- prefix: "qa-ruby-gems-debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}"
+ prefix: "qa-ruby-gems-${BUILD_OS}-${OS_VERSION}-ruby-${RUBY_VERSION}"
files:
- qa/Gemfile.lock
paths:
@@ -484,7 +484,7 @@
.use-buildx:
extends: .use-docker-in-docker
- image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-${DEBIAN_VERSION}-slim:docker-${DOCKER_VERSION}
+ image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/${BUILD_OS}-${OS_VERSION}-slim:docker-${DOCKER_VERSION}
variables:
QEMU_IMAGE: tonistiigi/binfmt:qemu-v7.0.0
before_script:
diff --git a/.gitlab/ci/preflight.gitlab-ci.yml b/.gitlab/ci/preflight.gitlab-ci.yml
index 3d9caa74060a..d680006dfe79 100644
--- a/.gitlab/ci/preflight.gitlab-ci.yml
+++ b/.gitlab/ci/preflight.gitlab-ci.yml
@@ -5,7 +5,7 @@
needs: []
.qa-preflight-job:
- image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}:bundler-${BUNDLER_VERSION}-chrome-${CHROME_VERSION}-docker-${DOCKER_VERSION}
+ image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/${BUILD_OS}-${OS_VERSION}-ruby-${RUBY_VERSION}:bundler-${BUNDLER_VERSION}-chrome-${CHROME_VERSION}-docker-${DOCKER_VERSION}
extends:
- .preflight-job-base
- .qa-cache
diff --git a/.gitlab/ci/qa-common/main.gitlab-ci.yml b/.gitlab/ci/qa-common/main.gitlab-ci.yml
index 1895c8574104..159dd7842b01 100644
--- a/.gitlab/ci/qa-common/main.gitlab-ci.yml
+++ b/.gitlab/ci/qa-common/main.gitlab-ci.yml
@@ -36,7 +36,7 @@ stages:
.ruby-image:
# Because this pipeline template can be included directly in other projects,
# image path and registry needs to be defined explicitly
- image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}:bundler-${BUNDLER_VERSION}
+ image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/${BUILD_OS}-${OS_VERSION}-ruby-${RUBY_VERSION}:bundler-${BUNDLER_VERSION}
.bundler-variables:
variables:
diff --git a/.gitlab/ci/qa.gitlab-ci.yml b/.gitlab/ci/qa.gitlab-ci.yml
index 7d0910f7ba59..70308d3cc173 100644
--- a/.gitlab/ci/qa.gitlab-ci.yml
+++ b/.gitlab/ci/qa.gitlab-ci.yml
@@ -1,5 +1,5 @@
.qa-job-base:
- image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}:bundler-${BUNDLER_VERSION}-chrome-${CHROME_VERSION}-docker-${DOCKER_VERSION}
+ image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/${BUILD_OS}-${OS_VERSION}-ruby-${RUBY_VERSION}:bundler-${BUNDLER_VERSION}-chrome-${CHROME_VERSION}-docker-${DOCKER_VERSION}
extends:
- .default-retry
- .qa-cache
@@ -29,7 +29,8 @@
- RUBY_VERSION
- BUNDLER_VERSION
- DOCKER_VERSION
- - DEBIAN_VERSION
+ - BUILD_OS
+ - OS_VERSION
- REGISTRY_GROUP
- REGISTRY_HOST
trigger:
diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml
index ec4dfe797b58..235c4b87710d 100644
--- a/.gitlab/ci/rails.gitlab-ci.yml
+++ b/.gitlab/ci/rails.gitlab-ci.yml
@@ -49,12 +49,10 @@ setup-test-env:
setup-test-env-fips:
extends:
- setup-test-env
- before_script:
- - yum install -y libpq libpq-devel
- - source scripts/prepare_build.sh
- image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/ubi-${UBI_VERSION}-ruby-${RUBY_VERSION}-golang-${GO_VERSION}-rust-${RUST_VERSION}:rubygems-${RUBYGEMS_VERSION}-git-2.36-exiftool-12.60
+ image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/${BUILD_OS}-${OS_VERSION}-ruby-${RUBY_VERSION}-golang-${GO_VERSION}-rust-${RUST_VERSION}:rubygems-${RUBYGEMS_VERSION}-git-2.36-exiftool-12.60
variables:
- DEBIAN_VERSION: ubi-${UBI_VERSION}
+ BUILD_OS: "ubi"
+ OS_VERSION: ${UBI_VERSION}
update-setup-test-env-cache:
extends:
@@ -64,6 +62,12 @@ update-setup-test-env-cache:
artifacts:
paths: [] # This job's purpose is only to update the cache.
+update-setup-test-env-cache-fips:
+ extends:
+ - setup-test-env-fips
+ - .setup-test-env-cache-push
+ - .shared:rules:update-cache
+
update-gitaly-binaries-cache:
extends:
- setup-test-env
@@ -72,6 +76,12 @@ update-gitaly-binaries-cache:
artifacts:
paths: [] # This job's purpose is only to update the cache.
+update-gitaly-binaries-cache-fips:
+ extends:
+ - setup-test-env-fips
+ - .gitaly-binaries-cache-push
+ - .shared:rules:update-gitaly-binaries-cache
+
update-ruby-gems-coverage-cache-push:
extends:
- .ruby-gems-coverage-cache-push
@@ -461,7 +471,7 @@ rspec:artifact-collector ee remainder:
- !reference ['.rails:rules:ee-only-system', rules]
rspec:coverage:
- image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-${DEBIAN_VERSION}-slim-ruby-${RUBY_VERSION}
+ image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/${BUILD_OS}-${OS_VERSION}-slim-ruby-${RUBY_VERSION}
extends:
- .coverage-base
- .rails:rules:rspec-coverage
@@ -512,7 +522,7 @@ rspec:coverage:
path: coverage/coverage.xml
rspec:undercoverage:
- image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-${DEBIAN_VERSION}-slim-ruby-${RUBY_VERSION}
+ image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/${BUILD_OS}-${OS_VERSION}-slim-ruby-${RUBY_VERSION}
extends:
- .coverage-base
- .rails:rules:rspec-undercoverage
diff --git a/.gitlab/ci/review-apps/qa.gitlab-ci.yml b/.gitlab/ci/review-apps/qa.gitlab-ci.yml
index 1aa2272492e3..e8969e35256a 100644
--- a/.gitlab/ci/review-apps/qa.gitlab-ci.yml
+++ b/.gitlab/ci/review-apps/qa.gitlab-ci.yml
@@ -15,7 +15,7 @@ review-qa-smoke:
- .bundle-base
- .default-retry
- .rules:qa-smoke
- image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}:bundler-${BUNDLER_VERSION}-git-2.36-lfs-2.9-chrome-${CHROME_VERSION}-docker-${DOCKER_VERSION}-gcloud-383-kubectl-1.23
+ image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/${BUILD_OS}-${OS_VERSION}-ruby-${RUBY_VERSION}:bundler-${BUNDLER_VERSION}-git-2.36-lfs-2.9-chrome-${CHROME_VERSION}-docker-${DOCKER_VERSION}-gcloud-383-kubectl-1.23
stage: qa
needs: [review-deploy]
variables:
diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml
index 99ff65180c1d..516f0747bf2b 100644
--- a/.gitlab/ci/review.gitlab-ci.yml
+++ b/.gitlab/ci/review.gitlab-ci.yml
@@ -94,7 +94,8 @@ start-review-app-pipeline:
- REVIEW_APPS_GCP_REGION
- REVIEW_APPS_IMAGE
- RUBY_VERSION
- - DEBIAN_VERSION
+ - BUILD_OS
+ - OS_VERSION
- DOCKER_VERSION
- CHROME_VERSION
- BUNDLER_VERSION
diff --git a/.gitlab/ci/test-on-gdk/main.gitlab-ci.yml b/.gitlab/ci/test-on-gdk/main.gitlab-ci.yml
index 58f34b77d0ca..75655b2eb658 100644
--- a/.gitlab/ci/test-on-gdk/main.gitlab-ci.yml
+++ b/.gitlab/ci/test-on-gdk/main.gitlab-ci.yml
@@ -54,7 +54,7 @@ include:
- mv $CI_BUILDS_DIR/*.log $CI_PROJECT_DIR/
.gdk-qa-base:
- image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}:bundler-${BUNDLER_VERSION}-git-2.36-lfs-2.9-chrome-${CHROME_VERSION}-docker-${DOCKER_VERSION}-gcloud-383-kubectl-1.23
+ image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/${BUILD_OS}-${OS_VERSION}-ruby-${RUBY_VERSION}:bundler-${BUNDLER_VERSION}-git-2.36-lfs-2.9-chrome-${CHROME_VERSION}-docker-${DOCKER_VERSION}-gcloud-383-kubectl-1.23
extends:
- .qa-cache
- .default-retry
diff --git a/.gitlab/ci/workhorse.gitlab-ci.yml b/.gitlab/ci/workhorse.gitlab-ci.yml
index b6bbfa512ac5..a58cf96b12cf 100644
--- a/.gitlab/ci/workhorse.gitlab-ci.yml
+++ b/.gitlab/ci/workhorse.gitlab-ci.yml
@@ -10,7 +10,7 @@ workhorse:verify:
.workhorse:test:
extends: .workhorse:rules:workhorse
- image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}-golang-${GO_VERSION}-rust-${RUST_VERSION}:rubygems-${RUBYGEMS_VERSION}-git-2.36-exiftool-12.60
+ image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/${BUILD_OS}-${OS_VERSION}-ruby-${RUBY_VERSION}-golang-${GO_VERSION}-rust-${RUST_VERSION}:rubygems-${RUBYGEMS_VERSION}-git-2.36-exiftool-12.60
services:
- name: redis:${REDIS_VERSION}-alpine
variables:
@@ -51,10 +51,11 @@ workhorse:test fips:
matrix:
- GO_VERSION: ["1.20", "1.21"]
REDIS_VERSION: ["7.0", "6.2"]
- image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/ubi-${UBI_VERSION}-ruby-${RUBY_VERSION}-golang-${GO_VERSION}-rust-${RUST_VERSION}:rubygems-${RUBYGEMS_VERSION}-git-2.36-exiftool-12.60
+ image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/${BUILD_OS}-${OS_VERSION}-ruby-${RUBY_VERSION}-golang-${GO_VERSION}-rust-${RUST_VERSION}:rubygems-${RUBYGEMS_VERSION}-git-2.36-exiftool-12.60
variables:
FIPS_MODE: 1
- DEBIAN_VERSION: ubi-${UBI_VERSION}
+ BUILD_OS: "ubi"
+ OS_VERSION: ${UBI_VERSION}
workhorse:test race:
extends: .workhorse:test
--
GitLab