From c44fd5146dc405c7b1f6dd0c0541e78a7ff2bf29 Mon Sep 17 00:00:00 2001
From: Lin Jen-Shin <jen-shin@gitlab.com>
Date: Fri, 22 Dec 2023 20:43:34 +0800
Subject: [PATCH] Document .repo-from-artifacts and CI_FETCH_REPO_GIT_STRATEGY
---
.gitlab-ci.yml | 1 +
.gitlab/ci/rails/shared.gitlab-ci.yml | 2 +-
doc/development/pipelines/internals.md | 1 +
doc/development/pipelines/performance.md | 39 ++++++++++++++++++++++++
4 files changed, 42 insertions(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index bc07e5fea9110..cdeadda887ae8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -155,6 +155,7 @@ variables:
GIT_STRATEGY: "clone"
GIT_SUBMODULE_STRATEGY: "none"
GET_SOURCES_ATTEMPTS: "3"
+ # CI_FETCH_REPO_GIT_STRATEGY: "none" is from artifacts. "clone" is from cloning
CI_FETCH_REPO_GIT_STRATEGY: "none"
DEBIAN_VERSION: "bullseye"
UBI_VERSION: "8.6"
diff --git a/.gitlab/ci/rails/shared.gitlab-ci.yml b/.gitlab/ci/rails/shared.gitlab-ci.yml
index aa5b7c33385e4..1ad1f60820a11 100644
--- a/.gitlab/ci/rails/shared.gitlab-ci.yml
+++ b/.gitlab/ci/rails/shared.gitlab-ci.yml
@@ -67,7 +67,7 @@ include:
extends:
- .rails-job-base
- .base-artifacts
- - .repo-from-artifacts # Comment this to clone instead of using artifacts
+ - .repo-from-artifacts
stage: test
variables:
RUBY_GC_MALLOC_LIMIT: 67108864
diff --git a/doc/development/pipelines/internals.md b/doc/development/pipelines/internals.md
index 1fdb2014c1fa5..b3b4cbec02d27 100644
--- a/doc/development/pipelines/internals.md
+++ b/doc/development/pipelines/internals.md
@@ -149,6 +149,7 @@ that are scoped to a single [configuration keyword](../../ci/yaml/index.md#job-k
|------------------|-------------|
| `.default-retry` | Allows a job to [retry](../../ci/yaml/index.md#retry) upon `unknown_failure`, `api_failure`, `runner_system_failure`, `job_execution_timeout`, or `stuck_or_timeout_failure`. |
| `.default-before_script` | Allows a job to use a default `before_script` definition suitable for Ruby/Rails tasks that may need a database running (for example, tests). |
+| `.repo-from-artifacts` | Allows a job to fetch the repository from artifacts in `clone-gitlab-repo` instead of cloning. This should reduce GitLab.com Gitaly load and also slightly improve the speed because downloading from artifacts is faster than cloning. Note that this should be avoided to be used with jobs having `needs: []` because otherwise it'll start later and we normally want all jobs to start as soon as possible. Use this only on jobs which has other dependencies so that we don't wait longer than just cloning. Note that this behavior can be controlled via `CI_FETCH_REPO_GIT_STRATEGY`. See [Fetch repository via artifacts instead of cloning/fetching from Gitaly](performance.md#fetch-repository-via-artifacts-instead-of-cloningfetching-from-gitaly) for more details. |
| `.setup-test-env-cache` | Allows a job to use a default `cache` definition suitable for setting up test environment for subsequent Ruby/Rails tasks. |
| `.ruby-cache` | Allows a job to use a default `cache` definition suitable for Ruby tasks. |
| `.static-analysis-cache` | Allows a job to use a default `cache` definition suitable for static analysis tasks. |
diff --git a/doc/development/pipelines/performance.md b/doc/development/pipelines/performance.md
index 0cd4f4270fe71..d9019e6053bb8 100644
--- a/doc/development/pipelines/performance.md
+++ b/doc/development/pipelines/performance.md
@@ -29,6 +29,45 @@ This works well for the following reasons:
- We use [shallow clone](../../ci/pipelines/settings.md#limit-the-number-of-changes-fetched-during-clone) to avoid downloading the full Git
history for every job.
+### Fetch repository via artifacts instead of cloning/fetching from Gitaly
+
+Lately we see errors from Gitaly look like this: (see [the issue](https://gitlab.com/gitlab-org/gitlab/-/issues/435456))
+
+```plaintext
+fatal: remote error: GitLab is currently unable to handle this request due to load.
+```
+
+While GitLab.com uses [pack-objects cache](../../administration/gitaly/configure_gitaly.md#pack-objects-cache),
+sometimes the load is still too heavy for Gitaly to handle, and
+[thundering herds](https://gitlab.com/gitlab-org/gitlab/-/issues/423830) can
+also be a concern that we have a lot of jobs cloning the repository around
+the same time.
+
+To mitigate and reduce loads for Gitaly, we changed some jobs to fetch the
+repository from artifacts in a job instead of all cloning from Gitaly at once.
+
+For now this applies to most of the RSpec jobs, which has the most concurrent
+jobs in most pipelines. This also slightly improved the speed because fetching
+from the artifacts is also slightly faster than cloning, at the cost of saving
+more artifacts for each pipeline.
+
+Based on the numbers on 2023-12-20 at [Fetch repo from artifacts for RSpec jobs](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/140330),
+the extra storage cost was about 280M for each pipeline, and we save 15 seconds
+for each RSpec jobs.
+
+We do not apply this to jobs having no other job dependencies because we don't
+want to delay any jobs from starting.
+
+This behavior can be controlled by variable `CI_FETCH_REPO_GIT_STRATEGY`:
+
+- Set to `none` means jobs using `.repo-from-artifacts` fetch repository from
+ artifacts in job `clone-gitlab-repo` rather than cloning.
+- Set to `clone` means jobs using `.repo-from-artifacts` clone repository
+ as usual. Job `clone-gitlab-repo` does not run in this case.
+
+To disable it, set `CI_FETCH_REPO_GIT_STRATEGY` to `clone`. To enable it,
+set `CI_FETCH_REPO_GIT_STRATEGY` to `none`.
+
## Caching strategy
1. All jobs must only pull caches by default.
--
GitLab