From 6d8cd551702d4b4796e2904f74f318aff98be84c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9my=20Coutable?= <remy@rymai.me>
Date: Thu, 13 Oct 2022 09:35:46 +0100
Subject: [PATCH] ci: Don't use dependency proxy for project access tokens
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Rémy Coutable <remy@rymai.me>
---
.gitlab-ci.yml | 8 ++++++++
.gitlab/ci/_skip.yml | 2 +-
.gitlab/ci/docs.gitlab-ci.yml | 2 +-
.gitlab/ci/frontend.gitlab-ci.yml | 6 +++---
.gitlab/ci/global.gitlab-ci.yml | 2 +-
.gitlab/ci/notify.gitlab-ci.yml | 2 +-
.gitlab/ci/releases.gitlab-ci.yml | 2 +-
.gitlab/ci/review-apps/main.gitlab-ci.yml | 2 +-
.gitlab/ci/setup.gitlab-ci.yml | 6 +++---
.gitlab/ci/test-metadata.gitlab-ci.yml | 2 +-
.gitlab/ci/workhorse.gitlab-ci.yml | 2 +-
doc/development/pipelines.md | 24 +++++++++++++++++++++--
12 files changed, 44 insertions(+), 16 deletions(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 2b3230cbb72c..135315666311 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -56,6 +56,11 @@ workflow:
NOTIFY_PIPELINE_FAILURE_CHANNEL: "f_ruby3"
OMNIBUS_GITLAB_RUBY3_BUILD: "true"
OMNIBUS_GITLAB_CACHE_EDITION: "GITLAB_RUBY3"
+ # This work around https://gitlab.com/gitlab-org/gitlab/-/issues/332411 whichs prevents usage of dependency proxy
+ # when pipeline is triggered by a project access token.
+ - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $GITLAB_USER_LOGIN =~ /project_\d+_bot\d*/'
+ variables:
+ GITLAB_DEPENDENCY_PROXY_ADDRESS: ""
# For `$CI_DEFAULT_BRANCH` branch, create a pipeline (this includes on schedules, pushes, merges, etc.).
- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
# For tags, create a pipeline.
@@ -71,6 +76,9 @@ workflow:
variables:
PG_VERSION: "12"
DEFAULT_CI_IMAGE: "${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/debian-${DEBIAN_VERSION}-ruby-${RUBY_VERSION}.patched-golang-${GO_VERSION}-node-16.14-postgresql-${PG_VERSION}:rubygems-3.2-git-2.36-lfs-2.9-chrome-${CHROME_VERSION}-yarn-1.22-graphicsmagick-1.3.36"
+ # We set $GITLAB_DEPENDENCY_PROXY to another variable (since it's set at the group level and has higher precedence than .gitlab-ci.yml)
+ # so that we can override $GITLAB_DEPENDENCY_PROXY_ADDRESS in workflow rules.
+ GITLAB_DEPENDENCY_PROXY_ADDRESS: "${GITLAB_DEPENDENCY_PROXY}"
RAILS_ENV: "test"
NODE_ENV: "test"
BUNDLE_WITHOUT: "production:development"
diff --git a/.gitlab/ci/_skip.yml b/.gitlab/ci/_skip.yml
index 27a3ff5b836f..9d3745cf2f17 100644
--- a/.gitlab/ci/_skip.yml
+++ b/.gitlab/ci/_skip.yml
@@ -1,7 +1,7 @@
# no-op pipeline template for skipping whole child pipeline execution
no-op:
- image: ${GITLAB_DEPENDENCY_PROXY}alpine:latest
+ image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}alpine:latest
stage: test
variables:
GIT_STRATEGY: none
diff --git a/.gitlab/ci/docs.gitlab-ci.yml b/.gitlab/ci/docs.gitlab-ci.yml
index 7e1571711831..022f1c17a93f 100644
--- a/.gitlab/ci/docs.gitlab-ci.yml
+++ b/.gitlab/ci/docs.gitlab-ci.yml
@@ -2,7 +2,7 @@
extends:
- .default-retry
- .docs:rules:review-docs
- image: ${GITLAB_DEPENDENCY_PROXY}ruby:${RUBY_VERSION}-alpine
+ image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}ruby:${RUBY_VERSION}-alpine
stage: review
needs: []
variables:
diff --git a/.gitlab/ci/frontend.gitlab-ci.yml b/.gitlab/ci/frontend.gitlab-ci.yml
index c64704d5d496..00ac68782e62 100644
--- a/.gitlab/ci/frontend.gitlab-ci.yml
+++ b/.gitlab/ci/frontend.gitlab-ci.yml
@@ -301,17 +301,17 @@ coverage-frontend:
qa-frontend-node:14:
extends: .qa-frontend-node
- image: ${GITLAB_DEPENDENCY_PROXY}node:14
+ image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}node:14
qa-frontend-node:16:
extends: .qa-frontend-node
- image: ${GITLAB_DEPENDENCY_PROXY}node:16
+ image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}node:16
qa-frontend-node:latest:
extends:
- .qa-frontend-node
- .frontend:rules:qa-frontend-node-latest
- image: ${GITLAB_DEPENDENCY_PROXY}node:latest
+ image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}node:latest
webpack-dev-server:
extends:
diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml
index 77e17b2147fb..9be5eb7bcd7b 100644
--- a/.gitlab/ci/global.gitlab-ci.yml
+++ b/.gitlab/ci/global.gitlab-ci.yml
@@ -342,7 +342,7 @@
FOSS_ONLY: '1'
.use-docker-in-docker:
- image: ${GITLAB_DEPENDENCY_PROXY}docker:${DOCKER_VERSION}
+ image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}docker:${DOCKER_VERSION}
services:
- docker:${DOCKER_VERSION}-dind
variables:
diff --git a/.gitlab/ci/notify.gitlab-ci.yml b/.gitlab/ci/notify.gitlab-ci.yml
index 95318d5ce08a..c945d4dc7808 100644
--- a/.gitlab/ci/notify.gitlab-ci.yml
+++ b/.gitlab/ci/notify.gitlab-ci.yml
@@ -1,5 +1,5 @@
.notify-slack:
- image: ${GITLAB_DEPENDENCY_PROXY}alpine
+ image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}alpine
stage: notify
dependencies: []
cache: {}
diff --git a/.gitlab/ci/releases.gitlab-ci.yml b/.gitlab/ci/releases.gitlab-ci.yml
index 77f23814f3c9..df7b07f5545c 100644
--- a/.gitlab/ci/releases.gitlab-ci.yml
+++ b/.gitlab/ci/releases.gitlab-ci.yml
@@ -4,7 +4,7 @@
.merge-train-sync:
# We don't need/want any global before/after commands, so we overwrite these
# settings.
- image: ${GITLAB_DEPENDENCY_PROXY}alpine:edge
+ image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}alpine:edge
stage: sync
before_script:
- apk add --no-cache --update curl bash jq
diff --git a/.gitlab/ci/review-apps/main.gitlab-ci.yml b/.gitlab/ci/review-apps/main.gitlab-ci.yml
index 4c0a3579c928..6be781fb06dc 100644
--- a/.gitlab/ci/review-apps/main.gitlab-ci.yml
+++ b/.gitlab/ci/review-apps/main.gitlab-ci.yml
@@ -32,7 +32,7 @@ review-build-cng-env:
extends:
- .default-retry
- .review:rules:review-build-cng
- image: ${GITLAB_DEPENDENCY_PROXY}ruby:3.0-alpine3.13
+ image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}ruby:3.0-alpine3.13
stage: prepare
needs: []
before_script:
diff --git a/.gitlab/ci/setup.gitlab-ci.yml b/.gitlab/ci/setup.gitlab-ci.yml
index 7f9edd1650a5..e417b054cd69 100644
--- a/.gitlab/ci/setup.gitlab-ci.yml
+++ b/.gitlab/ci/setup.gitlab-ci.yml
@@ -30,7 +30,7 @@ cache gems:
.absolutely-minimal-job:
extends:
- .minimal-job
- image: ${GITLAB_DEPENDENCY_PROXY}alpine:edge
+ image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}alpine:edge
variables:
GIT_STRATEGY: none
@@ -79,7 +79,7 @@ verify-ruby-2.7:
verify-tests-yml:
extends:
- .setup:rules:verify-tests-yml
- image: ${GITLAB_DEPENDENCY_PROXY}ruby:${RUBY_VERSION}-alpine3.13
+ image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}ruby:${RUBY_VERSION}-alpine3.13
stage: test
needs: []
script:
@@ -116,7 +116,7 @@ generate-frontend-fixtures-mapping:
detect-tests:
extends: .rails:rules:detect-tests
- image: ${GITLAB_DEPENDENCY_PROXY}ruby:${RUBY_VERSION}
+ image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}ruby:${RUBY_VERSION}
needs: []
stage: prepare
variables:
diff --git a/.gitlab/ci/test-metadata.gitlab-ci.yml b/.gitlab/ci/test-metadata.gitlab-ci.yml
index f4fa39300b6d..e147305e25a3 100644
--- a/.gitlab/ci/test-metadata.gitlab-ci.yml
+++ b/.gitlab/ci/test-metadata.gitlab-ci.yml
@@ -1,5 +1,5 @@
.tests-metadata-state:
- image: ${GITLAB_DEPENDENCY_PROXY}ruby:${RUBY_VERSION}
+ image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}ruby:${RUBY_VERSION}
before_script:
- source scripts/utils.sh
artifacts:
diff --git a/.gitlab/ci/workhorse.gitlab-ci.yml b/.gitlab/ci/workhorse.gitlab-ci.yml
index 4ed674948cf8..a11d5f000cff 100644
--- a/.gitlab/ci/workhorse.gitlab-ci.yml
+++ b/.gitlab/ci/workhorse.gitlab-ci.yml
@@ -1,6 +1,6 @@
workhorse:verify:
extends: .workhorse:rules:workhorse
- image: ${GITLAB_DEPENDENCY_PROXY}golang:${GO_VERSION}
+ image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}golang:${GO_VERSION}
stage: test
needs: []
script:
diff --git a/doc/development/pipelines.md b/doc/development/pipelines.md
index 130fa17c73fd..debb33a28544 100644
--- a/doc/development/pipelines.md
+++ b/doc/development/pipelines.md
@@ -587,8 +587,9 @@ The current stages are:
### Dependency Proxy
Some of the jobs are using images from Docker Hub, where we also use
-`${GITLAB_DEPENDENCY_PROXY}` as a prefix to the image path, so that we pull
+`${GITLAB_DEPENDENCY_PROXY_ADDRESS}` as a prefix to the image path, so that we pull
images from our [Dependency Proxy](../user/packages/dependency_proxy/index.md).
+By default, this variable is set from the value of `${GITLAB_DEPENDENCY_PROXY}`.
`${GITLAB_DEPENDENCY_PROXY}` is a group CI/CD variable defined in
[`gitlab-org`](https://gitlab.com/gitlab-org) as
@@ -596,13 +597,32 @@ images from our [Dependency Proxy](../user/packages/dependency_proxy/index.md).
defined as:
```yaml
-image: ${GITLAB_DEPENDENCY_PROXY}alpine:edge
+image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}alpine:edge
```
Projects in the `gitlab-org` group pull from the Dependency Proxy, while
forks that reside on any other personal namespaces or groups fall back to
Docker Hub unless `${GITLAB_DEPENDENCY_PROXY}` is also defined there.
+#### Work around for when a pipeline is started by a Project access token user
+
+When a pipeline is started by a Project access token user (e.g. the `release-tools approver bot` user which
+automatically updates the Gitaly version used in the main project),
+[the Dependency proxy isn't accessible](https://gitlab.com/gitlab-org/gitlab/-/issues/332411#note_1130388163)
+and the job fails at the `Preparing the "docker+machine" executor` step.
+To work around that, we have a special workflow rule, that overrides the
+`${GITLAB_DEPENDENCY_PROXY_ADDRESS}` variable so that Depdendency proxy isn't used in that case:
+
+```yaml
+- if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $GITLAB_USER_LOGIN =~ /project_\d+_bot\d*/'
+ variables:
+ GITLAB_DEPENDENCY_PROXY_ADDRESS: ""
+```
+
+NOTE:
+We don't directly override the `${GITLAB_DEPENDENCY_PROXY}` variable because group-level
+variables have higher precedence over `.gitlab-ci.yml` variables.
+
### Common job definitions
Most of the jobs [extend from a few CI definitions](../ci/yaml/index.md#extends)
--
GitLab