From 73b00ad42166cde48ff9df2c6c5110aede1eb43e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9my=20Coutable?= <remy@rymai.me>
Date: Mon, 31 Jan 2022 17:45:33 +0100
Subject: [PATCH] ci: Use the trigger keyword for CNG downstream pipelines
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Rémy Coutable <remy@rymai.me>
---
.gitlab/ci/cng.gitlab-ci.yml | 53 ++++++++--
.gitlab/ci/docs.gitlab-ci.yml | 4 +-
.gitlab/ci/qa.gitlab-ci.yml | 4 +-
.gitlab/ci/rails.gitlab-ci.yml | 2 +-
.gitlab/ci/review-apps/main.gitlab-ci.yml | 48 ++++++++-
.gitlab/ci/rules.gitlab-ci.yml | 2 +-
doc/development/documentation/review_apps.md | 2 +-
scripts/{trigger-build => trigger-build.rb} | 100 +++++++++++--------
8 files changed, 153 insertions(+), 62 deletions(-)
rename scripts/{trigger-build => trigger-build.rb} (88%)
diff --git a/.gitlab/ci/cng.gitlab-ci.yml b/.gitlab/ci/cng.gitlab-ci.yml
index bf439288be29a..d720ec5ae452a 100644
--- a/.gitlab/ci/cng.gitlab-ci.yml
+++ b/.gitlab/ci/cng.gitlab-ci.yml
@@ -1,10 +1,51 @@
+cloud-native-image-env:
+ extends:
+ - .default-retry
+ - .cng:rules
+ image: ${GITLAB_DEPENDENCY_PROXY}ruby:2.7-alpine3.13
+ stage: post-test
+ before_script:
+ - source ./scripts/utils.sh
+ - install_gitlab_gem
+ script:
+ - 'ruby -r./scripts/trigger-build.rb -e "puts Trigger.variables_for_env_file(Trigger::CNG.new.variables)" > build.env'
+ - cat build.env
+ artifacts:
+ reports:
+ dotenv: build.env
+ paths:
+ - build.env
+ expire_in: 7 days
+ when: always
+
cloud-native-image:
extends: .cng:rules
- image: ${GITLAB_DEPENDENCY_PROXY}ruby:2.7-alpine
- dependencies: []
stage: post-test
+ needs: ["cloud-native-image-env"]
+ inherit:
+ variables: false
variables:
- GIT_DEPTH: "1"
- script:
- - install_gitlab_gem
- - ./scripts/trigger-build cng
+ TOP_UPSTREAM_SOURCE_PROJECT: "${TOP_UPSTREAM_SOURCE_PROJECT}"
+ TOP_UPSTREAM_SOURCE_REF: "${TOP_UPSTREAM_SOURCE_REF}"
+ TOP_UPSTREAM_SOURCE_JOB: "${TOP_UPSTREAM_SOURCE_JOB}"
+ TOP_UPSTREAM_SOURCE_SHA: "${TOP_UPSTREAM_SOURCE_SHA}"
+ TOP_UPSTREAM_MERGE_REQUEST_PROJECT_ID: "${TOP_UPSTREAM_MERGE_REQUEST_PROJECT_ID}"
+ TOP_UPSTREAM_MERGE_REQUEST_IID: "${TOP_UPSTREAM_MERGE_REQUEST_IID}"
+ GITLAB_REF_SLUG: "${GITLAB_REF_SLUG}"
+ # CNG pipeline specific variables
+ GITLAB_VERSION: "${GITLAB_VERSION}"
+ GITLAB_TAG: "${GITLAB_TAG}"
+ GITLAB_ASSETS_TAG: "${GITLAB_ASSETS_TAG}"
+ FORCE_RAILS_IMAGE_BUILDS: "${FORCE_RAILS_IMAGE_BUILDS}"
+ CE_PIPELINE: "${CE_PIPELINE}" # Based on https://docs.gitlab.com/ee/ci/jobs/job_control.html#check-if-a-variable-exists, `if: '$CE_PIPELINE'` will evaluate to `false` when this variable is empty
+ EE_PIPELINE: "${EE_PIPELINE}" # Based on https://docs.gitlab.com/ee/ci/jobs/job_control.html#check-if-a-variable-exists, `if: '$EE_PIPELINE'` will evaluate to `false` when this variable is empty
+ GITLAB_SHELL_VERSION: "${GITLAB_SHELL_VERSION}"
+ GITLAB_ELASTICSEARCH_INDEXER_VERSION: "${GITLAB_ELASTICSEARCH_INDEXER_VERSION}"
+ GITLAB_KAS_VERSION: "${GITLAB_KAS_VERSION}"
+ GITLAB_WORKHORSE_VERSION: "${GITLAB_WORKHORSE_VERSION}"
+ GITLAB_PAGES_VERSION: "${GITLAB_PAGES_VERSION}"
+ GITALY_SERVER_VERSION: "${GITALY_SERVER_VERSION}"
+ trigger:
+ project: gitlab-org/build/CNG
+ branch: $TRIGGER_BRANCH
+ strategy: depend
diff --git a/.gitlab/ci/docs.gitlab-ci.yml b/.gitlab/ci/docs.gitlab-ci.yml
index 796dd26182497..8b7691045cbbb 100644
--- a/.gitlab/ci/docs.gitlab-ci.yml
+++ b/.gitlab/ci/docs.gitlab-ci.yml
@@ -28,7 +28,7 @@
review-docs-deploy:
extends: .review-docs
script:
- - ./scripts/trigger-build docs deploy
+ - ./scripts/trigger-build.rb docs deploy
# Cleanup remote environment of gitlab-docs
review-docs-cleanup:
@@ -37,7 +37,7 @@ review-docs-cleanup:
name: review-docs/mr-${CI_MERGE_REQUEST_IID}
action: stop
script:
- - ./scripts/trigger-build docs cleanup
+ - ./scripts/trigger-build.rb docs cleanup
docs-lint markdown:
extends:
diff --git a/.gitlab/ci/qa.gitlab-ci.yml b/.gitlab/ci/qa.gitlab-ci.yml
index b12f76f2823e2..77d12a3e848ed 100644
--- a/.gitlab/ci/qa.gitlab-ci.yml
+++ b/.gitlab/ci/qa.gitlab-ci.yml
@@ -73,7 +73,7 @@ update-qa-cache:
- echo $exit_code
- |
if [ $exit_code -eq 0 ]; then
- ./scripts/trigger-build omnibus
+ ./scripts/trigger-build.rb omnibus
elif [ $exit_code -eq 1 ]; then
exit 1
else
@@ -108,7 +108,7 @@ update-qa-cache:
if [[ $feature_flags ]]; then
export GITLAB_QA_OPTIONS="--set-feature-flags $feature_flags"
echo $GITLAB_QA_OPTIONS
- ./scripts/trigger-build omnibus
+ ./scripts/trigger-build.rb omnibus
else
echo "No changed feature flag found to test. The tests are skipped if the flag was removed."
fi
diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml
index 702bba8ae97ba..2ae81dbb5bae4 100644
--- a/.gitlab/ci/rails.gitlab-ci.yml
+++ b/.gitlab/ci/rails.gitlab-ci.yml
@@ -438,7 +438,7 @@ db:gitlabcom-database-testing:
script:
- source scripts/utils.sh
- install_gitlab_gem
- - ./scripts/trigger-build gitlab-com-database-testing
+ - ./scripts/trigger-build.rb gitlab-com-database-testing
gitlab:setup:
extends: .db-job-base
diff --git a/.gitlab/ci/review-apps/main.gitlab-ci.yml b/.gitlab/ci/review-apps/main.gitlab-ci.yml
index 16ef3c82cda52..d083f876b03fc 100644
--- a/.gitlab/ci/review-apps/main.gitlab-ci.yml
+++ b/.gitlab/ci/review-apps/main.gitlab-ci.yml
@@ -16,20 +16,58 @@ include:
- source ./scripts/review_apps/review-apps.sh
- install_api_client_dependencies_with_apk
-review-build-cng:
+review-build-cng-env:
extends:
- .default-retry
- .review:rules:review-build-cng
image: ${GITLAB_DEPENDENCY_PROXY}ruby:2.7-alpine3.13
stage: prepare
- variables:
- CNG_PROJECT_ACCESS_TOKEN: "${CNG_MIRROR_PROJECT_ACCESS_TOKEN}" # "Multi-pipeline (from 'gitlab-org/gitlab' 'review-build-cng' job)" at https://gitlab.com/gitlab-org/build/CNG-mirror/-/settings/access_tokens
- CNG_PROJECT_PATH: "gitlab-org/build/CNG-mirror"
+ needs: []
before_script:
- source ./scripts/utils.sh
- install_gitlab_gem
script:
- - ./scripts/trigger-build cng
+ - 'ruby -r./scripts/trigger-build.rb -e "puts Trigger.variables_for_env_file(Trigger::CNG.new.variables)" > build.env'
+ - cat build.env
+ artifacts:
+ reports:
+ dotenv: build.env
+ paths:
+ - build.env
+ expire_in: 7 days
+ when: always
+
+review-build-cng:
+ extends: .review:rules:review-build-cng
+ stage: prepare
+ needs: ["review-build-cng-env"]
+ inherit:
+ variables: false
+ variables:
+ TOP_UPSTREAM_SOURCE_PROJECT: "${TOP_UPSTREAM_SOURCE_PROJECT}"
+ TOP_UPSTREAM_SOURCE_REF: "${TOP_UPSTREAM_SOURCE_REF}"
+ TOP_UPSTREAM_SOURCE_JOB: "${TOP_UPSTREAM_SOURCE_JOB}"
+ TOP_UPSTREAM_SOURCE_SHA: "${TOP_UPSTREAM_SOURCE_SHA}"
+ TOP_UPSTREAM_MERGE_REQUEST_PROJECT_ID: "${TOP_UPSTREAM_MERGE_REQUEST_PROJECT_ID}"
+ TOP_UPSTREAM_MERGE_REQUEST_IID: "${TOP_UPSTREAM_MERGE_REQUEST_IID}"
+ GITLAB_REF_SLUG: "${GITLAB_REF_SLUG}"
+ # CNG pipeline specific variables
+ GITLAB_VERSION: "${GITLAB_VERSION}"
+ GITLAB_TAG: "${GITLAB_TAG}"
+ GITLAB_ASSETS_TAG: "${GITLAB_ASSETS_TAG}"
+ FORCE_RAILS_IMAGE_BUILDS: "${FORCE_RAILS_IMAGE_BUILDS}"
+ CE_PIPELINE: "${CE_PIPELINE}" # Based on https://docs.gitlab.com/ee/ci/jobs/job_control.html#check-if-a-variable-exists, `if: '$CE_PIPELINE'` will evaluate to `false` when this variable is empty
+ EE_PIPELINE: "${EE_PIPELINE}" # Based on https://docs.gitlab.com/ee/ci/jobs/job_control.html#check-if-a-variable-exists, `if: '$EE_PIPELINE'` will evaluate to `false` when this variable is empty
+ GITLAB_SHELL_VERSION: "${GITLAB_SHELL_VERSION}"
+ GITLAB_ELASTICSEARCH_INDEXER_VERSION: "${GITLAB_ELASTICSEARCH_INDEXER_VERSION}"
+ GITLAB_KAS_VERSION: "${GITLAB_KAS_VERSION}"
+ GITLAB_WORKHORSE_VERSION: "${GITLAB_WORKHORSE_VERSION}"
+ GITLAB_PAGES_VERSION: "${GITLAB_PAGES_VERSION}"
+ GITALY_SERVER_VERSION: "${GITALY_SERVER_VERSION}"
+ trigger:
+ project: gitlab-org/build/CNG-mirror
+ branch: $TRIGGER_BRANCH
+ strategy: depend
.review-workflow-base:
extends:
diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml
index 3e23f5f8bd7f2..2f1dbfce8607a 100644
--- a/.gitlab/ci/rules.gitlab-ci.yml
+++ b/.gitlab/ci/rules.gitlab-ci.yml
@@ -141,7 +141,7 @@
- ".gitlab/ci/review-apps/**/*"
- "scripts/review_apps/base-config.yaml"
- "scripts/review_apps/review-apps.sh"
- - "scripts/trigger-build"
+ - "scripts/trigger-build.rb"
- "{,ee/,jh/}{bin,config}/**/*.rb"
.ci-qa-patterns: &ci-qa-patterns
diff --git a/doc/development/documentation/review_apps.md b/doc/development/documentation/review_apps.md
index 844dc3e72d0b4..8cb9e6437b810 100644
--- a/doc/development/documentation/review_apps.md
+++ b/doc/development/documentation/review_apps.md
@@ -41,7 +41,7 @@ the GitLab team to run the job.
If you want to know the in-depth details, here's what's really happening:
1. You manually run the `review-docs-deploy` job in a merge request.
-1. The job runs the [`scripts/trigger-build`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/scripts/trigger-build)
+1. The job runs the [`scripts/trigger-build.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/scripts/trigger-build.rb)
script with the `docs deploy` flag, which triggers the "Triggered from `gitlab-org/gitlab` 'review-docs-deploy' job"
pipeline trigger in the `gitlab-org/gitlab-docs` project for the `$DOCS_BRANCH` (defaults to `main`).
1. The preview URL is shown both at the job output and in the merge request
diff --git a/scripts/trigger-build b/scripts/trigger-build.rb
similarity index 88%
rename from scripts/trigger-build
rename to scripts/trigger-build.rb
index d40e8de5a1f39..17cbd91a8eeff 100755
--- a/scripts/trigger-build
+++ b/scripts/trigger-build.rb
@@ -21,6 +21,12 @@ def self.non_empty_variable_value(variable)
variable_value
end
+ def self.variables_for_env_file(variables)
+ variables.map do |key, value|
+ %Q(#{key}=#{value})
+ end.join("\n")
+ end
+
class Base
# Can be overridden
def self.access_token
@@ -57,6 +63,21 @@ def invoke!(post_comment: false, downstream_job_name: nil)
end
end
+ def variables
+ simple_forwarded_variables.merge(base_variables, extra_variables, version_file_variables)
+ end
+
+ def simple_forwarded_variables
+ {
+ 'TRIGGER_SOURCE' => ENV['CI_JOB_URL'],
+ 'TOP_UPSTREAM_SOURCE_PROJECT' => ENV['CI_PROJECT_PATH'],
+ 'TOP_UPSTREAM_SOURCE_REF' => ENV['CI_COMMIT_REF_NAME'],
+ 'TOP_UPSTREAM_SOURCE_JOB' => ENV['CI_JOB_URL'],
+ 'TOP_UPSTREAM_MERGE_REQUEST_PROJECT_ID' => ENV['CI_MERGE_REQUEST_PROJECT_ID'],
+ 'TOP_UPSTREAM_MERGE_REQUEST_IID' => ENV['CI_MERGE_REQUEST_IID']
+ }
+ end
+
private
# Override to trigger and work with pipeline on different GitLab instance
@@ -95,23 +116,13 @@ def version_param_value(version_file)
ENV[version_file]&.strip || File.read(version_file).strip
end
- def variables
- base_variables.merge(extra_variables).merge(version_file_variables)
- end
-
def base_variables
# Use CI_MERGE_REQUEST_SOURCE_BRANCH_SHA for omnibus checkouts due to pipeline for merged results,
# and fallback to CI_COMMIT_SHA for the `detached` pipelines.
{
'GITLAB_REF_SLUG' => ENV['CI_COMMIT_TAG'] ? ENV['CI_COMMIT_REF_NAME'] : ENV['CI_COMMIT_REF_SLUG'],
'TRIGGERED_USER' => ENV['TRIGGERED_USER'] || ENV['GITLAB_USER_NAME'],
- 'TRIGGER_SOURCE' => ENV['CI_JOB_URL'],
- 'TOP_UPSTREAM_SOURCE_PROJECT' => ENV['CI_PROJECT_PATH'],
- 'TOP_UPSTREAM_SOURCE_JOB' => ENV['CI_JOB_URL'],
- 'TOP_UPSTREAM_SOURCE_SHA' => Trigger.non_empty_variable_value('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA') || ENV['CI_COMMIT_SHA'],
- 'TOP_UPSTREAM_SOURCE_REF' => ENV['CI_COMMIT_REF_NAME'],
- 'TOP_UPSTREAM_MERGE_REQUEST_PROJECT_ID' => ENV['CI_MERGE_REQUEST_PROJECT_ID'],
- 'TOP_UPSTREAM_MERGE_REQUEST_IID' => ENV['CI_MERGE_REQUEST_IID']
+ 'TOP_UPSTREAM_SOURCE_SHA' => Trigger.non_empty_variable_value('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA') || ENV['CI_COMMIT_SHA']
}
end
@@ -163,17 +174,16 @@ def extra_variables
end
class CNG < Base
- def self.access_token
- # Default to "Multi-pipeline (from 'gitlab-org/gitlab' 'cloud-native-image' job)" at https://gitlab.com/gitlab-org/build/CNG/-/settings/access_tokens
- ENV['CNG_PROJECT_ACCESS_TOKEN'] || super
+ def variables
+ # Delete variables that aren't useful when using native triggers.
+ super.tap do |hash|
+ hash.delete('TRIGGER_SOURCE')
+ hash.delete('TRIGGERED_USER')
+ end
end
private
- def downstream_project_path
- ENV.fetch('CNG_PROJECT_PATH', 'gitlab-org/build/CNG')
- end
-
def ref
return ENV['CI_COMMIT_REF_NAME'] if ENV['CI_COMMIT_REF_NAME'] =~ /^[\d-]+-stable(-ee)?$/
@@ -181,17 +191,17 @@ def ref
end
def extra_variables
- edition = Trigger.ee? ? 'EE' : 'CE'
# Use CI_MERGE_REQUEST_SOURCE_BRANCH_SHA (MR HEAD commit) so that the image is in sync with the assets and QA images.
source_sha = Trigger.non_empty_variable_value('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA') || ENV['CI_COMMIT_SHA']
{
- "ee" => Trigger.ee? ? "true" : "false",
+ "TRIGGER_BRANCH" => ref,
"GITLAB_VERSION" => source_sha,
- "GITLAB_TAG" => ENV['CI_COMMIT_TAG'],
+ "GITLAB_TAG" => ENV['CI_COMMIT_TAG'], # Always set a value, even an empty string, so that the downstream pipeline can correctly check it.
"GITLAB_ASSETS_TAG" => ENV['CI_COMMIT_TAG'] ? ENV['CI_COMMIT_REF_NAME'] : source_sha,
"FORCE_RAILS_IMAGE_BUILDS" => 'true',
- "#{edition}_PIPELINE" => 'true'
+ "CE_PIPELINE" => Trigger.ee? ? nil : "true", # Always set a value, even an empty string, so that the downstream pipeline can correctly check it.
+ "EE_PIPELINE" => Trigger.ee? ? "true" : nil # Always set a value, even an empty string, so that the downstream pipeline can correctly check it.
}
end
@@ -445,28 +455,30 @@ def status
Job = Class.new(Pipeline)
end
-case ARGV[0]
-when 'omnibus'
- Trigger::Omnibus.new.invoke!(post_comment: true, downstream_job_name: 'Trigger:qa-test').wait!
-when 'cng'
- Trigger::CNG.new.invoke!.wait!
-when 'gitlab-com-database-testing'
- Trigger::DatabaseTesting.new.invoke!
-when 'docs'
- docs_trigger = Trigger::Docs.new
-
- case ARGV[1]
- when 'deploy'
- docs_trigger.deploy!
- when 'cleanup'
- docs_trigger.cleanup!
+if $0 == __FILE__
+ case ARGV[0]
+ when 'omnibus'
+ Trigger::Omnibus.new.invoke!(post_comment: true, downstream_job_name: 'Trigger:qa-test').wait!
+ when 'cng'
+ Trigger::CNG.new.invoke!.wait!
+ when 'gitlab-com-database-testing'
+ Trigger::DatabaseTesting.new.invoke!
+ when 'docs'
+ docs_trigger = Trigger::Docs.new
+
+ case ARGV[1]
+ when 'deploy'
+ docs_trigger.deploy!
+ when 'cleanup'
+ docs_trigger.cleanup!
+ else
+ puts 'usage: trigger-build docs <deploy|cleanup>'
+ exit 1
+ end
else
- puts 'usage: trigger-build docs <deploy|cleanup>'
- exit 1
+ puts "Please provide a valid option:
+ omnibus - Triggers a pipeline that builds the omnibus-gitlab package
+ cng - Triggers a pipeline that builds images used by the GitLab helm chart
+ gitlab-com-database-testing - Triggers a pipeline that tests database changes on GitLab.com data"
end
-else
- puts "Please provide a valid option:
- omnibus - Triggers a pipeline that builds the omnibus-gitlab package
- cng - Triggers a pipeline that builds images used by the GitLab helm chart
- gitlab-com-database-testing - Triggers a pipeline that tests database changes on GitLab.com data"
end
--
GitLab