diff --git a/.gitlab/ci/cng.gitlab-ci.yml b/.gitlab/ci/cng.gitlab-ci.yml index bf439288be29a1c948f387d01f96166911bdda91..d720ec5ae452a520dfe333d45f75ec2ce5e62672 100644 --- a/.gitlab/ci/cng.gitlab-ci.yml +++ b/.gitlab/ci/cng.gitlab-ci.yml @@ -1,10 +1,51 @@ +cloud-native-image-env: + extends: + - .default-retry + - .cng:rules + image: ${GITLAB_DEPENDENCY_PROXY}ruby:2.7-alpine3.13 + stage: post-test + before_script: + - source ./scripts/utils.sh + - install_gitlab_gem + script: + - 'ruby -r./scripts/trigger-build.rb -e "puts Trigger.variables_for_env_file(Trigger::CNG.new.variables)" > build.env' + - cat build.env + artifacts: + reports: + dotenv: build.env + paths: + - build.env + expire_in: 7 days + when: always + cloud-native-image: extends: .cng:rules - image: ${GITLAB_DEPENDENCY_PROXY}ruby:2.7-alpine - dependencies: [] stage: post-test + needs: ["cloud-native-image-env"] + inherit: + variables: false variables: - GIT_DEPTH: "1" - script: - - install_gitlab_gem - - ./scripts/trigger-build cng + TOP_UPSTREAM_SOURCE_PROJECT: "${TOP_UPSTREAM_SOURCE_PROJECT}" + TOP_UPSTREAM_SOURCE_REF: "${TOP_UPSTREAM_SOURCE_REF}" + TOP_UPSTREAM_SOURCE_JOB: "${TOP_UPSTREAM_SOURCE_JOB}" + TOP_UPSTREAM_SOURCE_SHA: "${TOP_UPSTREAM_SOURCE_SHA}" + TOP_UPSTREAM_MERGE_REQUEST_PROJECT_ID: "${TOP_UPSTREAM_MERGE_REQUEST_PROJECT_ID}" + TOP_UPSTREAM_MERGE_REQUEST_IID: "${TOP_UPSTREAM_MERGE_REQUEST_IID}" + GITLAB_REF_SLUG: "${GITLAB_REF_SLUG}" + # CNG pipeline specific variables + GITLAB_VERSION: "${GITLAB_VERSION}" + GITLAB_TAG: "${GITLAB_TAG}" + GITLAB_ASSETS_TAG: "${GITLAB_ASSETS_TAG}" + FORCE_RAILS_IMAGE_BUILDS: "${FORCE_RAILS_IMAGE_BUILDS}" + CE_PIPELINE: "${CE_PIPELINE}" # Based on https://docs.gitlab.com/ee/ci/jobs/job_control.html#check-if-a-variable-exists, `if: '$CE_PIPELINE'` will evaluate to `false` when this variable is empty + EE_PIPELINE: "${EE_PIPELINE}" # Based on https://docs.gitlab.com/ee/ci/jobs/job_control.html#check-if-a-variable-exists, `if: '$EE_PIPELINE'` will evaluate to `false` when this variable is empty + GITLAB_SHELL_VERSION: "${GITLAB_SHELL_VERSION}" + GITLAB_ELASTICSEARCH_INDEXER_VERSION: "${GITLAB_ELASTICSEARCH_INDEXER_VERSION}" + GITLAB_KAS_VERSION: "${GITLAB_KAS_VERSION}" + GITLAB_WORKHORSE_VERSION: "${GITLAB_WORKHORSE_VERSION}" + GITLAB_PAGES_VERSION: "${GITLAB_PAGES_VERSION}" + GITALY_SERVER_VERSION: "${GITALY_SERVER_VERSION}" + trigger: + project: gitlab-org/build/CNG + branch: $TRIGGER_BRANCH + strategy: depend diff --git a/.gitlab/ci/docs.gitlab-ci.yml b/.gitlab/ci/docs.gitlab-ci.yml index 796dd261824976b78717e353110d53a5dde00d1a..8b7691045cbbb4452548e3922f087097110ad6a3 100644 --- a/.gitlab/ci/docs.gitlab-ci.yml +++ b/.gitlab/ci/docs.gitlab-ci.yml @@ -28,7 +28,7 @@ review-docs-deploy: extends: .review-docs script: - - ./scripts/trigger-build docs deploy + - ./scripts/trigger-build.rb docs deploy # Cleanup remote environment of gitlab-docs review-docs-cleanup: @@ -37,7 +37,7 @@ review-docs-cleanup: name: review-docs/mr-${CI_MERGE_REQUEST_IID} action: stop script: - - ./scripts/trigger-build docs cleanup + - ./scripts/trigger-build.rb docs cleanup docs-lint markdown: extends: diff --git a/.gitlab/ci/qa.gitlab-ci.yml b/.gitlab/ci/qa.gitlab-ci.yml index b12f76f2823e2f01d5282d010ca04fc730996d6d..77d12a3e848ed6ca0c069da1f47d687be8edc6f2 100644 --- a/.gitlab/ci/qa.gitlab-ci.yml +++ b/.gitlab/ci/qa.gitlab-ci.yml @@ -73,7 +73,7 @@ update-qa-cache: - echo $exit_code - | if [ $exit_code -eq 0 ]; then - ./scripts/trigger-build omnibus + ./scripts/trigger-build.rb omnibus elif [ $exit_code -eq 1 ]; then exit 1 else @@ -108,7 +108,7 @@ update-qa-cache: if [[ $feature_flags ]]; then export GITLAB_QA_OPTIONS="--set-feature-flags $feature_flags" echo $GITLAB_QA_OPTIONS - ./scripts/trigger-build omnibus + ./scripts/trigger-build.rb omnibus else echo "No changed feature flag found to test. The tests are skipped if the flag was removed." fi diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml index 702bba8ae97ba6bf4a94001a593d38e8c00ae27d..2ae81dbb5bae45dc0291c4fadba2de50f7e2130e 100644 --- a/.gitlab/ci/rails.gitlab-ci.yml +++ b/.gitlab/ci/rails.gitlab-ci.yml @@ -438,7 +438,7 @@ db:gitlabcom-database-testing: script: - source scripts/utils.sh - install_gitlab_gem - - ./scripts/trigger-build gitlab-com-database-testing + - ./scripts/trigger-build.rb gitlab-com-database-testing gitlab:setup: extends: .db-job-base diff --git a/.gitlab/ci/review-apps/main.gitlab-ci.yml b/.gitlab/ci/review-apps/main.gitlab-ci.yml index 16ef3c82cda5269a0ec6cfe4ab6811b8e166d448..d083f876b03fc89972f1fc9493e721061d435246 100644 --- a/.gitlab/ci/review-apps/main.gitlab-ci.yml +++ b/.gitlab/ci/review-apps/main.gitlab-ci.yml @@ -16,20 +16,58 @@ include: - source ./scripts/review_apps/review-apps.sh - install_api_client_dependencies_with_apk -review-build-cng: +review-build-cng-env: extends: - .default-retry - .review:rules:review-build-cng image: ${GITLAB_DEPENDENCY_PROXY}ruby:2.7-alpine3.13 stage: prepare - variables: - CNG_PROJECT_ACCESS_TOKEN: "${CNG_MIRROR_PROJECT_ACCESS_TOKEN}" # "Multi-pipeline (from 'gitlab-org/gitlab' 'review-build-cng' job)" at https://gitlab.com/gitlab-org/build/CNG-mirror/-/settings/access_tokens - CNG_PROJECT_PATH: "gitlab-org/build/CNG-mirror" + needs: [] before_script: - source ./scripts/utils.sh - install_gitlab_gem script: - - ./scripts/trigger-build cng + - 'ruby -r./scripts/trigger-build.rb -e "puts Trigger.variables_for_env_file(Trigger::CNG.new.variables)" > build.env' + - cat build.env + artifacts: + reports: + dotenv: build.env + paths: + - build.env + expire_in: 7 days + when: always + +review-build-cng: + extends: .review:rules:review-build-cng + stage: prepare + needs: ["review-build-cng-env"] + inherit: + variables: false + variables: + TOP_UPSTREAM_SOURCE_PROJECT: "${TOP_UPSTREAM_SOURCE_PROJECT}" + TOP_UPSTREAM_SOURCE_REF: "${TOP_UPSTREAM_SOURCE_REF}" + TOP_UPSTREAM_SOURCE_JOB: "${TOP_UPSTREAM_SOURCE_JOB}" + TOP_UPSTREAM_SOURCE_SHA: "${TOP_UPSTREAM_SOURCE_SHA}" + TOP_UPSTREAM_MERGE_REQUEST_PROJECT_ID: "${TOP_UPSTREAM_MERGE_REQUEST_PROJECT_ID}" + TOP_UPSTREAM_MERGE_REQUEST_IID: "${TOP_UPSTREAM_MERGE_REQUEST_IID}" + GITLAB_REF_SLUG: "${GITLAB_REF_SLUG}" + # CNG pipeline specific variables + GITLAB_VERSION: "${GITLAB_VERSION}" + GITLAB_TAG: "${GITLAB_TAG}" + GITLAB_ASSETS_TAG: "${GITLAB_ASSETS_TAG}" + FORCE_RAILS_IMAGE_BUILDS: "${FORCE_RAILS_IMAGE_BUILDS}" + CE_PIPELINE: "${CE_PIPELINE}" # Based on https://docs.gitlab.com/ee/ci/jobs/job_control.html#check-if-a-variable-exists, `if: '$CE_PIPELINE'` will evaluate to `false` when this variable is empty + EE_PIPELINE: "${EE_PIPELINE}" # Based on https://docs.gitlab.com/ee/ci/jobs/job_control.html#check-if-a-variable-exists, `if: '$EE_PIPELINE'` will evaluate to `false` when this variable is empty + GITLAB_SHELL_VERSION: "${GITLAB_SHELL_VERSION}" + GITLAB_ELASTICSEARCH_INDEXER_VERSION: "${GITLAB_ELASTICSEARCH_INDEXER_VERSION}" + GITLAB_KAS_VERSION: "${GITLAB_KAS_VERSION}" + GITLAB_WORKHORSE_VERSION: "${GITLAB_WORKHORSE_VERSION}" + GITLAB_PAGES_VERSION: "${GITLAB_PAGES_VERSION}" + GITALY_SERVER_VERSION: "${GITALY_SERVER_VERSION}" + trigger: + project: gitlab-org/build/CNG-mirror + branch: $TRIGGER_BRANCH + strategy: depend .review-workflow-base: extends: diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index 3e23f5f8bd7f2f6034cf2e973912349a502c8335..2f1dbfce8607aad3bb04a31c0e62eabc47f4e102 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -141,7 +141,7 @@ - ".gitlab/ci/review-apps/**/*" - "scripts/review_apps/base-config.yaml" - "scripts/review_apps/review-apps.sh" - - "scripts/trigger-build" + - "scripts/trigger-build.rb" - "{,ee/,jh/}{bin,config}/**/*.rb" .ci-qa-patterns: &ci-qa-patterns diff --git a/doc/development/documentation/review_apps.md b/doc/development/documentation/review_apps.md index 844dc3e72d0b46d042180d8ebce5c92532a10a8c..8cb9e6437b810faff31fa694a9a8975119684ed5 100644 --- a/doc/development/documentation/review_apps.md +++ b/doc/development/documentation/review_apps.md @@ -41,7 +41,7 @@ the GitLab team to run the job. If you want to know the in-depth details, here's what's really happening: 1. You manually run the `review-docs-deploy` job in a merge request. -1. The job runs the [`scripts/trigger-build`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/scripts/trigger-build) +1. The job runs the [`scripts/trigger-build.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/scripts/trigger-build.rb) script with the `docs deploy` flag, which triggers the "Triggered from `gitlab-org/gitlab` 'review-docs-deploy' job" pipeline trigger in the `gitlab-org/gitlab-docs` project for the `$DOCS_BRANCH` (defaults to `main`). 1. The preview URL is shown both at the job output and in the merge request diff --git a/scripts/trigger-build b/scripts/trigger-build.rb similarity index 88% rename from scripts/trigger-build rename to scripts/trigger-build.rb index d40e8de5a1f39dbcdbdf1c5b9708ccc70ea70cc1..17cbd91a8eeff6755fbaf4cc7afc9df4d366f83c 100755 --- a/scripts/trigger-build +++ b/scripts/trigger-build.rb @@ -21,6 +21,12 @@ def self.non_empty_variable_value(variable) variable_value end + def self.variables_for_env_file(variables) + variables.map do |key, value| + %Q(#{key}=#{value}) + end.join("\n") + end + class Base # Can be overridden def self.access_token @@ -57,6 +63,21 @@ def invoke!(post_comment: false, downstream_job_name: nil) end end + def variables + simple_forwarded_variables.merge(base_variables, extra_variables, version_file_variables) + end + + def simple_forwarded_variables + { + 'TRIGGER_SOURCE' => ENV['CI_JOB_URL'], + 'TOP_UPSTREAM_SOURCE_PROJECT' => ENV['CI_PROJECT_PATH'], + 'TOP_UPSTREAM_SOURCE_REF' => ENV['CI_COMMIT_REF_NAME'], + 'TOP_UPSTREAM_SOURCE_JOB' => ENV['CI_JOB_URL'], + 'TOP_UPSTREAM_MERGE_REQUEST_PROJECT_ID' => ENV['CI_MERGE_REQUEST_PROJECT_ID'], + 'TOP_UPSTREAM_MERGE_REQUEST_IID' => ENV['CI_MERGE_REQUEST_IID'] + } + end + private # Override to trigger and work with pipeline on different GitLab instance @@ -95,23 +116,13 @@ def version_param_value(version_file) ENV[version_file]&.strip || File.read(version_file).strip end - def variables - base_variables.merge(extra_variables).merge(version_file_variables) - end - def base_variables # Use CI_MERGE_REQUEST_SOURCE_BRANCH_SHA for omnibus checkouts due to pipeline for merged results, # and fallback to CI_COMMIT_SHA for the `detached` pipelines. { 'GITLAB_REF_SLUG' => ENV['CI_COMMIT_TAG'] ? ENV['CI_COMMIT_REF_NAME'] : ENV['CI_COMMIT_REF_SLUG'], 'TRIGGERED_USER' => ENV['TRIGGERED_USER'] || ENV['GITLAB_USER_NAME'], - 'TRIGGER_SOURCE' => ENV['CI_JOB_URL'], - 'TOP_UPSTREAM_SOURCE_PROJECT' => ENV['CI_PROJECT_PATH'], - 'TOP_UPSTREAM_SOURCE_JOB' => ENV['CI_JOB_URL'], - 'TOP_UPSTREAM_SOURCE_SHA' => Trigger.non_empty_variable_value('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA') || ENV['CI_COMMIT_SHA'], - 'TOP_UPSTREAM_SOURCE_REF' => ENV['CI_COMMIT_REF_NAME'], - 'TOP_UPSTREAM_MERGE_REQUEST_PROJECT_ID' => ENV['CI_MERGE_REQUEST_PROJECT_ID'], - 'TOP_UPSTREAM_MERGE_REQUEST_IID' => ENV['CI_MERGE_REQUEST_IID'] + 'TOP_UPSTREAM_SOURCE_SHA' => Trigger.non_empty_variable_value('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA') || ENV['CI_COMMIT_SHA'] } end @@ -163,17 +174,16 @@ def extra_variables end class CNG < Base - def self.access_token - # Default to "Multi-pipeline (from 'gitlab-org/gitlab' 'cloud-native-image' job)" at https://gitlab.com/gitlab-org/build/CNG/-/settings/access_tokens - ENV['CNG_PROJECT_ACCESS_TOKEN'] || super + def variables + # Delete variables that aren't useful when using native triggers. + super.tap do |hash| + hash.delete('TRIGGER_SOURCE') + hash.delete('TRIGGERED_USER') + end end private - def downstream_project_path - ENV.fetch('CNG_PROJECT_PATH', 'gitlab-org/build/CNG') - end - def ref return ENV['CI_COMMIT_REF_NAME'] if ENV['CI_COMMIT_REF_NAME'] =~ /^[\d-]+-stable(-ee)?$/ @@ -181,17 +191,17 @@ def ref end def extra_variables - edition = Trigger.ee? ? 'EE' : 'CE' # Use CI_MERGE_REQUEST_SOURCE_BRANCH_SHA (MR HEAD commit) so that the image is in sync with the assets and QA images. source_sha = Trigger.non_empty_variable_value('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA') || ENV['CI_COMMIT_SHA'] { - "ee" => Trigger.ee? ? "true" : "false", + "TRIGGER_BRANCH" => ref, "GITLAB_VERSION" => source_sha, - "GITLAB_TAG" => ENV['CI_COMMIT_TAG'], + "GITLAB_TAG" => ENV['CI_COMMIT_TAG'], # Always set a value, even an empty string, so that the downstream pipeline can correctly check it. "GITLAB_ASSETS_TAG" => ENV['CI_COMMIT_TAG'] ? ENV['CI_COMMIT_REF_NAME'] : source_sha, "FORCE_RAILS_IMAGE_BUILDS" => 'true', - "#{edition}_PIPELINE" => 'true' + "CE_PIPELINE" => Trigger.ee? ? nil : "true", # Always set a value, even an empty string, so that the downstream pipeline can correctly check it. + "EE_PIPELINE" => Trigger.ee? ? "true" : nil # Always set a value, even an empty string, so that the downstream pipeline can correctly check it. } end @@ -445,28 +455,30 @@ def status Job = Class.new(Pipeline) end -case ARGV[0] -when 'omnibus' - Trigger::Omnibus.new.invoke!(post_comment: true, downstream_job_name: 'Trigger:qa-test').wait! -when 'cng' - Trigger::CNG.new.invoke!.wait! -when 'gitlab-com-database-testing' - Trigger::DatabaseTesting.new.invoke! -when 'docs' - docs_trigger = Trigger::Docs.new - - case ARGV[1] - when 'deploy' - docs_trigger.deploy! - when 'cleanup' - docs_trigger.cleanup! +if $0 == __FILE__ + case ARGV[0] + when 'omnibus' + Trigger::Omnibus.new.invoke!(post_comment: true, downstream_job_name: 'Trigger:qa-test').wait! + when 'cng' + Trigger::CNG.new.invoke!.wait! + when 'gitlab-com-database-testing' + Trigger::DatabaseTesting.new.invoke! + when 'docs' + docs_trigger = Trigger::Docs.new + + case ARGV[1] + when 'deploy' + docs_trigger.deploy! + when 'cleanup' + docs_trigger.cleanup! + else + puts 'usage: trigger-build docs <deploy|cleanup>' + exit 1 + end else - puts 'usage: trigger-build docs <deploy|cleanup>' - exit 1 + puts "Please provide a valid option: + omnibus - Triggers a pipeline that builds the omnibus-gitlab package + cng - Triggers a pipeline that builds images used by the GitLab helm chart + gitlab-com-database-testing - Triggers a pipeline that tests database changes on GitLab.com data" end -else - puts "Please provide a valid option: - omnibus - Triggers a pipeline that builds the omnibus-gitlab package - cng - Triggers a pipeline that builds images used by the GitLab helm chart - gitlab-com-database-testing - Triggers a pipeline that tests database changes on GitLab.com data" end