From a4893e6226f8dd84e7bf23e5be3091c511ad2a49 Mon Sep 17 00:00:00 2001
From: Dat Tang <dattang@gitlab.com>
Date: Tue, 9 Jul 2024 06:55:47 +0000
Subject: [PATCH] Merge branch
'jennykim/release-environment-trigger-from-security' into 'master'
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Trigger release-environment pipeline from security stable branches
See merge request https://gitlab.com/gitlab-org/gitlab/-/merge_requests/154669
Merged-by: Peter Leitzen <pleitzen@gitlab.com>
Approved-by: Rémy Coutable <remy@rymai.me>
Approved-by: Peter Leitzen <pleitzen@gitlab.com>
Reviewed-by: Mayra Cabrera <mcabrera@gitlab.com>
Reviewed-by: Rémy Coutable <remy@rymai.me>
Co-authored-by: Rémy Coutable <remy@rymai.me>
Co-authored-by: Dat Tang <dattang@gitlab.com>
Co-authored-by: Jenny Kim <yjeankim@gitlab.com>
(cherry picked from commit b2728d7a4781d12b97978c4bbbcbea48e7b13eb3)
ee55efbe Security stable branch commits and commit tags trigger release-environment pipeline
e99dbd6c Build QA image for security stable branches
e7f9cf7f Remove temporary comment
38226b9f remove "security" path portion of CI_PROJECT_NAMESPACE
81031959 Match security mirror when processing TRIGGER_BRANCH
8d588eaa Correct image tags in VERSIONS variable
70388adf Remove resource group for release environments QA
765b0da5 Add docker login to security release environments QA job
70515ad4 Add rspec to test security environment name
9d4d7037 Apply 3 suggestion(s) to 2 file(s)
3da91fdf Remove unused method in construct RE script
166e712e Merge security cng template
386cc6e5 Merge branch 'master' into 'jennykim/release-environment-trigger-from-security'
Co-authored-by: Peter Leitzen <pleitzen@gitlab.com>
---
.gitlab/ci/cng/main.gitlab-ci.yml | 7 +-
.gitlab/ci/release-environments.gitlab-ci.yml | 28 ++++
.../release-environments/main.gitlab-ci.yml | 5 +-
.../security.gitlab-ci.yml | 143 ++++++++++++++++++
.gitlab/ci/rules.gitlab-ci.yml | 21 +++
...construct-release-environments-versions.rb | 34 ++++-
scripts/trigger-build.rb | 2 +-
.../release_environments_model_spec.rb | 43 ++++--
8 files changed, 258 insertions(+), 25 deletions(-)
create mode 100644 .gitlab/ci/release-environments/security.gitlab-ci.yml
diff --git a/.gitlab/ci/cng/main.gitlab-ci.yml b/.gitlab/ci/cng/main.gitlab-ci.yml
index 214669fa4d989..af33e3db6376c 100644
--- a/.gitlab/ci/cng/main.gitlab-ci.yml
+++ b/.gitlab/ci/cng/main.gitlab-ci.yml
@@ -1,3 +1,8 @@
+spec:
+ inputs:
+ cng_path:
+ type: string
+ default: 'build/CNG-mirror'
---
default:
interruptible: true
@@ -61,6 +66,6 @@ include:
TOP_UPSTREAM_MERGE_REQUEST_IID: "${TOP_UPSTREAM_MERGE_REQUEST_IID}"
TOP_UPSTREAM_SOURCE_SHA: "${TOP_UPSTREAM_SOURCE_SHA}"
trigger:
- project: ${CI_PROJECT_NAMESPACE}/build/CNG-mirror
+ project: '${CI_PROJECT_NAMESPACE}/$[[ inputs.cng_path ]]'
branch: $TRIGGER_BRANCH
strategy: depend
diff --git a/.gitlab/ci/release-environments.gitlab-ci.yml b/.gitlab/ci/release-environments.gitlab-ci.yml
index a6a0e268451c5..bcd1a3b047c3b 100644
--- a/.gitlab/ci/release-environments.gitlab-ci.yml
+++ b/.gitlab/ci/release-environments.gitlab-ci.yml
@@ -26,3 +26,31 @@ start-release-environments-pipeline:
- project: 'gitlab-org/gitlab'
ref: 'master'
file: '.gitlab/ci/release-environments/main.gitlab-ci.yml'
+
+start-release-environments-security-pipeline:
+ allow_failure: true
+ extends:
+ - .release-environments:rules:start-release-environments-security-pipeline
+ stage: release-environments
+ # We do not want to have ALL global variables passed as trigger variables,
+ # as they cannot be overridden. See this issue for more context:
+ #
+ # https://gitlab.com/gitlab-org/gitlab/-/issues/387183
+ inherit:
+ variables:
+ - RUBY_VERSION_DEFAULT
+ - RUBY_VERSION_NEXT
+ - RUBY_VERSION
+
+ # These variables are set in the pipeline schedules.
+ # They need to be explicitly passed on to the child pipeline.
+ # https://docs.gitlab.com/ee/ci/pipelines/multi_project_pipelines.html#pass-cicd-variables-to-a-downstream-pipeline-by-using-the-variables-keyword
+ variables:
+ # This is needed by `release-environments-build-cng-env` (`.gitlab/ci/release-environments/security.gitlab-ci.yml`).
+ PARENT_PIPELINE_ID: $CI_PIPELINE_ID
+ trigger:
+ strategy: depend
+ include:
+ - project: 'gitlab-org/security/gitlab'
+ ref: 'master'
+ file: '.gitlab/ci/release-environments/security.gitlab-ci.yml'
diff --git a/.gitlab/ci/release-environments/main.gitlab-ci.yml b/.gitlab/ci/release-environments/main.gitlab-ci.yml
index d1097e8326b53..a8a05965132ad 100644
--- a/.gitlab/ci/release-environments/main.gitlab-ci.yml
+++ b/.gitlab/ci/release-environments/main.gitlab-ci.yml
@@ -1,8 +1,10 @@
---
include:
- local: .gitlab/ci/cng/main.gitlab-ci.yml
+ inputs:
+ cng_path: 'build/CNG-mirror'
- project: 'gitlab-org/quality/pipeline-common'
- ref: '8.18.3'
+ ref: '8.18.4'
file: ci/base.gitlab-ci.yml
stages:
@@ -95,7 +97,6 @@ release-environments-qa:
GITLAB_INITIAL_ROOT_PASSWORD: "${RELEASE_ENVIRONMENTS_ROOT_PASSWORD}"
QA_PRAEFECT_REPOSITORY_STORAGE: "default"
SIGNUP_DISABLED: "true"
- resource_group: release-environment-${CI_COMMIT_REF_SLUG}
release-environments-notification-failure:
stage: finish
diff --git a/.gitlab/ci/release-environments/security.gitlab-ci.yml b/.gitlab/ci/release-environments/security.gitlab-ci.yml
new file mode 100644
index 0000000000000..0b68cab6e4bda
--- /dev/null
+++ b/.gitlab/ci/release-environments/security.gitlab-ci.yml
@@ -0,0 +1,143 @@
+# Similar to .gitlab/ci/release-environments/main.gitlab-ci.yml, for release-environment pipelines in the security mirror.
+# Referenced in .gitlab/ci/release-environments.gitlab-ci.yml to differentiate from the canonical (main) version.
+# This file includes .gitlab/ci/cng/security.gitlab-ci.yml, instead of .gitlab/ci/cng/main.gitlab-ci.yml.
+---
+include:
+ - local: .gitlab/ci/cng/main.gitlab-ci.yml
+ inputs:
+ cng_path: 'charts/components/images'
+ - project: 'gitlab-org/quality/pipeline-common'
+ ref: '8.18.4'
+ file: ci/base.gitlab-ci.yml
+
+stages:
+ - prepare
+ - start
+ - deploy
+ - qa
+ - finish
+
+.inherit_variables:
+ inherit:
+ variables:
+ - GIT_DEPTH
+ - GIT_STRATEGY
+
+workflow:
+ auto_cancel:
+ on_new_commit: none
+
+variables:
+ GIT_DEPTH: 20
+ GIT_STRATEGY: fetch
+
+release-environments-build-cng-env:
+ extends: .build-cng-env
+
+release-environments-build-cng:
+ extends: .build-cng
+ needs: ["release-environments-build-cng-env"]
+ variables:
+ IMAGE_TAG_EXT: "-${CI_COMMIT_SHORT_SHA}"
+
+release-environments-deploy-env:
+ stage: prepare
+ needs: ["release-environments-build-cng"]
+ variables:
+ DEPLOY_ENV: deploy.env
+ script:
+ - ./scripts/release_environment/construct-release-environments-versions.rb
+ artifacts:
+ reports:
+ dotenv: $DEPLOY_ENV
+ paths:
+ - $DEPLOY_ENV
+ expire_in: 7 days
+ when: always
+
+release-environments-update-resource-group:
+ stage: prepare
+ script:
+ # Make sure pipelines run in order
+ # See https://docs.gitlab.com/ee/ci/resource_groups/index.html#change-the-process-mode
+ - |
+ curl --request PUT --data "process_mode=oldest_first" --header "PRIVATE-TOKEN:${ENVIRONMENT_API_TOKEN}" \
+ "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/resource_groups/release-environment-${CI_COMMIT_REF_SLUG}"
+
+release-environments-notification-start:
+ stage: start
+ extends: .inherit_variables
+ variables:
+ RELEASE_ENVIRONMENT_NOTIFICATION_TYPE: "deploy"
+ script:
+ - ruby scripts/release_environment/notification.rb
+ needs: ["release-environments-deploy-env"]
+
+release-environments-deploy:
+ stage: deploy
+ inherit:
+ variables: false
+ variables:
+ VERSIONS: "${VERSIONS}"
+ ENVIRONMENT: "${ENVIRONMENT}"
+ trigger:
+ project: gitlab-com/gl-infra/release-environments
+ branch: main
+ strategy: depend
+ needs: ["release-environments-deploy-env"]
+ resource_group: release-environment-${CI_COMMIT_REF_SLUG}
+
+release-environments-qa:
+ stage: qa
+ extends:
+ - .qa-base
+ timeout: 30m
+ parallel: 5
+ variables:
+ QA_SCENARIO: "Test::Instance::Smoke"
+ RELEASE: "${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab-ee-qa:${CI_COMMIT_SHA}"
+ GITLAB_QA_OPTS: --address "https://gitlab.${ENVIRONMENT}.release.gke.gitlab.net"
+ GITLAB_INITIAL_ROOT_PASSWORD: "${RELEASE_ENVIRONMENTS_ROOT_PASSWORD}"
+ QA_PRAEFECT_REPOSITORY_STORAGE: "default"
+ SIGNUP_DISABLED: "true"
+ before_script:
+ - !reference [.qa-base, before_script]
+ - echo "$CI_REGISTRY_PASSWORD" | docker login "$CI_REGISTRY" -u "$CI_REGISTRY_USER" --password-stdin
+
+release-environments-notification-failure:
+ stage: finish
+ extends: .inherit_variables
+ variables:
+ RELEASE_ENVIRONMENT_NOTIFICATION_TYPE: "deploy"
+ script:
+ - ruby scripts/release_environment/notification.rb
+ needs:
+ - job: release-environments-deploy
+ artifacts: false
+ - job: release-environments-deploy-env
+ when: on_failure
+
+release-environments-notification-success:
+ stage: finish
+ extends: .inherit_variables
+ variables:
+ RELEASE_ENVIRONMENT_NOTIFICATION_TYPE: "deploy"
+ script:
+ - ruby scripts/release_environment/notification.rb
+ needs:
+ - job: release-environments-qa
+ artifacts: false
+ - job: release-environments-deploy-env
+
+release-environments-notification-qa-failure:
+ stage: finish
+ extends: .inherit_variables
+ variables:
+ RELEASE_ENVIRONMENT_NOTIFICATION_TYPE: "qa"
+ script:
+ - ruby scripts/release_environment/notification.rb
+ needs:
+ - job: release-environments-qa
+ artifacts: false
+ - job: release-environments-deploy-env
+ when: on_failure
diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml
index 20fbbcd9ace1c..3b632fefe5504 100644
--- a/.gitlab/ci/rules.gitlab-ci.yml
+++ b/.gitlab/ci/rules.gitlab-ci.yml
@@ -176,6 +176,9 @@
.if-dot-com-gitlab-org-ee-tag: &if-dot-com-gitlab-org-ee-tag
if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/gitlab" && $CI_COMMIT_TAG =~ /^v?[\d]+\.[\d]+\.[\d]+[\d\w-]*-ee$/'
+.if-dot-com-gitlab-org-security-ee-tag: &if-dot-com-gitlab-org-security-ee-tag
+ if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/security/gitlab" && $CI_COMMIT_TAG =~ /^v?[\d]+\.[\d]+\.[\d]+[\d\w-]*-ee$/'
+
.if-ruby-branch: &if-ruby-branch
if: '$CI_COMMIT_BRANCH =~ /^ruby\d+(_\d)*$/ || (($CI_MERGE_REQUEST_EVENT_TYPE == "merged_result" || $CI_MERGE_REQUEST_EVENT_TYPE == "detached") && $CI_MERGE_REQUEST_LABELS =~ /pipeline:run-in-ruby\d+(_\d)*/)'
@@ -951,6 +954,7 @@
ARCH: amd64,arm64
- !reference [".build-images:rules:build-qa-image-merge-requests", rules]
- !reference [".releases:rules:canonical-dot-com-gitlab-stable-branch-only-setup-test-env", rules]
+ - !reference [".releases:rules:canonical-dot-com-security-gitlab-stable-branch-only-setup-test-env", rules]
.build-images:rules:build-qa-image-as-if-foss:
rules:
@@ -2531,6 +2535,13 @@
when: never
- if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/security/gitlab" && $CI_COMMIT_REF_NAME =~ /^[\d-]+-stable-ee$/'
+.releases:rules:canonical-dot-com-security-gitlab-stable-branch-only-setup-test-env:
+ rules:
+ - if: '$CI_COMMIT_MESSAGE =~ /\[merge-train skip\]/'
+ when: never
+ - if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/security/gitlab" && $CI_COMMIT_REF_NAME =~ /^[\d-]+-stable-ee$/'
+ changes: *setup-test-env-patterns
+
#################
# Reports rules #
#################
@@ -3282,6 +3293,16 @@
when: always
- !reference [".releases:rules:canonical-dot-com-gitlab-stable-branch-only", rules]
+.release-environments:rules:start-release-environments-security-pipeline:
+ rules:
+ - <<: *if-not-ee
+ when: never
+ - <<: *if-merge-request-labels-pipeline-expedite
+ when: never
+ - <<: *if-dot-com-gitlab-org-security-ee-tag
+ when: always
+ - !reference [".releases:rules:canonical-dot-com-security-gitlab-stable-branch-only", rules]
+
###################
# Benchmark rules #
###################
diff --git a/scripts/release_environment/construct-release-environments-versions.rb b/scripts/release_environment/construct-release-environments-versions.rb
index 569f947b20a61..f4b1f97a34725 100755
--- a/scripts/release_environment/construct-release-environments-versions.rb
+++ b/scripts/release_environment/construct-release-environments-versions.rb
@@ -21,7 +21,7 @@ class ReleaseEnvironmentsModel
def generate_json
output_json = {}
COMPONENTS.each do |component|
- output_json[component.to_s] = "#{environment}-#{ENV['CI_COMMIT_SHORT_SHA']}"
+ output_json[component.to_s] = image_tag.to_s
end
JSON.generate(output_json)
end
@@ -39,17 +39,35 @@ def set_required_env_vars?
end
def environment
- match = ENV['CI_COMMIT_REF_SLUG'].match(/^v?([\d]+)\.([\d]+)\.[\d]+[\d\w-]*-ee$/)
- @environment ||= if match
- "#{match[1]}-#{match[2]}-stable"
- else
- ENV['CI_COMMIT_REF_SLUG'].sub("-ee", "")
- end
+ @environment ||= environment_base + (security_project? ? "-security" : "")
+ end
+
+ def image_tag
+ @image_tag ||= "#{environment_base}-#{ENV['CI_COMMIT_SHORT_SHA']}"
+ end
+
+ private
+
+ # This is to generate the environment name without "-security". It is used by the image tag
+ def environment_base
+ @environment_base ||= if release_tag_match
+ "#{release_tag_match[1]}-#{release_tag_match[2]}-stable"
+ else
+ ENV['CI_COMMIT_REF_SLUG'].delete_suffix('-ee')
+ end
+ end
+
+ def release_tag_match
+ @release_tag_match ||= ENV['CI_COMMIT_REF_SLUG'].match(/^v?([\d]+)\.([\d]+)\.[\d]+[\d\w-]*-ee$/)
+ end
+
+ def security_project?
+ ENV['CI_PROJECT_PATH'] == "gitlab-org/security/gitlab"
end
end
# Outputs in `dotenv` format the ENVIRONMENT and VERSIONS to pass to release environments e.g.
-# ENVIRONMENT=15-10-stable
+# ENVIRONMENT=15-10-stable(-security)
# VERSIONS={"gitaly":"15-10-stable-c7c5131c","registry":"15-10-stable-c7c5131c","kas":"15-10-stable-c7c5131c", ...
if $PROGRAM_NAME == __FILE__
model = ReleaseEnvironmentsModel.new
diff --git a/scripts/trigger-build.rb b/scripts/trigger-build.rb
index 19b39ce702313..0218e5424c1a4 100755
--- a/scripts/trigger-build.rb
+++ b/scripts/trigger-build.rb
@@ -136,7 +136,7 @@ def fallback_ref
def normalize_stable_branch_name(branch_name)
if ENV['CI_PROJECT_NAMESPACE'] == 'gitlab-cn'
branch_name.delete_suffix('-jh')
- elsif ENV['CI_PROJECT_NAMESPACE'] == 'gitlab-org'
+ elsif ["gitlab-org", "gitlab-org/security"].include?(ENV['CI_PROJECT_NAMESPACE'])
branch_name.delete_suffix('-ee')
end
end
diff --git a/spec/scripts/release_environment/release_environments_model_spec.rb b/spec/scripts/release_environment/release_environments_model_spec.rb
index 9b93ede90d6c8..ca020f03c161e 100644
--- a/spec/scripts/release_environment/release_environments_model_spec.rb
+++ b/spec/scripts/release_environment/release_environments_model_spec.rb
@@ -42,24 +42,41 @@
end
describe '#environment' do
- context 'for stable branch' do
- it 'returns the correct environment' do
- stub_env('CI_COMMIT_REF_SLUG', '15-10-stable-ee')
- expect(model.environment).to eq('15-10-stable')
+ context 'when CI_PROJECT_PATH is not gitlab-org/security/gitlab' do
+ before do
+ stub_env('CI_PROJECT_PATH', 'gitlab-org/gitlab')
end
- end
- context 'for RC tag' do
- it 'returns the correct environment' do
- stub_env('CI_COMMIT_REF_SLUG', 'v15.10.3-rc42-ee')
- expect(model.environment).to eq('15-10-stable')
+ context 'for stable branch' do
+ it 'returns the correct environment' do
+ stub_env('CI_COMMIT_REF_SLUG', '15-10-stable-ee')
+ expect(model.environment).to eq('15-10-stable')
+ end
+ end
+
+ context 'for RC tag' do
+ it 'returns the correct environment' do
+ stub_env('CI_COMMIT_REF_SLUG', 'v15.10.3-rc42-ee')
+ expect(model.environment).to eq('15-10-stable')
+ end
+ end
+
+ context 'for release tag' do
+ it 'returns the correct environment' do
+ stub_env('CI_COMMIT_REF_SLUG', 'v15.10.3-ee')
+ expect(model.environment).to eq('15-10-stable')
+ end
end
end
- context 'for release tag' do
- it 'returns the correct environment' do
- stub_env('CI_COMMIT_REF_SLUG', 'v15.10.3-ee')
- expect(model.environment).to eq('15-10-stable')
+ context 'when CI_PROJECT_PATH is gitlab-org/security/gitlab' do
+ before do
+ stub_env('CI_PROJECT_PATH', 'gitlab-org/security/gitlab')
+ stub_env('CI_COMMIT_REF_SLUG', '15-10-stable-ee')
+ end
+
+ it 'returns the environment with -security' do
+ expect(model.environment).to eq('15-10-stable-security')
end
end
end
--
GitLab