Merge branch 'jennykim/release-environment-trigger-from-security' into 'master'

Trigger release-environment pipeline from security stable branches

See merge request https://gitlab.com/gitlab-org/gitlab/-/merge_requests/154669



Merged-by: Peter Leitzen <pleitzen@gitlab.com>
Approved-by: Rémy Coutable <remy@rymai.me>
Approved-by: Peter Leitzen <pleitzen@gitlab.com>
Reviewed-by: Mayra Cabrera <mcabrera@gitlab.com>
Reviewed-by: Rémy Coutable <remy@rymai.me>
Co-authored-by: Rémy Coutable <remy@rymai.me>
Co-authored-by: Dat Tang <dattang@gitlab.com>
Co-authored-by: Jenny Kim <yjeankim@gitlab.com>

(cherry picked from commit b2728d7a)

ee55efbe Security stable branch commits and commit tags trigger release-environment pipeline
e99dbd6c Build QA image for security stable branches
e7f9cf7f Remove temporary comment
38226b9f remove "security" path portion of CI_PROJECT_NAMESPACE
81031959 Match security mirror when processing TRIGGER_BRANCH
8d588eaa Correct image tags in VERSIONS variable
70388adf Remove resource group for release environments QA
765b0da5 Add docker login to security release environments QA job
70515ad4 Add rspec to test security environment name
9d4d7037 Apply 3 suggestion(s) to 2 file(s)
3da91fdf Remove unused method in construct RE script
166e712e Merge security cng template
386cc6e5 Merge branch 'master' into 'jennykim/release-environment-trigger-from-security'

Co-authored-by: Peter Leitzen <pleitzen@gitlab.com>

.gitlab/ci/cng/main.gitlab-ci.yml

+spec:
+  inputs:
+    cng_path:
+      type: string
+      default: 'build/CNG-mirror'
 ---
 default:
   interruptible: true
@@ -61,6 +66,6 @@ include:
     TOP_UPSTREAM_MERGE_REQUEST_IID: "${TOP_UPSTREAM_MERGE_REQUEST_IID}"
     TOP_UPSTREAM_SOURCE_SHA: "${TOP_UPSTREAM_SOURCE_SHA}"
   trigger:
-    project: ${CI_PROJECT_NAMESPACE}/build/CNG-mirror
+    project: '${CI_PROJECT_NAMESPACE}/$[[ inputs.cng_path ]]'
     branch: $TRIGGER_BRANCH
     strategy: depend

.gitlab/ci/release-environments.gitlab-ci.yml

@@ -26,3 +26,31 @@ start-release-environments-pipeline:
       - project: 'gitlab-org/gitlab'
         ref: 'master'
         file: '.gitlab/ci/release-environments/main.gitlab-ci.yml'
+
+start-release-environments-security-pipeline:
+  allow_failure: true
+  extends:
+    - .release-environments:rules:start-release-environments-security-pipeline
+  stage: release-environments
+  # We do not want to have ALL global variables passed as trigger variables,
+  # as they cannot be overridden. See this issue for more context:
+  #
+  # https://gitlab.com/gitlab-org/gitlab/-/issues/387183
+  inherit:
+    variables:
+      - RUBY_VERSION_DEFAULT
+      - RUBY_VERSION_NEXT
+      - RUBY_VERSION
+
+  # These variables are set in the pipeline schedules.
+  # They need to be explicitly passed on to the child pipeline.
+  # https://docs.gitlab.com/ee/ci/pipelines/multi_project_pipelines.html#pass-cicd-variables-to-a-downstream-pipeline-by-using-the-variables-keyword
+  variables:
+    # This is needed by `release-environments-build-cng-env` (`.gitlab/ci/release-environments/security.gitlab-ci.yml`).
+    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
+  trigger:
+    strategy: depend
+    include:
+      - project: 'gitlab-org/security/gitlab'
+        ref: 'master'
+        file: '.gitlab/ci/release-environments/security.gitlab-ci.yml'

.gitlab/ci/release-environments/main.gitlab-ci.yml

 ---
 include:
   - local: .gitlab/ci/cng/main.gitlab-ci.yml
+    inputs:
+      cng_path: 'build/CNG-mirror'
   - project: 'gitlab-org/quality/pipeline-common'
-    ref: '8.18.3'
+    ref: '8.18.4'
     file: ci/base.gitlab-ci.yml
 
 stages:
@@ -95,7 +97,6 @@ release-environments-qa:
     GITLAB_INITIAL_ROOT_PASSWORD: "${RELEASE_ENVIRONMENTS_ROOT_PASSWORD}"
     QA_PRAEFECT_REPOSITORY_STORAGE: "default"
     SIGNUP_DISABLED: "true"
-  resource_group: release-environment-${CI_COMMIT_REF_SLUG}
 
 release-environments-notification-failure:
   stage: finish

.gitlab/ci/release-environments/security.gitlab-ci.yml

+# Similar to .gitlab/ci/release-environments/main.gitlab-ci.yml, for release-environment pipelines in the security mirror.
+# Referenced in .gitlab/ci/release-environments.gitlab-ci.yml to differentiate from the canonical (main) version.
+# This file includes .gitlab/ci/cng/security.gitlab-ci.yml, instead of .gitlab/ci/cng/main.gitlab-ci.yml.
+---
+include:
+  - local: .gitlab/ci/cng/main.gitlab-ci.yml
+    inputs:
+      cng_path: 'charts/components/images'
+  - project: 'gitlab-org/quality/pipeline-common'
+    ref: '8.18.4'
+    file: ci/base.gitlab-ci.yml
+
+stages:
+  - prepare
+  - start
+  - deploy
+  - qa
+  - finish
+
+.inherit_variables:
+  inherit:
+    variables:
+      - GIT_DEPTH
+      - GIT_STRATEGY
+
+workflow:
+  auto_cancel:
+    on_new_commit: none
+
+variables:
+  GIT_DEPTH: 20
+  GIT_STRATEGY: fetch
+
+release-environments-build-cng-env:
+  extends: .build-cng-env
+
+release-environments-build-cng:
+  extends: .build-cng
+  needs: ["release-environments-build-cng-env"]
+  variables:
+    IMAGE_TAG_EXT: "-${CI_COMMIT_SHORT_SHA}"
+
+release-environments-deploy-env:
+  stage: prepare
+  needs: ["release-environments-build-cng"]
+  variables:
+    DEPLOY_ENV: deploy.env
+  script:
+    - ./scripts/release_environment/construct-release-environments-versions.rb
+  artifacts:
+    reports:
+      dotenv: $DEPLOY_ENV
+    paths:
+      - $DEPLOY_ENV
+    expire_in: 7 days
+    when: always
+
+release-environments-update-resource-group:
+  stage: prepare
+  script:
+    # Make sure pipelines run in order
+    # See https://docs.gitlab.com/ee/ci/resource_groups/index.html#change-the-process-mode
+    - |
+      curl --request PUT --data "process_mode=oldest_first" --header "PRIVATE-TOKEN:${ENVIRONMENT_API_TOKEN}" \
+      "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/resource_groups/release-environment-${CI_COMMIT_REF_SLUG}"
+
+release-environments-notification-start:
+  stage: start
+  extends: .inherit_variables
+  variables:
+    RELEASE_ENVIRONMENT_NOTIFICATION_TYPE: "deploy"
+  script:
+    - ruby scripts/release_environment/notification.rb
+  needs: ["release-environments-deploy-env"]
+
+release-environments-deploy:
+  stage: deploy
+  inherit:
+    variables: false
+  variables:
+    VERSIONS: "${VERSIONS}"
+    ENVIRONMENT: "${ENVIRONMENT}"
+  trigger:
+    project: gitlab-com/gl-infra/release-environments
+    branch: main
+    strategy: depend
+  needs: ["release-environments-deploy-env"]
+  resource_group: release-environment-${CI_COMMIT_REF_SLUG}
+
+release-environments-qa:
+  stage: qa
+  extends:
+    - .qa-base
+  timeout: 30m
+  parallel: 5
+  variables:
+    QA_SCENARIO: "Test::Instance::Smoke"
+    RELEASE: "${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab-ee-qa:${CI_COMMIT_SHA}"
+    GITLAB_QA_OPTS: --address "https://gitlab.${ENVIRONMENT}.release.gke.gitlab.net"
+    GITLAB_INITIAL_ROOT_PASSWORD: "${RELEASE_ENVIRONMENTS_ROOT_PASSWORD}"
+    QA_PRAEFECT_REPOSITORY_STORAGE: "default"
+    SIGNUP_DISABLED: "true"
+  before_script:
+    - !reference [.qa-base, before_script]
+    - echo "$CI_REGISTRY_PASSWORD" | docker login "$CI_REGISTRY" -u "$CI_REGISTRY_USER" --password-stdin
+
+release-environments-notification-failure:
+  stage: finish
+  extends: .inherit_variables
+  variables:
+    RELEASE_ENVIRONMENT_NOTIFICATION_TYPE: "deploy"
+  script:
+    - ruby scripts/release_environment/notification.rb
+  needs:
+    - job: release-environments-deploy
+      artifacts: false
+    - job: release-environments-deploy-env
+  when: on_failure
+
+release-environments-notification-success:
+  stage: finish
+  extends: .inherit_variables
+  variables:
+    RELEASE_ENVIRONMENT_NOTIFICATION_TYPE: "deploy"
+  script:
+    - ruby scripts/release_environment/notification.rb
+  needs:
+    - job: release-environments-qa
+      artifacts: false
+    - job: release-environments-deploy-env
+
+release-environments-notification-qa-failure:
+  stage: finish
+  extends: .inherit_variables
+  variables:
+    RELEASE_ENVIRONMENT_NOTIFICATION_TYPE: "qa"
+  script:
+    - ruby scripts/release_environment/notification.rb
+  needs:
+    - job: release-environments-qa
+      artifacts: false
+    - job: release-environments-deploy-env
+  when: on_failure

.gitlab/ci/rules.gitlab-ci.yml

@@ -176,6 +176,9 @@
 .if-dot-com-gitlab-org-ee-tag: &if-dot-com-gitlab-org-ee-tag
   if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/gitlab" && $CI_COMMIT_TAG =~ /^v?[\d]+\.[\d]+\.[\d]+[\d\w-]*-ee$/'
 
+.if-dot-com-gitlab-org-security-ee-tag: &if-dot-com-gitlab-org-security-ee-tag
+  if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/security/gitlab" && $CI_COMMIT_TAG =~ /^v?[\d]+\.[\d]+\.[\d]+[\d\w-]*-ee$/'
+
 .if-ruby-branch: &if-ruby-branch
   if: '$CI_COMMIT_BRANCH =~ /^ruby\d+(_\d)*$/ || (($CI_MERGE_REQUEST_EVENT_TYPE == "merged_result" || $CI_MERGE_REQUEST_EVENT_TYPE == "detached") && $CI_MERGE_REQUEST_LABELS =~ /pipeline:run-in-ruby\d+(_\d)*/)'
 
@@ -951,6 +954,7 @@
         ARCH: amd64,arm64
     - !reference [".build-images:rules:build-qa-image-merge-requests", rules]
     - !reference [".releases:rules:canonical-dot-com-gitlab-stable-branch-only-setup-test-env", rules]
+    - !reference [".releases:rules:canonical-dot-com-security-gitlab-stable-branch-only-setup-test-env", rules]
 
 .build-images:rules:build-qa-image-as-if-foss:
   rules:
@@ -2531,6 +2535,13 @@
       when: never
     - if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/security/gitlab" && $CI_COMMIT_REF_NAME =~ /^[\d-]+-stable-ee$/'
 
+.releases:rules:canonical-dot-com-security-gitlab-stable-branch-only-setup-test-env:
+  rules:
+    - if: '$CI_COMMIT_MESSAGE =~ /\[merge-train skip\]/'
+      when: never
+    - if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/security/gitlab" && $CI_COMMIT_REF_NAME =~ /^[\d-]+-stable-ee$/'
+      changes: *setup-test-env-patterns
+
 #################
 # Reports rules #
 #################
@@ -3282,6 +3293,16 @@
       when: always
     - !reference [".releases:rules:canonical-dot-com-gitlab-stable-branch-only", rules]
 
+.release-environments:rules:start-release-environments-security-pipeline:
+  rules:
+    - <<: *if-not-ee
+      when: never
+    - <<: *if-merge-request-labels-pipeline-expedite
+      when: never
+    - <<: *if-dot-com-gitlab-org-security-ee-tag
+      when: always
+    - !reference [".releases:rules:canonical-dot-com-security-gitlab-stable-branch-only", rules]
+
 ###################
 # Benchmark rules #
 ###################

scripts/release_environment/construct-release-environments-versions.rb

@@ -21,7 +21,7 @@ class ReleaseEnvironmentsModel
   def generate_json
     output_json = {}
     COMPONENTS.each do |component|
-      output_json[component.to_s] = "#{environment}-#{ENV['CI_COMMIT_SHORT_SHA']}"
+      output_json[component.to_s] = image_tag.to_s
     end
     JSON.generate(output_json)
   end
@@ -39,17 +39,35 @@ def set_required_env_vars?
   end
 
   def environment
-    match = ENV['CI_COMMIT_REF_SLUG'].match(/^v?([\d]+)\.([\d]+)\.[\d]+[\d\w-]*-ee$/)
-    @environment ||= if match
-                       "#{match[1]}-#{match[2]}-stable"
-                     else
-                       ENV['CI_COMMIT_REF_SLUG'].sub("-ee", "")
-                     end
+    @environment ||= environment_base + (security_project? ? "-security" : "")
+  end
+
+  def image_tag
+    @image_tag ||= "#{environment_base}-#{ENV['CI_COMMIT_SHORT_SHA']}"
+  end
+
+  private
+
+  # This is to generate the environment name without "-security". It is used by the image tag
+  def environment_base
+    @environment_base ||= if release_tag_match
+                            "#{release_tag_match[1]}-#{release_tag_match[2]}-stable"
+                          else
+                            ENV['CI_COMMIT_REF_SLUG'].delete_suffix('-ee')
+                          end
+  end
+
+  def release_tag_match
+    @release_tag_match ||= ENV['CI_COMMIT_REF_SLUG'].match(/^v?([\d]+)\.([\d]+)\.[\d]+[\d\w-]*-ee$/)
+  end
+
+  def security_project?
+    ENV['CI_PROJECT_PATH'] == "gitlab-org/security/gitlab"
   end
 end
 
 # Outputs in `dotenv` format the ENVIRONMENT and VERSIONS to pass to release environments e.g.
-# ENVIRONMENT=15-10-stable
+# ENVIRONMENT=15-10-stable(-security)
 # VERSIONS={"gitaly":"15-10-stable-c7c5131c","registry":"15-10-stable-c7c5131c","kas":"15-10-stable-c7c5131c", ...
 if $PROGRAM_NAME == __FILE__
   model = ReleaseEnvironmentsModel.new

scripts/trigger-build.rb

@@ -136,7 +136,7 @@ def fallback_ref
     def normalize_stable_branch_name(branch_name)
       if ENV['CI_PROJECT_NAMESPACE'] == 'gitlab-cn'
         branch_name.delete_suffix('-jh')
-      elsif ENV['CI_PROJECT_NAMESPACE'] == 'gitlab-org'
+      elsif ["gitlab-org", "gitlab-org/security"].include?(ENV['CI_PROJECT_NAMESPACE'])
         branch_name.delete_suffix('-ee')
       end
     end

spec/scripts/release_environment/release_environments_model_spec.rb

@@ -42,24 +42,41 @@
   end
 
   describe '#environment' do
-    context 'for stable branch' do
-      it 'returns the correct environment' do
-        stub_env('CI_COMMIT_REF_SLUG', '15-10-stable-ee')
-        expect(model.environment).to eq('15-10-stable')
+    context 'when CI_PROJECT_PATH is not gitlab-org/security/gitlab' do
+      before do
+        stub_env('CI_PROJECT_PATH', 'gitlab-org/gitlab')
       end
-    end
 
-    context 'for RC tag' do
-      it 'returns the correct environment' do
-        stub_env('CI_COMMIT_REF_SLUG', 'v15.10.3-rc42-ee')
-        expect(model.environment).to eq('15-10-stable')
+      context 'for stable branch' do
+        it 'returns the correct environment' do
+          stub_env('CI_COMMIT_REF_SLUG', '15-10-stable-ee')
+          expect(model.environment).to eq('15-10-stable')
+        end
+      end
+
+      context 'for RC tag' do
+        it 'returns the correct environment' do
+          stub_env('CI_COMMIT_REF_SLUG', 'v15.10.3-rc42-ee')
+          expect(model.environment).to eq('15-10-stable')
+        end
+      end
+
+      context 'for release tag' do
+        it 'returns the correct environment' do
+          stub_env('CI_COMMIT_REF_SLUG', 'v15.10.3-ee')
+          expect(model.environment).to eq('15-10-stable')
+        end
       end
     end
 
-    context 'for release tag' do
-      it 'returns the correct environment' do
-        stub_env('CI_COMMIT_REF_SLUG', 'v15.10.3-ee')
-        expect(model.environment).to eq('15-10-stable')
+    context 'when CI_PROJECT_PATH is gitlab-org/security/gitlab' do
+      before do
+        stub_env('CI_PROJECT_PATH', 'gitlab-org/security/gitlab')
+        stub_env('CI_COMMIT_REF_SLUG', '15-10-stable-ee')
+      end
+
+      it 'returns the environment with -security' do
+        expect(model.environment).to eq('15-10-stable-security')
       end
     end
   end