diff --git a/.gitlab/ci/cng/main.gitlab-ci.yml b/.gitlab/ci/cng/main.gitlab-ci.yml index 214669fa4d9898e655e10948c59e3944cfedc8e8..af33e3db6376c8b79774f8603cbd93a0e77ea65b 100644 --- a/.gitlab/ci/cng/main.gitlab-ci.yml +++ b/.gitlab/ci/cng/main.gitlab-ci.yml @@ -1,3 +1,8 @@ +spec: + inputs: + cng_path: + type: string + default: 'build/CNG-mirror' --- default: interruptible: true @@ -61,6 +66,6 @@ include: TOP_UPSTREAM_MERGE_REQUEST_IID: "${TOP_UPSTREAM_MERGE_REQUEST_IID}" TOP_UPSTREAM_SOURCE_SHA: "${TOP_UPSTREAM_SOURCE_SHA}" trigger: - project: ${CI_PROJECT_NAMESPACE}/build/CNG-mirror + project: '${CI_PROJECT_NAMESPACE}/$[[ inputs.cng_path ]]' branch: $TRIGGER_BRANCH strategy: depend diff --git a/.gitlab/ci/release-environments.gitlab-ci.yml b/.gitlab/ci/release-environments.gitlab-ci.yml index a6a0e268451c5a9411b037d4235b8fee6a198f12..bcd1a3b047c3b3a1283aafab76b14566c7e1bc89 100644 --- a/.gitlab/ci/release-environments.gitlab-ci.yml +++ b/.gitlab/ci/release-environments.gitlab-ci.yml @@ -26,3 +26,31 @@ start-release-environments-pipeline: - project: 'gitlab-org/gitlab' ref: 'master' file: '.gitlab/ci/release-environments/main.gitlab-ci.yml' + +start-release-environments-security-pipeline: + allow_failure: true + extends: + - .release-environments:rules:start-release-environments-security-pipeline + stage: release-environments + # We do not want to have ALL global variables passed as trigger variables, + # as they cannot be overridden. See this issue for more context: + # + # https://gitlab.com/gitlab-org/gitlab/-/issues/387183 + inherit: + variables: + - RUBY_VERSION_DEFAULT + - RUBY_VERSION_NEXT + - RUBY_VERSION + + # These variables are set in the pipeline schedules. + # They need to be explicitly passed on to the child pipeline. + # https://docs.gitlab.com/ee/ci/pipelines/multi_project_pipelines.html#pass-cicd-variables-to-a-downstream-pipeline-by-using-the-variables-keyword + variables: + # This is needed by `release-environments-build-cng-env` (`.gitlab/ci/release-environments/security.gitlab-ci.yml`). + PARENT_PIPELINE_ID: $CI_PIPELINE_ID + trigger: + strategy: depend + include: + - project: 'gitlab-org/security/gitlab' + ref: 'master' + file: '.gitlab/ci/release-environments/security.gitlab-ci.yml' diff --git a/.gitlab/ci/release-environments/main.gitlab-ci.yml b/.gitlab/ci/release-environments/main.gitlab-ci.yml index d1097e8326b53be7beb67b76c8407dced45c6a33..a8a05965132ad264f0329d50b3c7353c95466762 100644 --- a/.gitlab/ci/release-environments/main.gitlab-ci.yml +++ b/.gitlab/ci/release-environments/main.gitlab-ci.yml @@ -1,8 +1,10 @@ --- include: - local: .gitlab/ci/cng/main.gitlab-ci.yml + inputs: + cng_path: 'build/CNG-mirror' - project: 'gitlab-org/quality/pipeline-common' - ref: '8.18.3' + ref: '8.18.4' file: ci/base.gitlab-ci.yml stages: @@ -95,7 +97,6 @@ release-environments-qa: GITLAB_INITIAL_ROOT_PASSWORD: "${RELEASE_ENVIRONMENTS_ROOT_PASSWORD}" QA_PRAEFECT_REPOSITORY_STORAGE: "default" SIGNUP_DISABLED: "true" - resource_group: release-environment-${CI_COMMIT_REF_SLUG} release-environments-notification-failure: stage: finish diff --git a/.gitlab/ci/release-environments/security.gitlab-ci.yml b/.gitlab/ci/release-environments/security.gitlab-ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..0b68cab6e4bda27e3275c9e75a06464eba710d76 --- /dev/null +++ b/.gitlab/ci/release-environments/security.gitlab-ci.yml @@ -0,0 +1,143 @@ +# Similar to .gitlab/ci/release-environments/main.gitlab-ci.yml, for release-environment pipelines in the security mirror. +# Referenced in .gitlab/ci/release-environments.gitlab-ci.yml to differentiate from the canonical (main) version. +# This file includes .gitlab/ci/cng/security.gitlab-ci.yml, instead of .gitlab/ci/cng/main.gitlab-ci.yml. +--- +include: + - local: .gitlab/ci/cng/main.gitlab-ci.yml + inputs: + cng_path: 'charts/components/images' + - project: 'gitlab-org/quality/pipeline-common' + ref: '8.18.4' + file: ci/base.gitlab-ci.yml + +stages: + - prepare + - start + - deploy + - qa + - finish + +.inherit_variables: + inherit: + variables: + - GIT_DEPTH + - GIT_STRATEGY + +workflow: + auto_cancel: + on_new_commit: none + +variables: + GIT_DEPTH: 20 + GIT_STRATEGY: fetch + +release-environments-build-cng-env: + extends: .build-cng-env + +release-environments-build-cng: + extends: .build-cng + needs: ["release-environments-build-cng-env"] + variables: + IMAGE_TAG_EXT: "-${CI_COMMIT_SHORT_SHA}" + +release-environments-deploy-env: + stage: prepare + needs: ["release-environments-build-cng"] + variables: + DEPLOY_ENV: deploy.env + script: + - ./scripts/release_environment/construct-release-environments-versions.rb + artifacts: + reports: + dotenv: $DEPLOY_ENV + paths: + - $DEPLOY_ENV + expire_in: 7 days + when: always + +release-environments-update-resource-group: + stage: prepare + script: + # Make sure pipelines run in order + # See https://docs.gitlab.com/ee/ci/resource_groups/index.html#change-the-process-mode + - | + curl --request PUT --data "process_mode=oldest_first" --header "PRIVATE-TOKEN:${ENVIRONMENT_API_TOKEN}" \ + "${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/resource_groups/release-environment-${CI_COMMIT_REF_SLUG}" + +release-environments-notification-start: + stage: start + extends: .inherit_variables + variables: + RELEASE_ENVIRONMENT_NOTIFICATION_TYPE: "deploy" + script: + - ruby scripts/release_environment/notification.rb + needs: ["release-environments-deploy-env"] + +release-environments-deploy: + stage: deploy + inherit: + variables: false + variables: + VERSIONS: "${VERSIONS}" + ENVIRONMENT: "${ENVIRONMENT}" + trigger: + project: gitlab-com/gl-infra/release-environments + branch: main + strategy: depend + needs: ["release-environments-deploy-env"] + resource_group: release-environment-${CI_COMMIT_REF_SLUG} + +release-environments-qa: + stage: qa + extends: + - .qa-base + timeout: 30m + parallel: 5 + variables: + QA_SCENARIO: "Test::Instance::Smoke" + RELEASE: "${CI_REGISTRY}/${CI_PROJECT_PATH}/gitlab-ee-qa:${CI_COMMIT_SHA}" + GITLAB_QA_OPTS: --address "https://gitlab.${ENVIRONMENT}.release.gke.gitlab.net" + GITLAB_INITIAL_ROOT_PASSWORD: "${RELEASE_ENVIRONMENTS_ROOT_PASSWORD}" + QA_PRAEFECT_REPOSITORY_STORAGE: "default" + SIGNUP_DISABLED: "true" + before_script: + - !reference [.qa-base, before_script] + - echo "$CI_REGISTRY_PASSWORD" | docker login "$CI_REGISTRY" -u "$CI_REGISTRY_USER" --password-stdin + +release-environments-notification-failure: + stage: finish + extends: .inherit_variables + variables: + RELEASE_ENVIRONMENT_NOTIFICATION_TYPE: "deploy" + script: + - ruby scripts/release_environment/notification.rb + needs: + - job: release-environments-deploy + artifacts: false + - job: release-environments-deploy-env + when: on_failure + +release-environments-notification-success: + stage: finish + extends: .inherit_variables + variables: + RELEASE_ENVIRONMENT_NOTIFICATION_TYPE: "deploy" + script: + - ruby scripts/release_environment/notification.rb + needs: + - job: release-environments-qa + artifacts: false + - job: release-environments-deploy-env + +release-environments-notification-qa-failure: + stage: finish + extends: .inherit_variables + variables: + RELEASE_ENVIRONMENT_NOTIFICATION_TYPE: "qa" + script: + - ruby scripts/release_environment/notification.rb + needs: + - job: release-environments-qa + artifacts: false + - job: release-environments-deploy-env + when: on_failure diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml index 20fbbcd9ace1c722f3a0b4e16f0dcefa2508290e..3b632fefe5504dc378d74baa7c04ddc8988a7543 100644 --- a/.gitlab/ci/rules.gitlab-ci.yml +++ b/.gitlab/ci/rules.gitlab-ci.yml @@ -176,6 +176,9 @@ .if-dot-com-gitlab-org-ee-tag: &if-dot-com-gitlab-org-ee-tag if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/gitlab" && $CI_COMMIT_TAG =~ /^v?[\d]+\.[\d]+\.[\d]+[\d\w-]*-ee$/' +.if-dot-com-gitlab-org-security-ee-tag: &if-dot-com-gitlab-org-security-ee-tag + if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/security/gitlab" && $CI_COMMIT_TAG =~ /^v?[\d]+\.[\d]+\.[\d]+[\d\w-]*-ee$/' + .if-ruby-branch: &if-ruby-branch if: '$CI_COMMIT_BRANCH =~ /^ruby\d+(_\d)*$/ || (($CI_MERGE_REQUEST_EVENT_TYPE == "merged_result" || $CI_MERGE_REQUEST_EVENT_TYPE == "detached") && $CI_MERGE_REQUEST_LABELS =~ /pipeline:run-in-ruby\d+(_\d)*/)' @@ -951,6 +954,7 @@ ARCH: amd64,arm64 - !reference [".build-images:rules:build-qa-image-merge-requests", rules] - !reference [".releases:rules:canonical-dot-com-gitlab-stable-branch-only-setup-test-env", rules] + - !reference [".releases:rules:canonical-dot-com-security-gitlab-stable-branch-only-setup-test-env", rules] .build-images:rules:build-qa-image-as-if-foss: rules: @@ -2531,6 +2535,13 @@ when: never - if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/security/gitlab" && $CI_COMMIT_REF_NAME =~ /^[\d-]+-stable-ee$/' +.releases:rules:canonical-dot-com-security-gitlab-stable-branch-only-setup-test-env: + rules: + - if: '$CI_COMMIT_MESSAGE =~ /\[merge-train skip\]/' + when: never + - if: '$CI_SERVER_HOST == "gitlab.com" && $CI_PROJECT_PATH == "gitlab-org/security/gitlab" && $CI_COMMIT_REF_NAME =~ /^[\d-]+-stable-ee$/' + changes: *setup-test-env-patterns + ################# # Reports rules # ################# @@ -3282,6 +3293,16 @@ when: always - !reference [".releases:rules:canonical-dot-com-gitlab-stable-branch-only", rules] +.release-environments:rules:start-release-environments-security-pipeline: + rules: + - <<: *if-not-ee + when: never + - <<: *if-merge-request-labels-pipeline-expedite + when: never + - <<: *if-dot-com-gitlab-org-security-ee-tag + when: always + - !reference [".releases:rules:canonical-dot-com-security-gitlab-stable-branch-only", rules] + ################### # Benchmark rules # ################### diff --git a/scripts/release_environment/construct-release-environments-versions.rb b/scripts/release_environment/construct-release-environments-versions.rb index 569f947b20a61760d7c07aa8d7e2e47747cb6688..f4b1f97a34725e87dd78c3ea39b6e9f71686a9a6 100755 --- a/scripts/release_environment/construct-release-environments-versions.rb +++ b/scripts/release_environment/construct-release-environments-versions.rb @@ -21,7 +21,7 @@ class ReleaseEnvironmentsModel def generate_json output_json = {} COMPONENTS.each do |component| - output_json[component.to_s] = "#{environment}-#{ENV['CI_COMMIT_SHORT_SHA']}" + output_json[component.to_s] = image_tag.to_s end JSON.generate(output_json) end @@ -39,17 +39,35 @@ def set_required_env_vars? end def environment - match = ENV['CI_COMMIT_REF_SLUG'].match(/^v?([\d]+)\.([\d]+)\.[\d]+[\d\w-]*-ee$/) - @environment ||= if match - "#{match[1]}-#{match[2]}-stable" - else - ENV['CI_COMMIT_REF_SLUG'].sub("-ee", "") - end + @environment ||= environment_base + (security_project? ? "-security" : "") + end + + def image_tag + @image_tag ||= "#{environment_base}-#{ENV['CI_COMMIT_SHORT_SHA']}" + end + + private + + # This is to generate the environment name without "-security". It is used by the image tag + def environment_base + @environment_base ||= if release_tag_match + "#{release_tag_match[1]}-#{release_tag_match[2]}-stable" + else + ENV['CI_COMMIT_REF_SLUG'].delete_suffix('-ee') + end + end + + def release_tag_match + @release_tag_match ||= ENV['CI_COMMIT_REF_SLUG'].match(/^v?([\d]+)\.([\d]+)\.[\d]+[\d\w-]*-ee$/) + end + + def security_project? + ENV['CI_PROJECT_PATH'] == "gitlab-org/security/gitlab" end end # Outputs in `dotenv` format the ENVIRONMENT and VERSIONS to pass to release environments e.g. -# ENVIRONMENT=15-10-stable +# ENVIRONMENT=15-10-stable(-security) # VERSIONS={"gitaly":"15-10-stable-c7c5131c","registry":"15-10-stable-c7c5131c","kas":"15-10-stable-c7c5131c", ... if $PROGRAM_NAME == __FILE__ model = ReleaseEnvironmentsModel.new diff --git a/scripts/trigger-build.rb b/scripts/trigger-build.rb index 19b39ce7023133f42e456395c5b4aaed14728c2d..0218e5424c1a4ca5cc14078f95f197ee43dd4b72 100755 --- a/scripts/trigger-build.rb +++ b/scripts/trigger-build.rb @@ -136,7 +136,7 @@ def fallback_ref def normalize_stable_branch_name(branch_name) if ENV['CI_PROJECT_NAMESPACE'] == 'gitlab-cn' branch_name.delete_suffix('-jh') - elsif ENV['CI_PROJECT_NAMESPACE'] == 'gitlab-org' + elsif ["gitlab-org", "gitlab-org/security"].include?(ENV['CI_PROJECT_NAMESPACE']) branch_name.delete_suffix('-ee') end end diff --git a/spec/scripts/release_environment/release_environments_model_spec.rb b/spec/scripts/release_environment/release_environments_model_spec.rb index 9b93ede90d6c8d8599a66fc7dd66b4a919945559..0bcd721002fccd3c8c578f398e89d3781ce983d0 100644 --- a/spec/scripts/release_environment/release_environments_model_spec.rb +++ b/spec/scripts/release_environment/release_environments_model_spec.rb @@ -42,24 +42,37 @@ end describe '#environment' do - context 'for stable branch' do - it 'returns the correct environment' do - stub_env('CI_COMMIT_REF_SLUG', '15-10-stable-ee') - expect(model.environment).to eq('15-10-stable') + context 'when CI_PROJECT_PATH is not gitlab-org/security/gitlab' do + context 'for stable branch' do + it 'returns the correct environment' do + stub_env('CI_COMMIT_REF_SLUG', '15-10-stable-ee') + expect(model.environment).to eq('15-10-stable') + end + end + + context 'for RC tag' do + it 'returns the correct environment' do + stub_env('CI_COMMIT_REF_SLUG', 'v15.10.3-rc42-ee') + expect(model.environment).to eq('15-10-stable') + end end - end - context 'for RC tag' do - it 'returns the correct environment' do - stub_env('CI_COMMIT_REF_SLUG', 'v15.10.3-rc42-ee') - expect(model.environment).to eq('15-10-stable') + context 'for release tag' do + it 'returns the correct environment' do + stub_env('CI_COMMIT_REF_SLUG', 'v15.10.3-ee') + expect(model.environment).to eq('15-10-stable') + end end end - context 'for release tag' do - it 'returns the correct environment' do - stub_env('CI_COMMIT_REF_SLUG', 'v15.10.3-ee') - expect(model.environment).to eq('15-10-stable') + context 'when CI_PROJECT_PATH is gitlab-org/security/gitlab' do + before do + stub_env('CI_PROJECT_PATH', 'gitlab-org/security/gitlab') + stub_env('CI_COMMIT_REF_SLUG', '15-10-stable-ee') + end + + it 'returns the environment with -security' do + expect(model.environment).to eq('15-10-stable-security') end end end