From 208220e3fbe02b116745f9ebfa6bdcacdff9700b Mon Sep 17 00:00:00 2001
From: Andrejs Cunskis <acunskis@gitlab.com>
Date: Tue, 6 Jun 2023 11:36:00 +0000
Subject: [PATCH] Extract omnibus-trigger build outside of package-and-test
pipeline
---
.gitlab/ci/build-images.gitlab-ci.yml | 43 +++++++++
.../main.gitlab-ci.yml | 10 ---
.../ci/package-and-test/main.gitlab-ci.yml | 24 -----
.gitlab/ci/qa-common/main.gitlab-ci.yml | 88 -------------------
.gitlab/ci/qa-common/rules.gitlab-ci.yml | 12 ---
.gitlab/ci/qa.gitlab-ci.yml | 11 +--
.gitlab/ci/setup.gitlab-ci.yml | 42 +++++++++
scripts/generate-e2e-pipeline | 1 -
8 files changed, 89 insertions(+), 142 deletions(-)
diff --git a/.gitlab/ci/build-images.gitlab-ci.yml b/.gitlab/ci/build-images.gitlab-ci.yml
index 33cc97cd1d5f6..c537644f78e68 100644
--- a/.gitlab/ci/build-images.gitlab-ci.yml
+++ b/.gitlab/ci/build-images.gitlab-ci.yml
@@ -62,3 +62,46 @@ build-assets-image as-if-foss:
- .as-if-foss
- .build-images:rules:build-assets-image-as-if-foss
needs: ["compile-production-assets as-if-foss"]
+
+trigger-omnibus:
+ stage: build-images
+ extends:
+ - .qa:rules:package-and-test-ee
+ needs:
+ - trigger-omnibus-env
+ inherit:
+ variables: false
+ variables:
+ GITALY_SERVER_VERSION: $GITALY_SERVER_VERSION
+ GITLAB_ELASTICSEARCH_INDEXER_VERSION: $GITLAB_ELASTICSEARCH_INDEXER_VERSION
+ GITLAB_KAS_VERSION: $GITLAB_KAS_VERSION
+ GITLAB_METRICS_EXPORTER_VERSION: $GITLAB_METRICS_EXPORTER_VERSION
+ GITLAB_PAGES_VERSION: $GITLAB_PAGES_VERSION
+ GITLAB_SHELL_VERSION: $GITLAB_SHELL_VERSION
+ GITLAB_WORKHORSE_VERSION: $GITLAB_WORKHORSE_VERSION
+ GITLAB_VERSION: $CI_COMMIT_SHA
+ GITLAB_ASSETS_TAG: $GITLAB_ASSETS_TAG
+ IMAGE_TAG: $CI_COMMIT_SHA
+ TOP_UPSTREAM_SOURCE_PROJECT: $CI_PROJECT_PATH
+ SECURITY_SOURCES: $SECURITY_SOURCES
+ CACHE_UPDATE: $OMNIBUS_GITLAB_CACHE_UPDATE
+ RUBY3_BUILD: $OMNIBUS_GITLAB_RUBY3_BUILD
+ RUBY2_BUILD: $OMNIBUS_GITLAB_RUBY2_BUILD
+ CACHE_EDITION: $OMNIBUS_GITLAB_CACHE_EDITION
+ BUILD_ON_ALL_OS: $OMNIBUS_GITLAB_BUILD_ON_ALL_OS
+ SKIP_QA_TEST: "true"
+ ee: $EE
+ trigger:
+ project: gitlab-org/build/omnibus-gitlab-mirror
+ branch: $TRIGGER_BRANCH
+ strategy: depend
+
+trigger-omnibus as-if-foss:
+ extends:
+ - trigger-omnibus
+ - .qa:rules:package-and-test-ce
+ needs:
+ - trigger-omnibus-env as-if-foss
+ variables:
+ # Override gitlab repository so that omnibus doesn't use foss repository for CE build
+ GITLAB_ALTERNATIVE_REPO: $CI_PROJECT_URL
diff --git a/.gitlab/ci/package-and-test-nightly/main.gitlab-ci.yml b/.gitlab/ci/package-and-test-nightly/main.gitlab-ci.yml
index bcd63e4f7ecdc..2f92c757a6cd7 100644
--- a/.gitlab/ci/package-and-test-nightly/main.gitlab-ci.yml
+++ b/.gitlab/ci/package-and-test-nightly/main.gitlab-ci.yml
@@ -18,16 +18,6 @@ workflow:
# ==========================================
# Prepare stage
# ==========================================
-trigger-omnibus-env:
- extends:
- - .trigger-omnibus-env
-
-trigger-omnibus:
- extends:
- - .trigger-omnibus
- needs:
- - trigger-omnibus-env
-
download-knapsack-report:
extends:
- .download-knapsack-report
diff --git a/.gitlab/ci/package-and-test/main.gitlab-ci.yml b/.gitlab/ci/package-and-test/main.gitlab-ci.yml
index 9b127728615d8..098365241138b 100644
--- a/.gitlab/ci/package-and-test/main.gitlab-ci.yml
+++ b/.gitlab/ci/package-and-test/main.gitlab-ci.yml
@@ -20,30 +20,6 @@ check-release-set:
echo "Omnibus installation image is set to '$RELEASE'"
fi
-trigger-omnibus-env:
- extends:
- - .trigger-omnibus-env
- - .rules:omnibus-build
-
-trigger-omnibus-env-ce:
- extends:
- - .trigger-omnibus-env-ce
- - .rules:omnibus-build-ce
-
-trigger-omnibus:
- extends:
- - .trigger-omnibus
- - .rules:omnibus-build
- needs:
- - trigger-omnibus-env
-
-trigger-omnibus-ce:
- extends:
- - .trigger-omnibus-ce
- - .rules:omnibus-build-ce
- needs:
- - trigger-omnibus-env-ce
-
download-knapsack-report:
extends:
- .download-knapsack-report
diff --git a/.gitlab/ci/qa-common/main.gitlab-ci.yml b/.gitlab/ci/qa-common/main.gitlab-ci.yml
index ddcd49dafb0ef..0f48a434a9908 100644
--- a/.gitlab/ci/qa-common/main.gitlab-ci.yml
+++ b/.gitlab/ci/qa-common/main.gitlab-ci.yml
@@ -67,94 +67,6 @@ stages:
# blocking merges due to flaky tests.
allow_failure: true
-.trigger-omnibus-env:
- stage: .pre
- needs:
- # We need this job because we need its `cached-assets-hash.txt` artifact, so that we can pass the assets image tag to the downstream omnibus-gitlab pipeline.
- - pipeline: $PARENT_PIPELINE_ID
- job: build-assets-image
- variables:
- BUILD_ENV: build.env
- before_script:
- - |
- # This is duplicating the function from `scripts/utils.sh` since `.gitlab/ci/package-and-test/main.gitlab-ci.yml` can be included in other projects.
- function assets_image_tag() {
- local cache_assets_hash_file="cached-assets-hash.txt"
-
- if [[ -n "${CI_COMMIT_TAG}" ]]; then
- echo -n "${CI_COMMIT_REF_NAME}"
- elif [[ -f "${cache_assets_hash_file}" ]]; then
- echo -n "assets-hash-$(cat ${cache_assets_hash_file} | cut -c1-10)"
- else
- echo -n "${CI_COMMIT_SHA}"
- fi
- }
- script:
- - |
- SECURITY_SOURCES=$([[ ! "$CI_PROJECT_NAMESPACE" =~ ^gitlab-org\/security ]] || echo "true")
- echo "SECURITY_SOURCES=${SECURITY_SOURCES:-false}" > $BUILD_ENV
- echo "OMNIBUS_GITLAB_CACHE_UPDATE=${OMNIBUS_GITLAB_CACHE_UPDATE:-false}" >> $BUILD_ENV
- for version_file in *_VERSION; do echo "$version_file=$(cat $version_file)" >> $BUILD_ENV; done
- echo "OMNIBUS_GITLAB_RUBY3_BUILD=${OMNIBUS_GITLAB_RUBY3_BUILD:-false}" >> $BUILD_ENV
- echo "OMNIBUS_GITLAB_RUBY2_BUILD=${OMNIBUS_GITLAB_RUBY2_BUILD:-false}" >> $BUILD_ENV
- echo "OMNIBUS_GITLAB_CACHE_EDITION=${OMNIBUS_GITLAB_CACHE_EDITION:-GITLAB}" >> $BUILD_ENV
- echo "OMNIBUS_GITLAB_BUILD_ON_ALL_OS=${OMNIBUS_GITLAB_BUILD_ON_ALL_OS:-false}" >> $BUILD_ENV
- echo "GITLAB_ASSETS_TAG=$(assets_image_tag)" >> $BUILD_ENV
- echo "EE=$([[ $FOSS_ONLY == '1' ]] && echo 'false' || echo 'true')" >> $BUILD_ENV
- target_branch_name="${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:-${CI_COMMIT_REF_NAME}}"
- echo "TRIGGER_BRANCH=$([[ "${target_branch_name}" =~ ^[0-9-]+-stable(-ee)?$ ]] && echo ${target_branch_name%-ee} || echo 'master')" >> $BUILD_ENV
- - |
- echo "Built environment file for omnibus build:"
- cat $BUILD_ENV
- artifacts:
- expire_in: 3 days
- reports:
- dotenv: $BUILD_ENV
- paths:
- - $BUILD_ENV
-
-.trigger-omnibus-env-ce:
- extends: .trigger-omnibus-env
- needs:
- - pipeline: $PARENT_PIPELINE_ID
- job: build-assets-image as-if-foss
-
-.trigger-omnibus:
- stage: .pre
- inherit:
- variables: false
- variables:
- GITALY_SERVER_VERSION: $GITALY_SERVER_VERSION
- GITLAB_ELASTICSEARCH_INDEXER_VERSION: $GITLAB_ELASTICSEARCH_INDEXER_VERSION
- GITLAB_KAS_VERSION: $GITLAB_KAS_VERSION
- GITLAB_METRICS_EXPORTER_VERSION: $GITLAB_METRICS_EXPORTER_VERSION
- GITLAB_PAGES_VERSION: $GITLAB_PAGES_VERSION
- GITLAB_SHELL_VERSION: $GITLAB_SHELL_VERSION
- GITLAB_WORKHORSE_VERSION: $GITLAB_WORKHORSE_VERSION
- GITLAB_VERSION: $CI_COMMIT_SHA
- GITLAB_ASSETS_TAG: $GITLAB_ASSETS_TAG
- IMAGE_TAG: $CI_COMMIT_SHA
- TOP_UPSTREAM_SOURCE_PROJECT: $CI_PROJECT_PATH
- SECURITY_SOURCES: $SECURITY_SOURCES
- CACHE_UPDATE: $OMNIBUS_GITLAB_CACHE_UPDATE
- RUBY3_BUILD: $OMNIBUS_GITLAB_RUBY3_BUILD
- RUBY2_BUILD: $OMNIBUS_GITLAB_RUBY2_BUILD
- CACHE_EDITION: $OMNIBUS_GITLAB_CACHE_EDITION
- BUILD_ON_ALL_OS: $OMNIBUS_GITLAB_BUILD_ON_ALL_OS
- SKIP_QA_TEST: "true"
- ee: $EE
- trigger:
- project: gitlab-org/build/omnibus-gitlab-mirror
- branch: $TRIGGER_BRANCH
- strategy: depend
-
-.trigger-omnibus-ce:
- extends:
- - .trigger-omnibus
- variables:
- # Override gitlab repository so that omnibus doesn't use foss repository for CE build
- GITLAB_ALTERNATIVE_REPO: $CI_PROJECT_URL
-
.download-knapsack-report:
extends:
- .gitlab-qa-image
diff --git a/.gitlab/ci/qa-common/rules.gitlab-ci.yml b/.gitlab/ci/qa-common/rules.gitlab-ci.yml
index 653af4eaa82a0..b5963d24b811d 100644
--- a/.gitlab/ci/qa-common/rules.gitlab-ci.yml
+++ b/.gitlab/ci/qa-common/rules.gitlab-ci.yml
@@ -56,18 +56,6 @@
rules:
- when: always
-.rules:omnibus-build:
- rules:
- - if: $SKIP_OMNIBUS_TRIGGER == "true"
- when: never
- - if: $FOSS_ONLY != "1"
-
-.rules:omnibus-build-ce:
- rules:
- - if: $SKIP_OMNIBUS_TRIGGER == "true"
- when: never
- - if: $FOSS_ONLY == "1"
-
.rules:update-cache:
rules:
- if: '$UPDATE_QA_CACHE == "true"'
diff --git a/.gitlab/ci/qa.gitlab-ci.yml b/.gitlab/ci/qa.gitlab-ci.yml
index d74adde886040..34504f6a21311 100644
--- a/.gitlab/ci/qa.gitlab-ci.yml
+++ b/.gitlab/ci/qa.gitlab-ci.yml
@@ -84,8 +84,8 @@ e2e:package-and-test-ee:
- .e2e-trigger-base
- .qa:rules:package-and-test-ee
needs:
- - build-assets-image
- build-qa-image
+ - trigger-omnibus
- e2e-test-pipeline-generate
variables:
RELEASE: "${REGISTRY_HOST}/${REGISTRY_GROUP}/build/omnibus-gitlab-mirror/gitlab-ee:${CI_COMMIT_SHA}"
@@ -98,8 +98,8 @@ e2e:package-and-test-ce:
- e2e:package-and-test-ee
- .qa:rules:package-and-test-ce
needs:
- - build-assets-image as-if-foss
- build-qa-image as-if-foss
+ - trigger-omnibus as-if-foss
- e2e-test-pipeline-generate
variables:
FOSS_ONLY: "1"
@@ -126,10 +126,8 @@ e2e:package-and-test-nightly:
- .e2e-trigger-base
- .qa:rules:package-and-test-nightly
needs:
- - build-assets-image
- - build-assets-image as-if-foss
- build-qa-image
- - build-qa-image as-if-foss
+ - trigger-omnibus
- e2e-test-pipeline-generate
variables:
GITLAB_SEMVER_VERSION: $GITLAB_SEMVER_VERSION
@@ -145,8 +143,7 @@ e2e:test-on-gdk:
stage: qa
needs:
- build-gdk-image
- - job: e2e-test-pipeline-generate
- artifacts: true
+ - e2e-test-pipeline-generate
variables:
ALLURE_JOB_NAME: e2e-test-on-gdk
QA_RUN_TYPE: e2e-test-on-gdk
diff --git a/.gitlab/ci/setup.gitlab-ci.yml b/.gitlab/ci/setup.gitlab-ci.yml
index bacd7efce3e9d..a9047522413f3 100644
--- a/.gitlab/ci/setup.gitlab-ci.yml
+++ b/.gitlab/ci/setup.gitlab-ci.yml
@@ -172,3 +172,45 @@ e2e-test-pipeline-generate:
paths:
- '*-pipeline.yml'
- "${CI_PROJECT_DIR}/qa_tests_vars.env"
+
+trigger-omnibus-env:
+ stage: prepare
+ extends:
+ - .qa:rules:package-and-test-ee
+ needs:
+ # We need this job because we need its `cached-assets-hash.txt` artifact, so that we can pass the assets image tag to the downstream omnibus-gitlab pipeline.
+ - compile-production-assets
+ variables:
+ BUILD_ENV: build.env
+ before_script:
+ - source scripts/utils.sh
+ script:
+ - |
+ SECURITY_SOURCES=$([[ ! "$CI_PROJECT_NAMESPACE" =~ ^gitlab-org\/security ]] || echo "true")
+ echo "SECURITY_SOURCES=${SECURITY_SOURCES:-false}" > $BUILD_ENV
+ echo "OMNIBUS_GITLAB_CACHE_UPDATE=${OMNIBUS_GITLAB_CACHE_UPDATE:-false}" >> $BUILD_ENV
+ for version_file in *_VERSION; do echo "$version_file=$(cat $version_file)" >> $BUILD_ENV; done
+ echo "OMNIBUS_GITLAB_RUBY3_BUILD=${OMNIBUS_GITLAB_RUBY3_BUILD:-false}" >> $BUILD_ENV
+ echo "OMNIBUS_GITLAB_RUBY2_BUILD=${OMNIBUS_GITLAB_RUBY2_BUILD:-false}" >> $BUILD_ENV
+ echo "OMNIBUS_GITLAB_CACHE_EDITION=${OMNIBUS_GITLAB_CACHE_EDITION:-GITLAB}" >> $BUILD_ENV
+ echo "OMNIBUS_GITLAB_BUILD_ON_ALL_OS=${OMNIBUS_GITLAB_BUILD_ON_ALL_OS:-false}" >> $BUILD_ENV
+ echo "GITLAB_ASSETS_TAG=$(assets_image_tag)" >> $BUILD_ENV
+ echo "EE=$([[ $FOSS_ONLY == '1' ]] && echo 'false' || echo 'true')" >> $BUILD_ENV
+ target_branch_name="${CI_MERGE_REQUEST_TARGET_BRANCH_NAME:-${CI_COMMIT_REF_NAME}}"
+ echo "TRIGGER_BRANCH=$([[ "${target_branch_name}" =~ ^[0-9-]+-stable(-ee)?$ ]] && echo ${target_branch_name%-ee} || echo 'master')" >> $BUILD_ENV
+ - |
+ echo "Built environment file for omnibus build:"
+ cat $BUILD_ENV
+ artifacts:
+ expire_in: 3 days
+ reports:
+ dotenv: $BUILD_ENV
+ paths:
+ - $BUILD_ENV
+
+trigger-omnibus-env as-if-foss:
+ extends:
+ - trigger-omnibus-env
+ - .qa:rules:package-and-test-ce
+ needs:
+ - compile-production-assets as-if-foss
diff --git a/scripts/generate-e2e-pipeline b/scripts/generate-e2e-pipeline
index b0b2b0b0035bb..2b882094dc212 100755
--- a/scripts/generate-e2e-pipeline
+++ b/scripts/generate-e2e-pipeline
@@ -40,7 +40,6 @@ variables:
GIT_SUBMODULE_STRATEGY: "none"
GITLAB_QA_CACHE_KEY: "$qa_cache_key"
GITLAB_SEMVER_VERSION: "$(cat VERSION)"
- SKIP_OMNIBUS_TRIGGER: "false"
QA_EXPORT_TEST_METRICS: "${QA_EXPORT_TEST_METRICS:-true}"
QA_FEATURE_FLAGS: "${QA_FEATURE_FLAGS}"
QA_FRAMEWORK_CHANGES: "${QA_FRAMEWORK_CHANGES:-false}"
--
GitLab