Merge branch 'andrey-pass-ruby-version-qa-trigger' into 'master'

Use correct RUBY_VERSION in all e2e child pipelines and improve pipeline code Closes gitlab-org/quality/analytics/team#39 See merge request https://gitlab.com/gitlab-org/gitlab/-/merge_requests/182280 Merged-by: Sanad Liaquat <sliaquat@gitlab.com> Approved-by: David Dieulivol <ddieulivol@gitlab.com> Approved-by: Sanad Liaquat <sliaquat@gitlab.com> Co-authored-by: Andrejs Cunskis <acunskis@gitlab.com>

Merge branch 'andrey-pass-ruby-version-qa-trigger' into 'master'
34b59324 · Sanad Liaquat · GitLab · 2530e8b3 · 36604b36 · 34b59324
--- a/.gitlab/ci/qa-common/main.gitlab-ci.yml
+++ b/.gitlab/ci/qa-common/main.gitlab-ci.yml
@@ -16,10 +16,11 @@ include:
      gitlab_auth_token_variable_name: "PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE"
      allure_job_name: "${QA_RUN_TYPE}"
  - project: gitlab-org/quality/pipeline-common
-    ref: 9.11.1
+    ref: 9.13.0
    file:
-      - /ci/base.gitlab-ci.yml
+      - /ci/notify-slack.gitlab-ci.yml
-      - /ci/knapsack-report.yml
+      - /ci/qa-report.gitlab-ci.yml
+      - /ci/junit.gitlab-ci.yml
 stages:
  - test
@@ -43,29 +44,16 @@ stages:
  # image path and registry needs to be defined explicitly
  image: "${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/${BUILD_OS}-${OS_VERSION}-ruby-${RUBY_VERSION}"
-.bundler-variables:
+.qa-install:
+  extends:
+    - .ruby-image
  variables:
    BUNDLE_SUPPRESS_INSTALL_USING_MESSAGES: "true"
    BUNDLE_SILENCE_ROOT_WARNING: "true"
    BUNDLE_WITHOUT: development
+  before_script:
-.qa-run-e2e-with-bundler:
+    - echo "SUITE_RAN=true" > $QA_SUITE_STATUS_ENV_FILE
-  script:
+    - cd qa && bundle install
-    - source $CI_PROJECT_DIR/scripts/utils.sh
-    - source $CI_PROJECT_DIR/scripts/rspec_helpers.sh
-    - run_e2e_specs "${QA_GITLAB_URL}" "${QA_TESTS}" "${QA_RSPEC_TAGS}"
-.qa-install:
-  extends:
-    - .bundler-variables
-    - .gitlab-qa-install
-.update-script:
-  script:
-    - !reference [.bundle-prefix]
-    - export QA_COMMAND="$BUNDLE_PREFIX gitlab-qa ${QA_SCENARIO:-Test::Omnibus::UpdateFromPrevious} $RELEASE $GITLAB_SEMVER_VERSION $UPDATE_TYPE $UPDATE_FROM_EDITION -- $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"
-    - echo "Running - '$QA_COMMAND'"
-    - eval "$QA_COMMAND"
 .docker-in-docker:
  services:
@@ -76,28 +64,38 @@ stages:
    DOCKER_TLS_VERIFY: 1
    DOCKER_CERT_PATH: "$DOCKER_TLS_CERTDIR/client"
-.qa:
+.e2e-test-variables:
-  extends:
-    - .bundler-variables
-    - .qa-base
-  stage: test
-  tags:
-    - e2e
  variables:
    QA_GENERATE_ALLURE_REPORT: "true"
    QA_CAN_TEST_PRAEFECT: "false"
    QA_INTERCEPT_REQUESTS: "true"
+    QA_ARTIFACTS_DIR: $CI_PROJECT_DIR
+    QA_SUITE_STATUS_ENV_FILE: $CI_PROJECT_DIR/suite_status.env
+    QA_ALLOW_LOCAL_REQUESTS: "true"
    GITLAB_LICENSE_MODE: test
    GITLAB_QA_ADMIN_ACCESS_TOKEN: $QA_ADMIN_ACCESS_TOKEN
-  # Allow QA jobs to fail as they are flaky. The top level `package-and-e2e:ee`
+    FF_NETWORK_PER_BUILD: "true"
-  # pipeline is not allowed to fail, so without allowing QA to fail, we will be
-  # blocking merges due to flaky tests.
+.e2e-test-base:
-  allow_failure: true
+  extends:
+    - .docker-in-docker
+    - .qa-install
+    - .e2e-test-variables
+  image: "${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/${BUILD_OS}-${OS_VERSION}-ruby-${RUBY_VERSION}:git-${GIT_VERSION}-lfs-${LFS_VERSION}-chrome-${CHROME_VERSION}-docker-${DOCKER_VERSION}-kubectl-${KUBECTL_VERSION}-helm-${HELM_VERSION}-kind-${KIND_VERSION}"
+  variables:
+    # variables related to failure issue reporting
+    # default values from /ci/qa-report.gitlab-ci.yml will work with gitlab-qa orchestrator but not with cng and gdk tests
+    QA_RSPEC_JSON_FILE_PATTERN: $CI_PROJECT_DIR/qa/tmp/rspec-*.json
+    QA_TEST_METRICS_JSON_FILE_PATTERN: $CI_PROJECT_DIR/qa/tmp/test-metrics-*.json
+    QA_SYSTEM_LOG_FILE_PATTERN: $CI_PROJECT_DIR/test_output/logs
+  script:
+    - source $CI_PROJECT_DIR/scripts/utils.sh
+    - source $CI_PROJECT_DIR/scripts/rspec_helpers.sh
+    - run_e2e_specs "${QA_GITLAB_URL}" "${QA_TESTS}" "${QA_RSPEC_TAGS}"
 .upload-test-runtime-data:
  extends:
    - .qa-install
-    - .ruby-image
  stage: report
  when: on_success
  variables:
@@ -108,33 +106,32 @@ stages:
 .export-test-metrics:
  extends:
    - .qa-install
-    - .ruby-image
  stage: report
  when: always
  variables:
-    QA_METRICS_REPORT_FILE_PATTERN: $CI_PROJECT_DIR/gitlab-qa-run-*/**/test-metrics-*.json
+    QA_METRICS_REPORT_FILE_PATTERN: $CI_PROJECT_DIR/qa/tmp/test-metrics-*.json
  script:
    - bundle exec rake "ci:export_test_metrics[$QA_METRICS_REPORT_FILE_PATTERN]"
 .export-code-paths-mapping:
  extends:
    - .qa-install
-    - .ruby-image
  stage: report
  when: always
  variables:
-    QA_CODE_PATHS_MAPPING_FILE_PATTERN: $CI_PROJECT_DIR/gitlab-qa-run-*/**/test-code-paths-mapping-*.json
+    QA_CODE_PATHS_MAPPING_FILE_PATTERN: $CI_PROJECT_DIR/qa/tmp/test-code-paths-mapping-*.json
  script:
    - bundle exec rake "ci:export_code_paths_mapping[$QA_CODE_PATHS_MAPPING_FILE_PATTERN]"
+  rules:
+    - if: '$COVERBAND_ENABLED == "true"'
 .notify-slack:
  extends:
-    - .notify-slack-qa
+    - .post-to-slack
    - .qa-install
-    - .ruby-image
  stage: notify
  variables:
-    QA_RSPEC_XML_FILE_PATTERN: "${CI_PROJECT_DIR}/gitlab-qa-run-*/**/rspec-*.xml"
+    QA_RSPEC_XML_FILE_PATTERN: "$CI_PROJECT_DIR/qa/tmp/rspec-*.xml"
    GITLAB_API_TOKEN: $PROJECT_TOKEN_FOR_CI_SCRIPTS_API_USAGE
    SLACK_ICON_EMOJI: ci_failing
    STATUS_SYM: ☠️
@@ -149,7 +146,7 @@ stages:
        exit 0
      fi
    - bundle exec prepare-stage-reports --input-files "${QA_RSPEC_XML_FILE_PATTERN}"
-    - !reference [.notify-slack-qa, script]
+    - !reference [.post-to-slack, script]
 .failure-videos:
  variables:
@@ -160,17 +157,6 @@ stages:
 # ==========================================
 # Pre stage
 # ==========================================
-.check-release-set:
-  stage: .pre
-  script:
-    - |
-      if [ -z "$RELEASE" ]; then
-        echo "E2E test pipeline requires omnibus installation docker image to be set via $RELEASE environment variable"
-        exit 1
-      else
-        echo "Omnibus installation image is set to '$RELEASE'"
-      fi
 dont-interrupt-me:
  stage: .pre
  interruptible: false

--- a/.gitlab/ci/qa-common/omnibus.gitlab-ci.yml
+++ b/.gitlab/ci/qa-common/omnibus.gitlab-ci.yml
+include:
+  - local: .gitlab/ci/global.gitlab-ci.yml
+  - local: .gitlab/ci/qa-common/main.gitlab-ci.yml
+.check-release-set:
+  stage: .pre
+  variables:
+    GIT_STRATEGY: none
+  script:
+    - |
+      if [ -z "$RELEASE" ]; then
+        echo "E2E test pipeline requires omnibus installation docker image to be set via $RELEASE environment variable"
+        exit 1
+      else
+        echo "Omnibus installation image is set to '$RELEASE'"
+      fi
+.update-script:
+  script:
+    - !reference [.bundle-prefix]
+    - export QA_COMMAND="bundle exec gitlab-qa ${QA_SCENARIO:-Test::Omnibus::UpdateFromPrevious} $RELEASE $GITLAB_SEMVER_VERSION $UPDATE_TYPE $UPDATE_FROM_EDITION -- $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"
+    - echo "Running - '$QA_COMMAND'"
+    - eval "$QA_COMMAND"
 trigger-omnibus-env:
  stage: .pre
  extends:

--- a/.gitlab/ci/qa-common/variables.gitlab-ci.yml
+++ b/.gitlab/ci/qa-common/variables.gitlab-ci.yml
@@ -10,9 +10,6 @@ variables:
  ALLURE_JOB_NAME: $CI_PROJECT_NAME
  COLORIZED_LOGS: "true"
  QA_LOG_LEVEL: "info"
-  # run all tests by default when test-on-omnibus is included natively in other projects
-  # this will be overridden when selective test execution is used in gitlab canonical project
-  QA_RUN_ALL_TESTS: "true"
  # Used by gitlab-qa to set up a volume for `${CI_PROJECT_DIR}/qa/rspec:/home/gitlab/qa/rspec/`
  QA_RSPEC_REPORT_PATH: "${CI_PROJECT_DIR}/qa/rspec"
  QA_OMNIBUS_MR_TESTS: "only-smoke"
@@ -20,3 +17,5 @@ variables:
  QA_RETRY_FAILED_SPECS: "true"
  # helm chart ref used by test-on-cng pipeline
  GITLAB_HELM_CHART_REF: "3f89420cf319778195403711af12c57b4aab6511"
+  # makes sure some of the common scripts from pipeline-common use bundler to execute commands
+  RUN_WITH_BUNDLE: "true"
--- a/.gitlab/ci/qa.gitlab-ci.yml
+++ b/.gitlab/ci/qa.gitlab-ci.yml
@@ -33,33 +33,31 @@
  extends: .production  # this makes sure GITLAB_ALLOW_SEPARATE_CI_DATABASE is passed to the child pipeline
  stage: qa
  variables:
-    # This is needed by `trigger-omnibus-env` (`.gitlab/ci/test-on-omnibus/main.gitlab-ci.yml`).
+    REGISTRY_GROUP: $REGISTRY_GROUP
-    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
+    REGISTRY_HOST: $REGISTRY_HOST
+    FOSS_ONLY: $FOSS_ONLY
    PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE
-    GITLAB_QA_IMAGE: "${CI_REGISTRY_IMAGE}/gitlab-ee-qa:${CI_COMMIT_SHA}"  # This is used for the CI job image from https://gitlab.com/gitlab-org/quality/pipeline-common/-/blob/8.4.4/ci/base.gitlab-ci.yml
-    QA_IMAGE: "${CI_REGISTRY_IMAGE}/gitlab-ee-qa:${CI_COMMIT_SHA}"  # This is used for the gitlab-qa image pulled from the E2E tests. If this is not defined, it'll be inferred from RELEASE_IMAGE_TAG below which is wrong because it should not contain the RUBY_VERSION suffix
    RELEASE_IMAGE_TAG: "${CI_COMMIT_SHA}-ruby${RUBY_VERSION}"  # This contains a RUBY_VERSION suffix because Omnibus build is depending on the RUBY_VERSION
-    RUN_WITH_BUNDLE: "true"  # instructs pipeline to install and run gitlab-qa gem via bundler
-    QA_PATH: qa  # sets the optional path for bundler to run from
    DYNAMIC_PIPELINE_YML: "${CI_PROJECT_DIR}/qa/tmp/test-on-omnibus-pipeline.yml"  # yml files are generated by generate_e2e_pipelines task in qa/tasks/ci.rake
+    PARENT_PIPELINE_ID: $CI_PIPELINE_ID # This is needed by trigger-omnibus-env and build-cng-env jobs
    QA_EXPORT_TEST_METRICS: "true"  # Export test metrics directly to influxdb by default
  inherit:
-    variables:
+    variables: false
-      - REGISTRY_GROUP
-      - REGISTRY_HOST
-      - OMNIBUS_GITLAB_CACHE_UPDATE
-      - OMNIBUS_GITLAB_CACHE_EDITION
-      - OMNIBUS_GITLAB_BUILD_ON_ALL_OS
-      - FOSS_ONLY
  trigger:
    strategy: depend
-    forward:
-      yaml_variables: true
-      pipeline_variables: true
    include:
      - artifact: $DYNAMIC_PIPELINE_YML
        job: e2e-test-pipeline-generate
+.omnibus-trigger-base:
+  variables:
+    QA_IMAGE: "${CI_REGISTRY_IMAGE}/gitlab-ee-qa:${CI_COMMIT_SHA}"
+  inherit:
+    variables:
+      - OMNIBUS_GITLAB_CACHE_UPDATE
+      - OMNIBUS_GITLAB_CACHE_EDITION
+      - OMNIBUS_GITLAB_BUILD_ON_ALL_OS
 # ==========================================
 # /qa caching
 # ==========================================
@@ -144,6 +142,7 @@ qa:rspec-internal:
 e2e:test-on-omnibus-ee:
  extends:
    - .e2e-trigger-base
+    - .omnibus-trigger-base
    - .qa:rules:test-on-omnibus-ee
  needs:
    - build-assets-image
@@ -174,6 +173,7 @@ e2e:test-on-omnibus-ce:
 e2e:test-on-omnibus-ee-nightly:
  extends:
    - .e2e-trigger-base
+    - .omnibus-trigger-base
    - .qa:rules:test-on-omnibus-nightly
  needs:
    - build-assets-image
@@ -181,7 +181,6 @@ e2e:test-on-omnibus-ee-nightly:
    - e2e-test-pipeline-generate
  variables:
    RELEASE: "${REGISTRY_HOST}/${REGISTRY_GROUP}/build/omnibus-gitlab-mirror/gitlab-ee:${RELEASE_IMAGE_TAG}"
-    GITLAB_SEMVER_VERSION: $GITLAB_SEMVER_VERSION
    QA_RUN_TYPE: nightly
    PIPELINE_NAME: E2E Omnibus GitLab Nightly
    DYNAMIC_PIPELINE_YML: ${CI_PROJECT_DIR}/qa/tmp/test-on-omnibus-nightly-pipeline.yml

--- a/.gitlab/ci/test-on-cng/main.gitlab-ci.yml
+++ b/.gitlab/ci/test-on-cng/main.gitlab-ci.yml
 # This pipeline runs E2E tests against CNG build which is deployed via helm chart on a job local kind kubernetes cluster
 include:
+  - local: .gitlab/ci/global.gitlab-ci.yml
  - local: .gitlab/ci/cng/main.gitlab-ci.yml
  - local: .gitlab/ci/qa-common/main.gitlab-ci.yml
  - local: .gitlab/ci/qa-common/variables.gitlab-ci.yml
@@ -18,7 +19,6 @@ workflow:
 .cng-deploy-cmd:
  script:
-    - cd qa && bundle install
    # save extra values to be available for after_script if created dynamically
    - echo "${EXTRA_DEPLOY_VALUES}" > $CI_PROJECT_DIR/EXTRA_DEPLOY_VALUES
    - export GITLAB_DOMAIN="$(getent hosts docker | awk '{ print $1 }' | head -n1).nip.io"
@@ -34,18 +34,15 @@ workflow:
    - export QA_GITLAB_URL="http://gitlab.${GITLAB_DOMAIN}"
 .cng-test:
-  image: "${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/${BUILD_OS}-${OS_VERSION}-ruby-${RUBY_VERSION}:git-${GIT_VERSION}-lfs-${LFS_VERSION}-chrome-${CHROME_VERSION}-docker-${DOCKER_VERSION}-kubectl-${KUBECTL_VERSION}-helm-${HELM_VERSION}-kind-${KIND_VERSION}"
  stage: test
  extends:
-    - .cng-qa-cache
+    - .e2e-test-base
-    - .docker-in-docker
+    - .cng-qa-cache # cng-cache includes additional cached helm chart
-    - .qa-run-e2e-with-bundler
  needs:
    - build-cng
  tags:
    - e2e
  variables:
-    FF_NETWORK_PER_BUILD: "true"
    NAMESPACE: gitlab
    KUBECONFIG: /root/.kube/config
    BUNDLE_WITHOUT: development
@@ -53,22 +50,14 @@ workflow:
    CNG_FORCE_COLOR: "true"
    DEPLOYMENT_TYPE: kind
    QA_DOCKER_NETWORK: host
-    QA_GENERATE_ALLURE_REPORT: "true"
-    QA_CAN_TEST_PRAEFECT: "false"
-    QA_ALLOW_LOCAL_REQUESTS: "true"
-    QA_SUITE_STATUS_ENV_FILE: $CI_PROJECT_DIR/suite_status.env
    QA_RUN_IN_PARALLEL: "true"
    QA_PARALLEL_PROCESSES: 4
    GITLAB_QA_ADMIN_ACCESS_TOKEN: $QA_ADMIN_ACCESS_TOKEN
  before_script:
-    - echo "SUITE_RAN=true" > "$QA_SUITE_STATUS_ENV_FILE"
+    - !reference [.qa-install, before_script]
    - !reference [.cng-deploy-cmd, script]
  after_script:
-    - |
+    - !reference [.gitlab-qa-report, after_script]
-      if [ "$CI_JOB_STATUS" == "failed" ]; then
-        echo "SUITE_FAILED=true" >> "$QA_SUITE_STATUS_ENV_FILE"
-      fi
-    - cd qa
    - bundle exec orchestrator log events --save
    - bundle exec orchestrator log pods --save --containers all --no-fail-on-missing-pods
    # This command prints all the necessary arguments to be able to recreate the same deployment as on CI
@@ -119,7 +108,8 @@ cng-helm-cache:
    - .cng-test
    - .cng-deploy-cmd
    - .cng-qa-cache-push
-  before_script: []
+  before_script:
+    - !reference [.qa-install, before_script]
  after_script: []
  rules:
    - <<: *default-branch
@@ -167,23 +157,15 @@ export-test-metrics:
  extends:
    - .export-test-metrics
    - .rules:report:process-results
-  variables:
-    QA_METRICS_REPORT_FILE_PATTERN: $CI_PROJECT_DIR/qa/tmp/test-metrics-*.json
 export-code-paths-mapping:
  extends:
    - .export-code-paths-mapping
-  variables:
-    QA_CODE_PATHS_MAPPING_FILE_PATTERN: $CI_PROJECT_DIR/qa/tmp/test-code-paths-mapping-*.json
-  rules:
-    - if: '$COVERBAND_ENABLED == "true"'
 notify-slack:
  extends:
    - .notify-slack
    - .rules:report:process-results
-  variables:
-    QA_RSPEC_XML_FILE_PATTERN: $CI_PROJECT_DIR/qa/tmp/rspec-*.xml
 upload-test-runtime-data:
  extends:

--- a/.gitlab/ci/test-on-gdk/main.gitlab-ci.yml
+++ b/.gitlab/ci/test-on-gdk/main.gitlab-ci.yml
@@ -29,12 +29,9 @@ workflow:
    - mv $CI_BUILDS_DIR/*.log $CI_PROJECT_DIR/
 .gdk-qa-base:
-  image: "${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/${BUILD_OS}-${OS_VERSION}-ruby-${RUBY_VERSION}:git-${GIT_VERSION}-lfs-${LFS_VERSION}-chrome-${CHROME_VERSION}-docker-${DOCKER_VERSION}-gcloud-${GCLOUD_VERSION}-kubectl-${KUBECTL_VERSION}-helm-${HELM_VERSION}"
  extends:
+    - .e2e-test-base
    - .qa-cache
-    - .docker-in-docker
-    - .gitlab-qa-report
-    - .qa-run-e2e-with-bundler
    # ignore runtime data from gdk because it's significantly slower than cng and runtime data for
    # optimizing parallel distribution should be geared towards test-on-cng
    - .with-ignored-runtime-data
@@ -49,24 +46,13 @@ workflow:
  tags:
    - e2e
  variables:
-    QA_GENERATE_ALLURE_REPORT: "true"
-    QA_CAN_TEST_PRAEFECT: "false"
-    QA_INTERCEPT_REQUESTS: "false"
-    QA_SUITE_STATUS_ENV_FILE: "$CI_PROJECT_DIR/suite_status.env"
    QA_DOCKER_NETWORK: host
-    QA_DISABLE_RSPEC_RETRY: "true"
    QA_GITLAB_URL: http://gdk.test:3000
    GITLAB_DEVELOPMENT_USE_PRECOMPILED_ASSETS: "true"
    GITLAB_QA_ADMIN_ACCESS_TOKEN: $QA_ADMIN_ACCESS_TOKEN
-    FF_NETWORK_PER_BUILD: "true"
    RSPEC_LAST_RUN_RESULTS_FILE: "$CI_PROJECT_DIR/qa/tmp/examples.txt"
    COVERBAND_ENABLED: "$COVERBAND_ENABLED"  # explicitly define variable so it is passed in to gdk service container
    GITLAB_CRON_JOBS_POLL_INTERVAL: "0"
-  before_script:
-    - echo "SUITE_RAN=true" > "$QA_SUITE_STATUS_ENV_FILE"
-    - echo -e "\e[0Ksection_start:`date +%s`:install_gems[collapsed=true]\r\e[0KInstall gems"
-    - cd qa && bundle install
-    - echo -e "\e[0Ksection_end:`date +%s`:install_gems\r\e[0K"
  after_script:
    - !reference [.with-gdk-log, after_script]
    - !reference [.gitlab-qa-report, after_script]
@@ -134,25 +120,12 @@ export-test-metrics:
  extends:
    - .export-test-metrics
    - .rules:report:process-results
-  variables:
-    QA_METRICS_REPORT_FILE_PATTERN: $CI_PROJECT_DIR/qa/tmp/test-metrics-*.json
 export-code-paths-mapping:
  extends:
    - .export-code-paths-mapping
-  variables:
-    QA_CODE_PATHS_MAPPING_FILE_PATTERN: $CI_PROJECT_DIR/qa/tmp/test-code-paths-mapping-*.json
-  rules:
-    - if: '$COVERBAND_ENABLED == "true"'
-.gitlab-qa-report:
-  variables:
-    QA_RSPEC_JSON_FILE_PATTERN: $CI_PROJECT_DIR/qa/tmp/rspec-*.json
-    QA_SYSTEM_LOG_FILE_PATTERN: $CI_PROJECT_DIR/test_output/logs
 notify-slack:
  extends:
    - .notify-slack
    - .rules:report:process-results
-  variables:
-    QA_RSPEC_XML_FILE_PATTERN: $CI_PROJECT_DIR/qa/tmp/rspec-*.xml
--- a/.gitlab/ci/test-on-omnibus-nightly/main.gitlab-ci.yml
+++ b/.gitlab/ci/test-on-omnibus-nightly/main.gitlab-ci.yml
 include:
-  - local: .gitlab/ci/qa-common/main.gitlab-ci.yml
  - local: .gitlab/ci/qa-common/rules.gitlab-ci.yml
  - local: .gitlab/ci/qa-common/variables.gitlab-ci.yml
  - local: .gitlab/ci/qa-common/omnibus.gitlab-ci.yml
@@ -25,7 +24,7 @@ check-release-set:
 # ------------------------------------------
 airgapped:
  extends:
-    - .qa
+    - .omnibus-e2e
    - .parallel
    - .with-ignored-runtime-data
  variables:
@@ -34,7 +33,7 @@ airgapped:
 instance-image-slow-network:
  extends:
-    - .qa
+    - .omnibus-e2e
    - .parallel
    - .with-ignored-runtime-data
  variables:
@@ -45,7 +44,7 @@ instance-image-slow-network:
 # ------------------------------------------
 nplus1-instance-image:
  extends:
-    - .qa
+    - .omnibus-e2e
    - .parallel
    - .with-ignored-runtime-data
  variables:
@@ -57,7 +56,7 @@ nplus1-instance-image:
 # ------------------------------------------
 integration-elasticsearch-compatibility-version-7:
  extends:
-    - .qa
+    - .omnibus-e2e
    - .with-ignored-runtime-data
  variables:
    ELASTIC_VERSION: "7.17.0"
@@ -66,7 +65,7 @@ integration-elasticsearch-compatibility-version-7:
 integration-elasticsearch-compatibility-version-8:
  extends:
-    - .qa
+    - .omnibus-e2e
    - .with-ignored-runtime-data
  variables:
    ELASTIC_VERSION: "8.11.4"
@@ -78,7 +77,7 @@ integration-elasticsearch-compatibility-version-8:
 # ------------------------------------------
 integration-opensearch-compatibility-version-1:
  extends:
-    - .qa
+    - .omnibus-e2e
  variables:
    OPENSEARCH_VERSION: "1.3.3"
    QA_SCENARIO: "Test::Integration::Opensearch"
@@ -86,7 +85,7 @@ integration-opensearch-compatibility-version-1:
 integration-opensearch-compatibility-version-2:
  extends:
-    - .qa
+    - .omnibus-e2e
  variables:
    OPENSEARCH_VERSION: "2.0.1"
    QA_SCENARIO: "Test::Integration::Opensearch"
@@ -97,7 +96,7 @@ integration-opensearch-compatibility-version-2:
 # ------------------------------------------
 relative-url:
  extends:
-    - .qa
+    - .omnibus-e2e
    - .parallel
    - .with-ignored-runtime-data
  variables:
@@ -109,7 +108,7 @@ relative-url:
 # ------------------------------------------
 registry-with-cdn:
  extends:
-    - .qa
+    - .omnibus-e2e
  variables:
    QA_SCENARIO: Test::Integration::RegistryWithCDN
    GCS_CDN_BUCKET_NAME: $QA_GCS_CDN_BUCKET_NAME
@@ -123,7 +122,7 @@ registry-with-cdn:
 # ------------------------------------------
 object-storage:
  extends:
-    - .qa
+    - .omnibus-e2e
  variables:
    QA_SCENARIO: Test::Instance::Image
    QA_RSPEC_TAGS: --tag object_storage
@@ -160,7 +159,7 @@ object-storage-gcs:
 # ------------------------------------------
 decomposition-single-db:
  extends:
-    - .qa
+    - .omnibus-e2e
    - .parallel
    - .with-ignored-runtime-data
  variables:
@@ -169,7 +168,7 @@ decomposition-single-db:
 decomposition-multiple-db:
  extends:
-    - .qa
+    - .omnibus-e2e
    - .parallel
    - .with-ignored-runtime-data
  variables:
@@ -179,14 +178,14 @@ decomposition-multiple-db:
 gitlab-pages:
  extends:
-    - .qa
+    - .omnibus-e2e
  variables:
    QA_SCENARIO: Test::Integration::GitlabPages
 # ========== Update From Previous with AI components ===========
 update-minor-ee-ai-components:
  extends:
-    - .qa
+    - .omnibus-e2e
    - .update-script
    - .with-ignored-runtime-data
  variables:
@@ -215,7 +214,11 @@ upload-test-runtime-data:
 export-test-metrics:
  extends:
    - .export-test-metrics
+  variables:
+    QA_METRICS_REPORT_FILE_PATTERN: $CI_PROJECT_DIR/gitlab-qa-run-*/**/test-metrics-*.json
 notify-slack:
  extends:
    - .notify-slack
+  variables:
+    QA_RSPEC_XML_FILE_PATTERN: "${CI_PROJECT_DIR}/gitlab-qa-run-*/**/rspec-*.xml"
--- a/.gitlab/ci/test-on-omnibus/external.gitlab-ci.yml
+++ b/.gitlab/ci/test-on-omnibus/external.gitlab-ci.yml
+# This file is loaded when omnibus tests are triggered from a different repository
+# It overrides base job definition using pipeline-common definition which does not require gitlab repo code to be fetched
+include:
+  - project: gitlab-org/quality/pipeline-common
+    ref: 9.13.0
+    file:
+      - /ci/base.gitlab-ci.yml
+variables:
+  RUN_WITH_BUNDLE: "false"
+# Populate runner cache with preinstalled version of gitlab-qa gem
+cache-gitlab-qa:
+  extends:
+    - .gitlab-qa-install
+  stage: .pre
+  tags:
+    - e2e
+  variables:
+    GIT_STRATEGY: none
+  script: echo "Successfully installed gitlab-qa gem"
+  cache:
+    policy: pull-push
+  rules:
+    - when: always
+      allow_failure: true
+.omnibus-e2e:
+  extends:
+    - .qa-base
+    - .e2e-test-variables
+  stage: test
+  tags:
+    - e2e
+  variables:
+    GIT_STRATEGY: none
+  allow_failure: true
--- a/.gitlab/ci/test-on-omnibus/internal.gitlab-ci.yml
+++ b/.gitlab/ci/test-on-omnibus/internal.gitlab-ci.yml
+# E2E test job base used in pipelines triggered within gitlab projects which have access to `gitlab` repo code
+.omnibus-e2e:
+  image: ${CI_REGISTRY}/gitlab-org/gitlab-build-images/${BUILD_OS}-${OS_VERSION}-ruby-${RUBY_VERSION}:docker-${DOCKER_VERSION}
+  extends:
+    - .qa-cache
+    - .qa-install
+    - .docker-in-docker
+    - .e2e-test-variables
+  stage: test
+  tags:
+    - e2e
+  retry:
+    max: 2
+    when: runner_system_failure
+  script:
+    - export QA_COMMAND="bundle exec gitlab-qa ${QA_SCENARIO:=Test::Instance::Image} $RELEASE $GITLAB_QA_OPTS -- $QA_TESTS $QA_RSPEC_TAGS $RSPEC_REPORT_OPTS"
+    - echo "Running - '$QA_COMMAND'"
+    - eval "$QA_COMMAND"
+  after_script:
+    - !reference [.merge-junit, after_script]
+    - !reference [.gitlab-qa-report, after_script]
+  allow_failure: true
+  artifacts:
+    when: always
+    expire_in: 30d
+    paths:
+      - gitlab-qa-run-*
+    reports:
+      junit: gitlab-qa-run-*/**/rspec-*.xml
+      dotenv: $QA_SUITE_STATUS_ENV_FILE
--- a/.gitlab/ci/test-on-omnibus/main.gitlab-ci.yml
+++ b/.gitlab/ci/test-on-omnibus/main.gitlab-ci.yml
-# E2E tests pipeline loaded dynamically by script: scripts/generate-e2e-pipeline
-# For adding new tests, refer to: doc/development/testing_guide/end_to_end/test_pipelines.md
 include:
-  - local: .gitlab/ci/qa-common/main.gitlab-ci.yml
  - local: .gitlab/ci/qa-common/rules.gitlab-ci.yml
  - local: .gitlab/ci/qa-common/variables.gitlab-ci.yml
  - local: .gitlab/ci/qa-common/omnibus.gitlab-ci.yml
+  # Variable $GITLAB_PROJECT_PIPELINE is set to a literal value by dynamically generated pipeline due to an issue
+  # where include is not supporting variables in dynamically generated pipelines: https://gitlab.com/gitlab-org/gitlab/-/issues/378717
+  - local: .gitlab/ci/test-on-omnibus/internal.gitlab-ci.yml
+    rules:
+      - if: '$GITLAB_PROJECT_PIPELINE'
+  # External e2e test job definitions are required for project including omnibus tests directly without access to gitlab repo code
+  - local: .gitlab/ci/test-on-omnibus/external.gitlab-ci.yml
+    rules:
+      - if: '$GITLAB_PROJECT_PIPELINE'
+        when: never
+      - when: always
 variables:
  QA_PARALLEL_PROCESSES: 6
@@ -30,7 +38,7 @@ check-release-set:
 #   and optionally QA_TESTS to run specific quarantined tests
 _quarantine:
  extends:
-    - .qa
+    - .omnibus-e2e
  stage: test
  variables:
    QA_RSPEC_TAGS: --tag quarantine
@@ -52,7 +60,7 @@ _quarantine:
 # Execute smallest test suite to validate omnibus package in dependency update merge request pipelines
 health-check:
  extends:
-    - .qa
+    - .omnibus-e2e
  variables:
    QA_SCENARIO: Test::Instance::Image
    QA_RUN_IN_PARALLEL: true
@@ -64,7 +72,7 @@ health-check:
 instance:
  extends:
    - .parallel
-    - .qa
+    - .omnibus-e2e
  variables:
    QA_SCENARIO: Test::Instance::Image
    QA_RUN_IN_PARALLEL: true
@@ -77,7 +85,7 @@ instance:
 praefect:
  extends:
    - .parallel
-    - .qa
+    - .omnibus-e2e
  variables:
    QA_SCENARIO: Test::Integration::Praefect
    QA_SUITE_CLASS_NAME: Test::Instance::Create # overrides default 'All' suite to only run 'Create' group specs
@@ -93,7 +101,7 @@ praefect:
 gitaly-transactions:
  extends:
    - .parallel
-    - .qa
+    - .omnibus-e2e
    - .with-ignored-runtime-data
  variables:
    QA_SCENARIO: Test::Integration::Praefect
@@ -112,7 +120,7 @@ gitaly-transactions:
 gitaly-reftables-backend:
  extends:
    - .parallel
-    - .qa
+    - .omnibus-e2e
    - .with-ignored-runtime-data
  variables:
    QA_SCENARIO: Test::Integration::Praefect
@@ -128,7 +136,7 @@ git-sha256-repositories:
  when: manual
  extends:
    - .parallel
-    - .qa
+    - .omnibus-e2e
    - .with-ignored-runtime-data
  variables:
    QA_SCENARIO: Test::Instance::Image
@@ -139,7 +147,7 @@ git-sha256-repositories:
    - if: $QA_SUITES =~ /Test::Instance::All/
 gitaly-cluster:
-  extends: .qa
+  extends: .omnibus-e2e
  variables:
    QA_SCENARIO: Test::Integration::GitalyCluster
  rules:
@@ -147,7 +155,7 @@ gitaly-cluster:
    - if: $QA_SUITES =~ /Test::Integration::GitalyCluster/
 group-saml:
-  extends: .qa
+  extends: .omnibus-e2e
  variables:
    QA_SCENARIO: Test::Integration::GroupSAML
  rules:
@@ -155,7 +163,7 @@ group-saml:
 oauth:
  extends:
-    - .qa
+    - .omnibus-e2e
  variables:
    QA_SCENARIO: Test::Instance::Image
    QA_GITLAB_HOSTNAME: "gitlab"
@@ -167,7 +175,7 @@ oauth:
    - if: $QA_SUITES =~ /Test::Integration::OAuth/
 instance-saml:
-  extends: .qa
+  extends: .omnibus-e2e
  variables:
    QA_SCENARIO: Test::Integration::InstanceSAML
  rules:
@@ -176,7 +184,7 @@ instance-saml:
 jira:
  extends:
-    - .qa
+    - .omnibus-e2e
  variables:
    QA_SCENARIO: Test::Integration::Jira
  rules:
@@ -184,7 +192,7 @@ jira:
    - if: $QA_SUITES =~ /Test::Integration::Jira/
 integrations:
-  extends: .qa
+  extends: .omnibus-e2e
  variables:
    QA_SCENARIO: Test::Integration::Integrations
  rules:
@@ -192,7 +200,7 @@ integrations:
    - if: $QA_SUITES =~ /Test::Integration::Integrations/
 ldap-no-server:
-  extends: .qa
+  extends: .omnibus-e2e
  variables:
    QA_SCENARIO: Test::Integration::LDAPNoServer
  rules:
@@ -200,7 +208,7 @@ ldap-no-server:
    - if: $QA_SUITES =~ /Test::Integration::LDAPNoServer/
 ldap-tls:
-  extends: .qa
+  extends: .omnibus-e2e
  variables:
    QA_SCENARIO: Test::Integration::LDAPTLS
  rules:
@@ -208,7 +216,7 @@ ldap-tls:
    - if: $QA_SUITES =~ /Test::Integration::LDAPTLS/
 ldap-no-tls:
-  extends: .qa
+  extends: .omnibus-e2e
  variables:
    QA_SCENARIO: Test::Integration::LDAPNoTLS
  rules:
@@ -216,7 +224,7 @@ ldap-no-tls:
    - if: $QA_SUITES =~ /Test::Integration::LDAPNoTLS/
 mtls:
-  extends: .qa
+  extends: .omnibus-e2e
  variables:
    QA_SCENARIO: Test::Integration::MTLS
  rules:
@@ -225,7 +233,7 @@ mtls:
 mattermost:
  extends:
-    - .qa
+    - .omnibus-e2e
  variables:
    QA_SCENARIO: Test::Integration::Mattermost
  rules:
@@ -234,7 +242,7 @@ mattermost:
 registry:
  extends:
-    - .qa
+    - .omnibus-e2e
  variables:
    QA_SCENARIO: Test::Integration::Registry
  rules:
@@ -243,7 +251,7 @@ registry:
 repository-storage:
  extends:
-    - .qa
+    - .omnibus-e2e
  variables:
    QA_SCENARIO: Test::Instance::RepositoryStorage
  rules:
@@ -252,7 +260,7 @@ repository-storage:
 service-ping-disabled:
  extends:
-    - .qa
+    - .omnibus-e2e
    - .failure-videos
  variables:
    QA_SCENARIO: Test::Integration::ServicePingDisabled
@@ -262,7 +270,7 @@ service-ping-disabled:
 smtp:
  extends:
-    - .qa
+    - .omnibus-e2e
  variables:
    QA_SCENARIO: Test::Integration::SMTP
  rules:
@@ -271,7 +279,7 @@ smtp:
 cloud-activation:
  extends:
-    - .qa
+    - .omnibus-e2e
  variables:
    QA_SCENARIO: Test::Instance::Image
    QA_RSPEC_TAGS: --tag cloud_activation
@@ -280,7 +288,7 @@ cloud-activation:
    - if: $QA_SUITES =~ /Test::Instance::CloudActivation/
 metrics:
-  extends: .qa
+  extends: .omnibus-e2e
  variables:
    QA_SCENARIO: Test::Integration::Metrics
  rules:
@@ -288,12 +296,12 @@ metrics:
    - if: $QA_SUITES =~ /Test::Integration::Metrics/
 elasticsearch:
-  extends: .qa
+  extends: .omnibus-e2e
  variables:
    QA_SCENARIO: "Test::Integration::Elasticsearch"
    ELASTIC_URL: "http://elastic68:9200"
  before_script:
-    - !reference [.qa, before_script]
+    - !reference [.omnibus-e2e, before_script]
  rules:
    - !reference [.rules:test:dependency-update-never, rules]
    - !reference [.rules:test:ee-only, rules]
@@ -301,7 +309,7 @@ elasticsearch:
 importers:
  extends:
-    - .qa
+    - .omnibus-e2e
  variables:
    QA_SCENARIO: Test::Integration::Import
    QA_MOCK_GITHUB: "true"
@@ -311,7 +319,7 @@ importers:
 import-with-smtp:
  extends:
-    - .qa
+    - .omnibus-e2e
    - .failure-videos
  variables:
    QA_SCENARIO: Test::Integration::ImportWithSMTP
@@ -322,7 +330,7 @@ import-with-smtp:
 # ========== ai-gateway ===========
 ai-gateway:
  extends:
-    - .qa
+    - .omnibus-e2e
  variables:
    QA_SCENARIO: Test::Integration::AiGateway
  rules:
@@ -331,7 +339,7 @@ ai-gateway:
 ai-gateway-no-seat-assigned:
  extends:
-    - .qa
+    - .omnibus-e2e
  variables:
    QA_SCENARIO: Test::Integration::AiGatewayNoSeatAssigned
  rules:
@@ -340,24 +348,24 @@ ai-gateway-no-seat-assigned:
 ai-gateway-no-add-on:
  extends:
-    - .qa
+    - .omnibus-e2e
  variables:
    QA_SCENARIO: Test::Integration::AiGatewayNoAddOn
  before_script:
    - export QA_EE_ACTIVATION_CODE=$QA_EE_ACTIVATION_CODE_NO_ADD_ON
-    - !reference [.qa, before_script]
+    - !reference [.omnibus-e2e, before_script]
  rules:
    - !reference [.rules:test:dependency-update-never, rules]
    - if: $QA_SUITES =~ /Test::Integration::AiGatewayNoAddOn/
 ai-gateway-no-license:
  extends:
-    - .qa
+    - .omnibus-e2e
  variables:
    QA_SCENARIO: Test::Integration::AiGatewayNoLicense
  before_script:
    - unset QA_EE_LICENSE
-    - !reference [.qa, before_script]
+    - !reference [.omnibus-e2e, before_script]
  rules:
    - !reference [.rules:test:dependency-update-never, rules]
    - if: $QA_SUITES =~ /Test::Integration::AiGatewayNoLicense/
@@ -366,7 +374,7 @@ ai-gateway-no-license:
 continuous-vulnerabiity-scanning:
  extends:
-    - .qa
+    - .omnibus-e2e
  variables:
    QA_SCENARIO: Test::Integration::ContinuousVulnerabilityScanning
    QA_MOCK_GITHUB: "true"
@@ -380,7 +388,7 @@ continuous-vulnerabiity-scanning:
 # ------------------------------------------
 update-minor:
  extends:
-    - .qa
+    - .omnibus-e2e
    - .update-script
  variables:
    UPDATE_TYPE: minor
@@ -391,7 +399,7 @@ update-minor:
 update-major:
  extends:
-    - .qa
+    - .omnibus-e2e
    - .update-script
  variables:
    UPDATE_TYPE: major
@@ -402,7 +410,7 @@ update-major:
 update-patch:
  extends:
-    - .qa
+    - .omnibus-e2e
    - .update-script
  variables:
    UPDATE_TYPE: patch
@@ -413,7 +421,7 @@ update-patch:
 update-from-patch-to-stable:
  extends:
-    - .qa
+    - .omnibus-e2e
    - .update-script
  variables:
    QA_RSPEC_TAGS: --tag health_check
@@ -424,7 +432,7 @@ update-from-patch-to-stable:
 update-ee-to-ce:
  extends:
-    - .qa
+    - .omnibus-e2e
    - .update-script
  variables:
    UPDATE_TYPE: minor
@@ -454,8 +462,12 @@ export-test-metrics:
  extends:
    - .export-test-metrics
    - .rules:report:process-results
+  variables:
+    QA_METRICS_REPORT_FILE_PATTERN: $CI_PROJECT_DIR/gitlab-qa-run-*/**/test-metrics-*.json
 notify-slack:
  extends:
    - .notify-slack
    - .rules:report:process-results
+  variables:
+    QA_RSPEC_XML_FILE_PATTERN: "${CI_PROJECT_DIR}/gitlab-qa-run-*/**/rspec-*.xml"
--- a/qa/qa/tools/ci/pipeline_creator.rb
+++ b/qa/qa/tools/ci/pipeline_creator.rb
@@ -64,7 +64,9 @@ def create(pipeline_types = SUPPORTED_PIPELINES)
          updated_pipeline_definitions(pipeline_types).each do |type, yaml|
            file_name = generated_yml_file_name(type)
-            File.write(file_name, yaml)
+            # mark pipeline as gitlab project pipeline due to issue with variables not working with 'include' keyword
+            # https://gitlab.com/gitlab-org/gitlab/-/issues/378717
+            File.write(file_name, yaml.gsub("$GITLAB_PROJECT_PIPELINE", '"true"'))
            logger.info("Pipeline definition file created: '#{file_name}'")
          end
        end
@@ -154,11 +156,10 @@ def scenario_examples
        # @return [String]
        def variables_section
          @pipeline_variables ||= "variables:\n".then do |variables|
-            qa_cache_digest = Digest::MD5.file("Gemfile.lock").hexdigest # rubocop:disable Fips/MD5 -- CI specific digest does not require FIPS compliance
            ruby_version = File.read(File.join(project_root, ".ruby-version")).strip
            vars = {
+              "RUBY_VERSION" => ENV["RUBY_VERSION"] || ruby_version,
              "GITLAB_SEMVER_VERSION" => File.read(File.join(project_root, "VERSION")),
-              "GITLAB_QA_CACHE_KEY" => "qa-e2e-ruby-#{ENV['RUBY_VERSION'] || ruby_version}-#{qa_cache_digest}",
              "FEATURE_FLAGS" => env["QA_FEATURE_FLAGS"],
              # QA_SUITES is only used by test-on-omnibus due to pipeline being reusable in external projects
              "QA_SUITES" => executable_qa_suites,

--- a/qa/spec/tools/ci/pipeline_creator_spec.rb
+++ b/qa/spec/tools/ci/pipeline_creator_spec.rb
@@ -97,8 +97,8 @@ module QA
      let(:variables) do
        {
+          "RUBY_VERSION" => ruby_version,
          "GITLAB_SEMVER_VERSION" => gitlab_version,
-          "GITLAB_QA_CACHE_KEY" => "qa-e2e-ruby-#{ruby_version}-#{md5_sum.hexdigest}",
          "FEATURE_FLAGS" => env["QA_FEATURE_FLAGS"],
          "QA_SUITES" => scenario_class.to_s
        }.compact