From 1c8768860c92667d1699c94fc7c4eb71ec7b43a1 Mon Sep 17 00:00:00 2001
From: Andrejs Cunskis <acunskis@gitlab.com>
Date: Mon, 18 Mar 2024 21:03:36 +0000
Subject: [PATCH] Add blocking suite to cng pipeline
Fix needs
Allow local requests for cng tests
Add additional logging for fetching ip address
Remove host assumptions from ip address detection flow
---
.gitlab/ci/review-apps/qa.gitlab-ci.yml | 70 -------------------
.gitlab/ci/review-apps/rules.gitlab-ci.yml | 22 ------
.gitlab/ci/review.gitlab-ci.yml | 6 +-
.gitlab/ci/test-on-cng/main.gitlab-ci.yml | 36 ++++++++--
qa/qa/runtime/ip_address.rb | 18 ++---
.../features/api/1_manage/rate_limits_spec.rb | 7 +-
.../group/restrict_by_ip_address_spec.rb | 24 +++----
scripts/generate-e2e-pipeline | 1 -
scripts/qa/cng_deploy/cng-kind.sh | 2 -
scripts/qa/cng_deploy/config/kind-config.yml | 7 +-
10 files changed, 60 insertions(+), 133 deletions(-)
diff --git a/.gitlab/ci/review-apps/qa.gitlab-ci.yml b/.gitlab/ci/review-apps/qa.gitlab-ci.yml
index e8969e35256a0..13f476d561094 100644
--- a/.gitlab/ci/review-apps/qa.gitlab-ci.yml
+++ b/.gitlab/ci/review-apps/qa.gitlab-ci.yml
@@ -1,50 +1,6 @@
include:
- - local: .gitlab/ci/qa-common/main.gitlab-ci.yml
- template: Verify/Browser-Performance.gitlab-ci.yml
-.bundle-base:
- extends:
- - .qa-cache
- - .ruby-image
- before_script:
- - cd qa && bundle install
-
-review-qa-smoke:
- extends:
- - .use-docker-in-docker
- - .bundle-base
- - .default-retry
- - .rules:qa-smoke
- image: ${REGISTRY_HOST}/${REGISTRY_GROUP}/gitlab-build-images/${BUILD_OS}-${OS_VERSION}-ruby-${RUBY_VERSION}:bundler-${BUNDLER_VERSION}-git-2.36-lfs-2.9-chrome-${CHROME_VERSION}-docker-${DOCKER_VERSION}-gcloud-383-kubectl-1.23
- stage: qa
- needs: [review-deploy]
- variables:
- RSPEC_REPORT_OPTS: --force-color --order random --format documentation --format RspecJunitFormatter --out tmp/rspec-${CI_JOB_ID}.xml
- GITLAB_USERNAME: "root"
- GITLAB_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}"
- GITLAB_ADMIN_USERNAME: "root"
- GITLAB_ADMIN_PASSWORD: "${REVIEW_APPS_ROOT_PASSWORD}"
- GITLAB_QA_ADMIN_ACCESS_TOKEN: "${REVIEW_APPS_ROOT_TOKEN}"
- GITHUB_ACCESS_TOKEN: "${QA_GITHUB_ACCESS_TOKEN}"
- COLORIZED_LOGS: "true"
- QA_GENERATE_ALLURE_REPORT: "true"
- QA_CAN_TEST_PRAEFECT: "false"
- script:
- - QA_COMMAND="bundle exec bin/qa Test::Instance::Smoke ${QA_GITLAB_URL} -- ${QA_TESTS} ${RSPEC_REPORT_OPTS}"
- - echo "Running - '${QA_COMMAND}'"
- - eval "$QA_COMMAND"
- after_script:
- - |
- echo "Sentry errors for the current review-app test run can be found via following url:"
- echo "https://new-sentry.gitlab.net/organizations/gitlab/releases/$(echo "${CI_COMMIT_SHA}" | cut -c1-11)/?environment=review&issuesType=all&project=19"
- artifacts:
- paths:
- - qa/tmp
- reports:
- junit: qa/tmp/rspec-*.xml
- expire_in: 7 days
- when: always
-
browser_performance:
extends:
- .default-retry
@@ -58,29 +14,3 @@ browser_performance:
DOCKER_HOST: tcp://docker:2375
DOCKER_TLS_CERTDIR: ""
URL: environment_url.txt
-
-e2e-test-report:
- extends: .rules:prepare-report
- stage: report
- variables:
- ALLURE_RESULTS_GLOB: "qa/tmp/allure-results"
-
-notify-slack:
- extends:
- - .notify-slack
- - .rules:main-run
- stage: report
- variables:
- QA_RSPEC_XML_FILE_PATTERN: ${CI_PROJECT_DIR}/qa/tmp/rspec-*.xml
- RUN_WITH_BUNDLE: "true"
- when: on_failure
-
-export-test-metrics:
- extends:
- - .export-test-metrics
- - .bundle-base
- - .rules:main-run
- stage: report
- variables:
- QA_METRICS_REPORT_FILE_PATTERN: tmp/test-metrics-*.json
- when: always
diff --git a/.gitlab/ci/review-apps/rules.gitlab-ci.yml b/.gitlab/ci/review-apps/rules.gitlab-ci.yml
index 3c8169a472291..cd04e31c3e5af 100644
--- a/.gitlab/ci/review-apps/rules.gitlab-ci.yml
+++ b/.gitlab/ci/review-apps/rules.gitlab-ci.yml
@@ -1,14 +1,6 @@
# ------------------------------------------
# Conditions
# ------------------------------------------
-# Triggered by change pattern
-.app-changes: &app-changes
- if: $APP_CHANGE_TRIGGER == "true"
-
-# Run all tests when framework changes present or explicitly enabled full suite execution
-.qa-run-all-tests: &qa-run-all-tests
- if: $QA_FRAMEWORK_CHANGES == "true" || $QA_RUN_ALL_TESTS == "true" || $QA_RUN_ALL_E2E_LABEL == "true"
-
.default-branch: &default-branch
if: $CI_COMMIT_REF_NAME == $CI_DEFAULT_BRANCH
@@ -68,20 +60,6 @@
# ------------------------------------------
# Test
# ------------------------------------------
-.rules:qa-smoke:
- rules:
- # always trigger smoke suite if review pipeline got triggered by specific changes in application code
- - <<: *app-changes
- variables:
- QA_TESTS: "" # unset QA_TESTS even if specific tests were inferred from stage label
- - *qa-run-all-tests
- - if: $QA_SUITES =~ /Test::Instance::Smoke/
- # keep option to trigger tests manually even if no rules match
- - when: manual
- allow_failure: true
- variables:
- QA_TESTS: ""
-
.review:rules:review-performance:
rules:
- if: '$DAST_RUN == "true"' # Skip this job when DAST is run
diff --git a/.gitlab/ci/review.gitlab-ci.yml b/.gitlab/ci/review.gitlab-ci.yml
index 516f0747bf2b0..0e138d19f042b 100644
--- a/.gitlab/ci/review.gitlab-ci.yml
+++ b/.gitlab/ci/review.gitlab-ci.yml
@@ -74,7 +74,6 @@ start-review-app-pipeline:
resource_group: review/${CI_COMMIT_REF_SLUG}${SCHEDULE_TYPE} # CI_ENVIRONMENT_SLUG is not available here and we want this to be the same as the environment
stage: review
needs:
- - job: e2e-test-pipeline-generate
- job: rails-production-server-boot-puma-example
optional: true
- job: rails-production-server-boot-puma-cng
@@ -108,10 +107,7 @@ start-review-app-pipeline:
PARENT_PIPELINE_ID: $CI_PIPELINE_ID
SCHEDULE_TYPE: $SCHEDULE_TYPE
DAST_RUN: $DAST_RUN
- SKIP_MESSAGE: Skipping review-app due to mr containing only quarantine changes!
- QA_RUN_TYPE: e2e-review-qa
trigger:
strategy: depend
include:
- - artifact: review-app-pipeline.yml
- job: e2e-test-pipeline-generate
+ - local: .gitlab/ci/review-apps/main.gitlab-ci.yml
diff --git a/.gitlab/ci/test-on-cng/main.gitlab-ci.yml b/.gitlab/ci/test-on-cng/main.gitlab-ci.yml
index ccbd419628b26..ea560725593ad 100644
--- a/.gitlab/ci/test-on-cng/main.gitlab-ci.yml
+++ b/.gitlab/ci/test-on-cng/main.gitlab-ci.yml
@@ -15,7 +15,7 @@ workflow:
stage: test
extends: .qa-cache
needs: [build-cng]
- tags: [saas-linux-2xlarge-amd64]
+ tags: [e2e]
services:
- docker:${DOCKER_VERSION}-dind
variables:
@@ -32,6 +32,7 @@ workflow:
QA_DOCKER_NETWORK: host
QA_GENERATE_ALLURE_REPORT: "true"
QA_CAN_TEST_PRAEFECT: "false"
+ QA_ALLOW_LOCAL_REQUESTS: "true"
QA_SUITE_STATUS_ENV_FILE: $CI_PROJECT_DIR/suite_status.env
before_script:
- echo "SUITE_RAN=true" > "$QA_SUITE_STATUS_ENV_FILE"
@@ -61,9 +62,8 @@ workflow:
junit: qa/tmp/rspec-*.xml
dotenv: $QA_SUITE_STATUS_ENV_FILE
paths:
- - "*.log"
- - qa/tmp/test-metrics-*.json
- - qa/tmp/allure-results
+ - qa/tmp
+ - ${CI_PROJECT_DIR}/*.log
# ==========================================
# Pre stage
@@ -77,15 +77,32 @@ build-cng:
extends: .build-cng
needs: [build-cng-env]
+download-knapsack-report:
+ extends:
+ - .download-knapsack-report
+ - .ruby-image
+ - .qa-cache
+ - .rules:download-knapsack
+ variables:
+ GIT_STRATEGY: clone
+ before_script:
+ - cd qa && bundle install
+ after_script: []
+
# ==========================================
# Test stage
# ==========================================
cng-qa-smoke:
- extends:
- - .cng-base
+ extends: .cng-base
variables:
QA_SCENARIO: Test::Instance::Smoke
+cng-qa-blocking:
+ extends: .cng-base
+ variables:
+ QA_SCENARIO: Test::Instance::Blocking
+ parallel: 10
+
# Test run against environment with minimum supported redis version defined in lib/system_check/app/redis_version_check.rb
cng-qa-min-redis-version:
extends: cng-qa-smoke
@@ -113,3 +130,10 @@ notify-slack:
- .rules:report:process-results
variables:
QA_RSPEC_XML_FILE_PATTERN: $CI_PROJECT_DIR/qa/tmp/rspec-*.xml
+
+upload-knapsack-report:
+ extends:
+ - .upload-knapsack-report
+ - .rules:report:process-results
+ variables:
+ QA_KNAPSACK_REPORT_FILE_PATTERN: $CI_PROJECT_DIR/qa/tmp/knapsack/*/*.json
diff --git a/qa/qa/runtime/ip_address.rb b/qa/qa/runtime/ip_address.rb
index ae83d10ffb508..f3100f8c68992 100644
--- a/qa/qa/runtime/ip_address.rb
+++ b/qa/qa/runtime/ip_address.rb
@@ -1,4 +1,5 @@
# frozen_string_literal: true
+
require 'socket'
module QA
@@ -13,23 +14,24 @@ module IPAddress
def fetch_current_ip_address
# When running on CI against a live environment such as staging.gitlab.com,
# we use the public facing IP address
- non_test_host = !URI.parse(Scenario.gitlab_address).host.include?('.test') # rubocop:disable Rails/NegateInclude
- has_no_public_ip = Env.running_in_ci? || Env.use_public_ip_api?
-
- ip_address = if has_no_public_ip && non_test_host
+ ip_address = if Env.use_public_ip_api?
+ Logger.debug 'Using public IP address'
response = get_public_ip_address
- raise HostUnreachableError, "#{PUBLIC_IP_ADDRESS_API} is unreachable" unless response.code == Support::API::HTTP_STATUS_OK
+ unless response.code == Support::API::HTTP_STATUS_OK
+ raise HostUnreachableError, "#{PUBLIC_IP_ADDRESS_API} is unreachable"
+ end
response.body
elsif page.current_host.include?('localhost')
+ Logger.debug 'Using loopback IP address'
LOOPBACK_ADDRESS
else
- Socket.ip_address_list.detect { |intf| intf.ipv4_private? }.ip_address
+ Logger.debug 'Using private IP address'
+ Socket.ip_address_list.detect(&:ipv4_private?).ip_address
end
- QA::Runtime::Logger.info "Current IP address: #{ip_address}"
-
+ Logger.info "Current IP address: #{ip_address}"
ip_address
end
diff --git a/qa/qa/specs/features/api/1_manage/rate_limits_spec.rb b/qa/qa/specs/features/api/1_manage/rate_limits_spec.rb
index 77e3618e2c7b3..5bc2c5da12d18 100644
--- a/qa/qa/specs/features/api/1_manage/rate_limits_spec.rb
+++ b/qa/qa/specs/features/api/1_manage/rate_limits_spec.rb
@@ -1,7 +1,9 @@
# frozen_string_literal: true
module QA
- RSpec.describe 'Manage', :requires_admin, :skip_live_env, except: { job: %w[review-qa-* gdk-qa-*] } do
+ RSpec.describe 'Manage', :requires_admin, :skip_live_env, only: {
+ condition: -> { ENV['QA_RUN_TYPE']&.match?("e2e-package-and-test") }
+ } do
describe 'rate limits', :reliable, product_group: :import_and_integrate do
let(:rate_limited_user) { create(:user) }
let(:api_client) { Runtime::API::Client.new(:gitlab, user: rate_limited_user) }
@@ -11,7 +13,8 @@ module QA
rate_limited_user.remove_via_api!
end
- it 'throttles authenticated api requests by user', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347881' do
+ it 'throttles authenticated api requests by user',
+ testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347881' do
with_application_settings(
throttle_authenticated_api_requests_per_period: 5,
throttle_authenticated_api_period_in_seconds: 60,
diff --git a/qa/qa/specs/features/ee/browser_ui/10_govern/group/restrict_by_ip_address_spec.rb b/qa/qa/specs/features/ee/browser_ui/10_govern/group/restrict_by_ip_address_spec.rb
index 4fd58c18cc559..90864d9e87da3 100644
--- a/qa/qa/specs/features/ee/browser_ui/10_govern/group/restrict_by_ip_address_spec.rb
+++ b/qa/qa/specs/features/ee/browser_ui/10_govern/group/restrict_by_ip_address_spec.rb
@@ -2,9 +2,8 @@
module QA
RSpec.describe 'Govern' do
- describe 'Group access',
- :requires_admin, :skip_live_env, :reliable, product_group: :authentication do
- include Runtime::IPAddress
+ describe 'Group access', :requires_admin, :skip_live_env, :reliable, product_group: :authentication do
+ let(:admin_api_client) { Runtime::API::Client.as_admin }
let(:sandbox_group) do
Resource::Sandbox.fabricate! do |sandbox_group|
@@ -14,12 +13,13 @@ module QA
let(:group) { create(:group, path: "ip-address-restricted-group-#{SecureRandom.hex(8)}", sandbox: sandbox_group) }
let(:project) { create(:project, :with_readme, name: 'project-in-ip-restricted-group', group: group) }
- let(:user) do
- Resource::User.fabricate_or_use(Runtime::Env.gitlab_qa_username_1, Runtime::Env.gitlab_qa_password_1)
- end
-
+ let(:user) { create(:user, api_client: admin_api_client) }
let(:api_client) { Runtime::API::Client.new(:gitlab, user: user) }
+ let!(:current_ip_address) do
+ Flow::Login.while_signed_in(as: user) { user.reload!.api_response[:current_sign_in_ip] }
+ end
+
before do
project.add_member(user)
@@ -37,7 +37,7 @@ module QA
end
context 'when restricted by another ip address' do
- let(:ip_address) { get_next_ip_address(fetch_current_ip_address) }
+ let(:ip_address) { get_next_ip_address(current_ip_address) }
context 'with UI' do
it 'denies access', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347923' do
@@ -85,7 +85,7 @@ module QA
end
context 'when restricted by user\'s ip address' do
- let(:ip_address) { fetch_current_ip_address }
+ let(:ip_address) { current_ip_address }
context 'with UI' do
it 'allows access', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/347926' do
@@ -151,12 +151,12 @@ def set_ip_address_restriction_to(ip_address)
end
end
- def get_next_ip_address(current_ip_address)
- current_last_part = current_ip_address.split(".").pop.to_i
+ def get_next_ip_address(address)
+ current_last_part = address.split(".").pop.to_i
updated_last_part = current_last_part < 255 ? current_last_part + 1 : 1
- current_ip_address.split(".")[0...-1].push(updated_last_part).join(".")
+ address.split(".")[0...-1].push(updated_last_part).join(".")
end
def enable_plan_on_group(group, plan)
diff --git a/scripts/generate-e2e-pipeline b/scripts/generate-e2e-pipeline
index eb699b63eec4c..ff56e065126df 100755
--- a/scripts/generate-e2e-pipeline
+++ b/scripts/generate-e2e-pipeline
@@ -15,7 +15,6 @@ declare -A qa_pipelines
# key/value pairs for qa pipeline yml definitions
qa_pipelines["package-and-test-pipeline.yml"]="package-and-test/main.gitlab-ci.yml"
qa_pipelines["package-and-test-nightly-pipeline.yml"]="package-and-test-nightly/main.gitlab-ci.yml"
-qa_pipelines["review-app-pipeline.yml"]="review-apps/main.gitlab-ci.yml"
qa_pipelines["test-on-gdk-pipeline.yml"]="test-on-gdk/main.gitlab-ci.yml"
qa_pipelines["test-on-cng-pipeline.yml"]="test-on-cng/main.gitlab-ci.yml"
diff --git a/scripts/qa/cng_deploy/cng-kind.sh b/scripts/qa/cng_deploy/cng-kind.sh
index e13e0f96844e1..58875b7a1b23d 100644
--- a/scripts/qa/cng_deploy/cng-kind.sh
+++ b/scripts/qa/cng_deploy/cng-kind.sh
@@ -59,8 +59,6 @@ global:
configureCertmanager: false
tls:
enabled: false
- shell:
- port: 32022
extraEnv:
GITLAB_LICENSE_MODE: test
CUSTOMER_PORTAL_URL: https://customers.staging.gitlab.com
diff --git a/scripts/qa/cng_deploy/config/kind-config.yml b/scripts/qa/cng_deploy/config/kind-config.yml
index e1f7af5c82eba..fa568522b5359 100644
--- a/scripts/qa/cng_deploy/config/kind-config.yml
+++ b/scripts/qa/cng_deploy/config/kind-config.yml
@@ -18,14 +18,11 @@ nodes:
extraPortMappings:
# containerPort below must match the values file:
# nginx-ingress.controller.service.nodePorts.http
- # Change hostPort if port 80 is already in use.
- containerPort: 32080
hostPort: 80
listenAddress: "0.0.0.0"
# containerPort below must match the values file:
- # nginx-ingress.controller.service.nodePorts.ssh
- # Using high-numbered hostPort assuming port 22 is
- # already in use.
+ # nginx-ingress.controller.service.nodePorts.gitlab-shell
- containerPort: 32022
- hostPort: 32022
+ hostPort: 22
listenAddress: "0.0.0.0"
--
GitLab