From 54a1a8d34e39ed6df014b7375183bdda2e97e8b3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9my=20Coutable?= <remy@rymai.me>
Date: Fri, 27 Jan 2023 12:41:22 +0100
Subject: [PATCH] ci: Only set up Geo database for the 'rspec-ee' jobs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Rémy Coutable <remy@rymai.me>
---
.gitlab-ci.yml | 3 ++-
.gitlab/ci/dev-fixtures.gitlab-ci.yml | 2 +-
.gitlab/ci/rails.gitlab-ci.yml | 19 ++++++++++++++++---
.gitlab/ci/setup.gitlab-ci.yml | 6 ++++--
{spec => ee/spec}/models/factories_spec.rb | 4 ++++
scripts/db_tasks | 21 +++++++++++++--------
scripts/prepare_build.sh | 10 ++++++----
scripts/rspec_helpers.sh | 10 +++++++++-
scripts/verify-tff-mapping | 4 ++--
tests.yml | 2 +-
10 files changed, 58 insertions(+), 23 deletions(-)
rename {spec => ee/spec}/models/factories_spec.rb (94%)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 50441492d60da..642b9bcc94eae 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -150,8 +150,9 @@ variables:
RSPEC_FOSS_IMPACT_PIPELINE_YML: rspec-foss-impact-pipeline.yml
RSPEC_LAST_RUN_RESULTS_FILE: rspec/rspec_last_run_results.txt
RSPEC_MATCHING_JS_FILES_PATH: rspec/js_matching_files.txt
- RSPEC_MATCHING_TESTS_FOSS_PATH: rspec/matching_tests-foss.txt
RSPEC_MATCHING_TESTS_PATH: rspec/matching_tests.txt
+ RSPEC_MATCHING_TESTS_FOSS_PATH: rspec/matching_tests-foss.txt
+ RSPEC_MATCHING_TESTS_EE_PATH: rspec/matching_tests-ee.txt
RSPEC_PACKED_TESTS_MAPPING_PATH: crystalball/packed-mapping.json
RSPEC_PROFILING_FOLDER_PATH: rspec/profiling
RSPEC_TESTS_MAPPING_PATH: crystalball/mapping.json
diff --git a/.gitlab/ci/dev-fixtures.gitlab-ci.yml b/.gitlab/ci/dev-fixtures.gitlab-ci.yml
index ea868ada62109..0ff469d71143d 100644
--- a/.gitlab/ci/dev-fixtures.gitlab-ci.yml
+++ b/.gitlab/ci/dev-fixtures.gitlab-ci.yml
@@ -16,7 +16,7 @@
.run-dev-fixtures-script: &run-dev-fixtures-script
- run_timed_command "scripts/gitaly-test-spawn"
- - run_timed_command "RAILS_ENV=test bundle exec rake db:seed_fu"
+ - run_timed_command "bundle exec rake db:seed_fu"
run-dev-fixtures:
extends:
diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml
index d0c5336de2537..6de0ad422350f 100644
--- a/.gitlab/ci/rails.gitlab-ci.yml
+++ b/.gitlab/ci/rails.gitlab-ci.yml
@@ -794,20 +794,33 @@ rspec-ee system pg13:
##################################################
# EE: Canonical MR pipelines
-rspec fail-fast:
+.rspec-fail-fast:
extends:
- - .rspec-ee-base-pg12 # This job also runs EE spec which needs elasticsearch
- .rails:rules:rspec fail-fast
stage: test
needs: ["setup-test-env", "retrieve-tests-metadata", "compile-test-assets", "detect-tests"]
script:
- !reference [.base-script, script]
- - rspec_fail_fast "${RSPEC_MATCHING_TESTS_PATH}" "--tag ~quarantine --tag ~zoekt"
+ - rspec_fail_fast "${MATCHING_TESTS_PATH}" "--tag ~quarantine --tag ~zoekt"
artifacts:
expire_in: 7d
paths:
- tmp/capybara/
+rspec fail-fast:
+ extends:
+ - .rspec-fail-fast
+ - .rspec-base-pg12
+ variables:
+ MATCHING_TESTS_PATH: "${RSPEC_MATCHING_TESTS_FOSS_PATH}"
+
+rspec-ee fail-fast:
+ extends:
+ - .rspec-fail-fast
+ - .rspec-base-pg12
+ variables:
+ MATCHING_TESTS_PATH: "${RSPEC_MATCHING_TESTS_EE_PATH}"
+
rspec-foss-impact:pipeline-generate:
extends:
- .rails:rules:rspec-foss-impact
diff --git a/.gitlab/ci/setup.gitlab-ci.yml b/.gitlab/ci/setup.gitlab-ci.yml
index 852c4739d9dd5..f9ef0f7ad4962 100644
--- a/.gitlab/ci/setup.gitlab-ci.yml
+++ b/.gitlab/ci/setup.gitlab-ci.yml
@@ -137,10 +137,11 @@ detect-tests:
tooling/bin/find_tests ${RSPEC_CHANGED_FILES_PATH} ${RSPEC_MATCHING_TESTS_PATH};
tooling/bin/find_changes ${RSPEC_CHANGED_FILES_PATH} ${RSPEC_MATCHING_TESTS_PATH} ${FRONTEND_FIXTURES_MAPPING_PATH};
filter_rspec_matched_foss_tests ${RSPEC_MATCHING_TESTS_PATH} ${RSPEC_MATCHING_TESTS_FOSS_PATH};
+ filter_rspec_matched_ee_tests ${RSPEC_MATCHING_TESTS_PATH} ${RSPEC_MATCHING_TESTS_EE_PATH};
tooling/bin/view_to_js_mappings ${RSPEC_CHANGED_FILES_PATH} ${RSPEC_MATCHING_JS_FILES_PATH};
echoinfo "Changed files: $(cat $RSPEC_CHANGED_FILES_PATH)";
- echoinfo "Related RSpec tests: $(cat $RSPEC_MATCHING_TESTS_PATH)";
echoinfo "Related FOSS RSpec tests: $(cat $RSPEC_MATCHING_TESTS_FOSS_PATH)";
+ echoinfo "Related EE RSpec tests: $(cat $RSPEC_MATCHING_TESTS_EE_PATH)";
echoinfo "Related JS files: $(cat $RSPEC_MATCHING_JS_FILES_PATH)";
fi
artifacts:
@@ -149,8 +150,9 @@ detect-tests:
- ${FRONTEND_FIXTURES_MAPPING_PATH}
- ${RSPEC_CHANGED_FILES_PATH}
- ${RSPEC_MATCHING_JS_FILES_PATH}
- - ${RSPEC_MATCHING_TESTS_FOSS_PATH}
- ${RSPEC_MATCHING_TESTS_PATH}
+ - ${RSPEC_MATCHING_TESTS_FOSS_PATH}
+ - ${RSPEC_MATCHING_TESTS_EE_PATH}
detect-previous-failed-tests:
extends:
diff --git a/spec/models/factories_spec.rb b/ee/spec/models/factories_spec.rb
similarity index 94%
rename from spec/models/factories_spec.rb
rename to ee/spec/models/factories_spec.rb
index d6e746986d65d..575f03bc5e01d 100644
--- a/spec/models/factories_spec.rb
+++ b/ee/spec/models/factories_spec.rb
@@ -2,6 +2,10 @@
require 'spec_helper'
+# We only test factories in EE as `FactoryBot.factories` include both FOSS and EE factories there.
+# Otherwise, we'd test EE factories even in `rspec` jobs which are supposed to run only FOSS tests.
+# It should be fine since FOSS is a subset of EE anyway, so if this test file passes on EE, it means
+# FOSS factories are valid (and FOSS doesn't **add** anything to EE so we're good).
# `:saas` is used to test `gitlab_subscription` factory.
# It's not available on FOSS but also this very factory is not.
RSpec.describe 'factories', :saas, :with_license, feature_category: :tooling do
diff --git a/scripts/db_tasks b/scripts/db_tasks
index 56c2eefef687a..c3b7bb84355ec 100755
--- a/scripts/db_tasks
+++ b/scripts/db_tasks
@@ -1,12 +1,17 @@
-#!/bin/bash
+#!/usr/bin/env ruby
+# frozen_string_literal: true
-root_path="$(cd "$(dirname "$0")/.." || exit ; pwd -P)"
-task=$1
+require 'yaml'
-shift
+rails_env = ENV.fetch('RAILS_ENV')
+database_config = YAML.load_file(File.join(File.expand_path('..', __dir__), 'config', 'database.yml'))[rails_env]
-if [[ -d "${root_path}/ee/" || "${DECOMPOSED_DB}" == "true" ]]; then
- task="${task}:main"
-fi
+task = ARGV.shift
+raise ArgumentError, 'You need to pass a task name!' unless task
-eval "bundle exec rake ${task} ${*}"
+task = "${task}:main" unless database_config.one?
+cmd = ['bundle', 'exec', 'rake', task, *ARGV]
+
+puts "Running: `#{cmd.join(' ')}`"
+
+system(*cmd)
diff --git a/scripts/prepare_build.sh b/scripts/prepare_build.sh
index d68432d9ec0bc..500e61ab76aa4 100644
--- a/scripts/prepare_build.sh
+++ b/scripts/prepare_build.sh
@@ -17,10 +17,12 @@ else
cp config/database.yml.postgresql config/database.yml
fi
-# Remove Geo database setting if `ee/` directory does not exist. When it does
-# not exist, it runs the GitLab test suite "as if FOSS", meaning the jobs run
-# in the context of gitlab-org/gitlab-foss where the Geo is not available.
-if [ ! -d "ee/" ] ; then
+# Set up Geo database if the job name matches `rspec-ee` or `geo`.
+# Since Geo is an EE feature, we shouldn't set it up for non-EE tests.
+if [[ "${CI_JOB_NAME}" =~ "rspec-ee" ]] || [[ "${CI_JOB_NAME}" =~ "geo" ]]; then
+ echoinfo "Geo DB will be set up."
+else
+ echoinfo "Geo DB won't be set up."
sed -i '/geo:/,/^$/d' config/database.yml
fi
diff --git a/scripts/rspec_helpers.sh b/scripts/rspec_helpers.sh
index 5845a6e3f2548..5b2c84059ee21 100644
--- a/scripts/rspec_helpers.sh
+++ b/scripts/rspec_helpers.sh
@@ -351,10 +351,18 @@ function filter_rspec_matched_foss_tests() {
local matching_tests_file="${1}"
local foss_matching_tests_file="${2}"
- # Keep only files that exists (i.e. exclude EE speficic files)
+ # Keep only FOSS files that exists
cat ${matching_tests_file} | ruby -e 'puts $stdin.read.split(" ").select { |f| f.start_with?("spec/") && File.exist?(f) }.join(" ")' > "${foss_matching_tests_file}"
}
+function filter_rspec_matched_ee_tests() {
+ local matching_tests_file="${1}"
+ local ee_matching_tests_file="${2}"
+
+ # Keep only EE files that exists
+ cat ${matching_tests_file} | ruby -e 'puts $stdin.read.split(" ").select { |f| f.start_with?("ee/spec/") && File.exist?(f) }.join(" ")' > "${ee_matching_tests_file}"
+}
+
function generate_frontend_fixtures_mapping() {
local pattern=""
diff --git a/scripts/verify-tff-mapping b/scripts/verify-tff-mapping
index 08d9d7a33fdd5..a947033776133 100755
--- a/scripts/verify-tff-mapping
+++ b/scripts/verify-tff-mapping
@@ -127,13 +127,13 @@ tests = [
{
explanation: 'FOSS factory should map to factories spec',
source: 'spec/factories/users.rb',
- expected: ['spec/models/factories_spec.rb']
+ expected: ['ee/spec/models/factories_spec.rb']
},
{
explanation: 'EE factory should map to factories spec',
source: 'ee/spec/factories/users.rb',
- expected: ['spec/models/factories_spec.rb']
+ expected: ['ee/spec/models/factories_spec.rb']
},
{
diff --git a/tests.yml b/tests.yml
index 4526c9cf1f576..1135d475cc4fc 100644
--- a/tests.yml
+++ b/tests.yml
@@ -63,7 +63,7 @@ mapping:
# EE/FOSS factory should map to factories spec
- source: (ee/)?spec/factories/.+\.rb
- test: spec/models/factories_spec.rb
+ test: ee/spec/models/factories_spec.rb
# Whats New should map to its respective spec
- source: data/whats_new/\w*.yml
--
GitLab