From e7ef852e1602ebdae1f8dc0acd7dd47d74f0ec8b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?R=C3=A9my=20Coutable?= <remy@rymai.me>
Date: Tue, 20 Feb 2024 13:31:54 +0100
Subject: [PATCH] ci: Detect flaky tests in
.gitlab/ci/rails/shared.gitlab-ci.yml
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Signed-off-by: Rémy Coutable <remy@rymai.me>
---
.gitlab/ci/rails/shared.gitlab-ci.yml | 20 +++++++++++++++++++-
scripts/rspec_helpers.sh | 9 ---------
2 files changed, 19 insertions(+), 10 deletions(-)
diff --git a/.gitlab/ci/rails/shared.gitlab-ci.yml b/.gitlab/ci/rails/shared.gitlab-ci.yml
index 0237d4e8e5506..12a3dde31d8cc 100644
--- a/.gitlab/ci/rails/shared.gitlab-ci.yml
+++ b/.gitlab/ci/rails/shared.gitlab-ci.yml
@@ -93,14 +93,30 @@ include:
- bundle exec gem list gitlab_quality-test_tooling
- |
if [ "$CREATE_RAILS_TEST_FAILURE_ISSUES" == "true" ]; then
+ input_file="rspec/rspec-${CI_JOB_ID}.json"
+
+ # The actual failures will always be part of the retry report
+ if [ -f "rspec/rspec-retry-${CI_JOB_ID}.json" ]; then
+ input_file="rspec/rspec-retry-${CI_JOB_ID}.json"
+ fi
+
bundle exec relate-failure-issue \
--token "${TEST_FAILURES_PROJECT_TOKEN}" \
--project "gitlab-org/gitlab" \
- --input-files "rspec/rspec-*.json" \
+ --input-files "${input_file}" \
--exclude-labels-for-search "QA,rspec:slow test,knapsack_report" \
--system-log-files "log" \
--related-issues-file "rspec/${CI_JOB_ID}-failed-test-issues.json";
fi
+
+ if [ "$CREATE_RAILS_FLAKY_TEST_ISSUES" == "true" && -f "rspec/rspec-retry-${CI_JOB_ID}.json" ]; then
+ bundle exec flaky-test-issues \
+ --token "${RAILS_FLAKY_TEST_PROJECT_TOKEN}" \
+ --project "gitlab-org/quality/engineering-productivity/flaky-tests" \
+ --merge_request_iid "$CI_MERGE_REQUEST_IID" \
+ --input-files "rspec/rspec-retry-${CI_JOB_ID}.json" || true # We don't want this command to fail the job.
+ fi
+
if [ "$CREATE_RAILS_SLOW_TEST_ISSUES" == "true" ]; then
bundle exec slow-test-issues \
--token "${TEST_FAILURES_PROJECT_TOKEN}" \
@@ -108,6 +124,7 @@ include:
--input-files "rspec/rspec-*.json" \
--related-issues-file "rspec/${CI_JOB_ID}-slow-test-issues.json";
fi
+
if [ "$ADD_SLOW_TEST_NOTE_TO_MERGE_REQUEST" == "true" ]; then
bundle exec slow-test-merge-request-report-note \
--token "${TEST_SLOW_NOTE_PROJECT_TOKEN}" \
@@ -115,6 +132,7 @@ include:
--input-files "rspec/rspec-*.json" \
--merge_request_iid "$CI_MERGE_REQUEST_IID";
fi
+
if [ "$ALLOW_KNAPSACK_REPORT_CREATE_ISSUES" == "true" ]; then
bundle exec knapsack-report-issues \
--token "${KNAPSACK_REPORT_ISSUES_PROJECT_TOKEN}" \
diff --git a/scripts/rspec_helpers.sh b/scripts/rspec_helpers.sh
index f42e7bbcfb74b..2a4d106b4b1ee 100644
--- a/scripts/rspec_helpers.sh
+++ b/scripts/rspec_helpers.sh
@@ -342,15 +342,6 @@ function retry_failed_rspec_examples() {
# The tests are flaky because they succeeded after being retried.
if [[ $rspec_run_status -eq 0 ]]; then
- # "53557338" is the project ID of https://gitlab.com/gitlab-org/quality/engineering-productivity/flaky-tests
- if [ "$CREATE_RAILS_FLAKY_TEST_ISSUES" == "true" ]; then
- bundle exec flaky-test-issues \
- --token "${RAILS_FLAKY_TEST_PROJECT_TOKEN}" \
- --project "53557338" \
- --merge_request_iid "$CI_MERGE_REQUEST_IID" \
- --input-files "rspec/rspec-retry-*.json" || true # We don't want this command to fail the job.
- fi
-
# Make the pipeline "pass with warnings" if the flaky tests are part of this MR.
warn_on_successfully_retried_test
fi
--
GitLab