diff --git a/.gitlab/ci/rails/shared.gitlab-ci.yml b/.gitlab/ci/rails/shared.gitlab-ci.yml
index 0237d4e8e5506dcbc96c84933bb873b175371070..12a3dde31d8ccb50adc2bfd06f395ddd8985228a 100644
--- a/.gitlab/ci/rails/shared.gitlab-ci.yml
+++ b/.gitlab/ci/rails/shared.gitlab-ci.yml
@@ -93,14 +93,30 @@ include:
     - bundle exec gem list gitlab_quality-test_tooling
     - |
       if [ "$CREATE_RAILS_TEST_FAILURE_ISSUES" == "true" ]; then
+        input_file="rspec/rspec-${CI_JOB_ID}.json"
+
+        # The actual failures will always be part of the retry report
+        if [ -f "rspec/rspec-retry-${CI_JOB_ID}.json" ]; then
+          input_file="rspec/rspec-retry-${CI_JOB_ID}.json"
+        fi
+
         bundle exec relate-failure-issue \
           --token "${TEST_FAILURES_PROJECT_TOKEN}" \
           --project "gitlab-org/gitlab" \
-          --input-files "rspec/rspec-*.json" \
+          --input-files "${input_file}" \
           --exclude-labels-for-search "QA,rspec:slow test,knapsack_report" \
           --system-log-files "log" \
           --related-issues-file "rspec/${CI_JOB_ID}-failed-test-issues.json";
       fi
+
+      if [ "$CREATE_RAILS_FLAKY_TEST_ISSUES" == "true" && -f "rspec/rspec-retry-${CI_JOB_ID}.json" ]; then
+        bundle exec flaky-test-issues \
+          --token "${RAILS_FLAKY_TEST_PROJECT_TOKEN}" \
+          --project "gitlab-org/quality/engineering-productivity/flaky-tests" \
+          --merge_request_iid "$CI_MERGE_REQUEST_IID" \
+          --input-files "rspec/rspec-retry-${CI_JOB_ID}.json" || true # We don't want this command to fail the job.
+      fi
+
       if [ "$CREATE_RAILS_SLOW_TEST_ISSUES" == "true" ]; then
         bundle exec slow-test-issues \
           --token "${TEST_FAILURES_PROJECT_TOKEN}" \
@@ -108,6 +124,7 @@ include:
           --input-files "rspec/rspec-*.json" \
           --related-issues-file "rspec/${CI_JOB_ID}-slow-test-issues.json";
       fi
+
       if [ "$ADD_SLOW_TEST_NOTE_TO_MERGE_REQUEST" == "true" ]; then
         bundle exec slow-test-merge-request-report-note \
           --token "${TEST_SLOW_NOTE_PROJECT_TOKEN}" \
@@ -115,6 +132,7 @@ include:
           --input-files "rspec/rspec-*.json" \
           --merge_request_iid "$CI_MERGE_REQUEST_IID";
       fi
+
       if [ "$ALLOW_KNAPSACK_REPORT_CREATE_ISSUES" == "true" ]; then
         bundle exec knapsack-report-issues \
           --token "${KNAPSACK_REPORT_ISSUES_PROJECT_TOKEN}" \
diff --git a/scripts/rspec_helpers.sh b/scripts/rspec_helpers.sh
index f42e7bbcfb74bf9a530ab47fdb5b75fcb8b4ca6c..2a4d106b4b1ee830ec68dfe5edaac855663202f5 100644
--- a/scripts/rspec_helpers.sh
+++ b/scripts/rspec_helpers.sh
@@ -342,15 +342,6 @@ function retry_failed_rspec_examples() {
 
   # The tests are flaky because they succeeded after being retried.
   if [[ $rspec_run_status -eq 0 ]]; then
-    # "53557338" is the project ID of https://gitlab.com/gitlab-org/quality/engineering-productivity/flaky-tests
-    if [ "$CREATE_RAILS_FLAKY_TEST_ISSUES" == "true" ]; then
-      bundle exec flaky-test-issues \
-        --token "${RAILS_FLAKY_TEST_PROJECT_TOKEN}" \
-        --project "53557338" \
-        --merge_request_iid "$CI_MERGE_REQUEST_IID" \
-        --input-files "rspec/rspec-retry-*.json" || true # We don't want this command to fail the job.
-    fi
-
     # Make the pipeline "pass with warnings" if the flaky tests are part of this MR.
     warn_on_successfully_retried_test
   fi