From 677aedf853e4d764325d59cb40d0c08302afe431 Mon Sep 17 00:00:00 2001
From: Lin Jen-Shin <jen-shin@gitlab.com>
Date: Tue, 30 Apr 2024 23:37:50 +0800
Subject: [PATCH] Move foss-impact to predictive in the as-if-foss pipeline

---
 .gitlab-ci.yml                                |  1 -
 .gitlab/ci/as-if-foss.gitlab-ci.yml           |  4 +
 .gitlab/ci/rails.gitlab-ci.yml                | 38 --------
 .../rails/rspec-foss-impact.gitlab-ci.yml.erb | 89 -------------------
 .gitlab/ci/rules.gitlab-ci.yml                | 17 +---
 danger/specs/Dangerfile                       |  2 +-
 ee/spec/policies/group_policy_spec.rb         |  2 +-
 scripts/setup/generate-as-if-foss-env.rb      | 12 ++-
 spec/dot_gitlab_ci/rules_spec.rb              | 25 +++---
 .../setup/generate_as_if_foss_env_spec.rb     | 37 ++++++--
 10 files changed, 63 insertions(+), 164 deletions(-)
 delete mode 100644 .gitlab/ci/rails/rspec-foss-impact.gitlab-ci.yml.erb

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index b31f02a275120..4bb4cb4dba284 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -213,7 +213,6 @@ variables:
   RSPEC_CHANGED_FILES_PATH: rspec/changed_files.txt
   RSPEC_FAIL_FAST_THRESHOLD: 20
   RSPEC_FAST_QUARANTINE_PATH: rspec/fast_quarantine-gitlab.txt
-  RSPEC_FOSS_IMPACT_PIPELINE_TEMPLATE_YML: .gitlab/ci/rails/rspec-foss-impact.gitlab-ci.yml.erb
   RSPEC_LAST_RUN_RESULTS_FILE: rspec/rspec_last_run_results.txt
   RSPEC_MATCHING_JS_FILES_PATH: rspec/js_matching_files.txt
   RSPEC_MATCHING_TESTS_EE_PATH: rspec/matching_tests-ee.txt
diff --git a/.gitlab/ci/as-if-foss.gitlab-ci.yml b/.gitlab/ci/as-if-foss.gitlab-ci.yml
index 04f15d895d42a..cdff908b45896 100644
--- a/.gitlab/ci/as-if-foss.gitlab-ci.yml
+++ b/.gitlab/ci/as-if-foss.gitlab-ci.yml
@@ -95,6 +95,10 @@ start-as-if-foss:
     ENABLE_GRAPHQL_SCHEMA_DUMP: $ENABLE_GRAPHQL_SCHEMA_DUMP
     ENABLE_JEST: $ENABLE_JEST
     ENABLE_JEST_INTEGRATION: $ENABLE_JEST_INTEGRATION
+    ENABLE_RSPEC_PREDICTIVE_PIPELINE_GENERATE: $ENABLE_RSPEC_PREDICTIVE_PIPELINE_GENERATE
+    ENABLE_RSPEC_PREDICTIVE_TRIGGER: $ENABLE_RSPEC_PREDICTIVE_TRIGGER
+    ENABLE_RSPEC_PREDICTIVE_TRIGGER_SINGLE_DB: $ENABLE_RSPEC_PREDICTIVE_TRIGGER_SINGLE_DB
+    ENABLE_RSPEC_PREDICTIVE_TRIGGER_SINGLE_DB_CI_CONNECTION: $ENABLE_RSPEC_PREDICTIVE_TRIGGER_SINGLE_DB_CI_CONNECTION
     ENABLE_RUBOCOP: $ENABLE_RUBOCOP
     ENABLE_QA_INTERNAL: $ENABLE_QA_INTERNAL
     ENABLE_QA_SELECTORS: $ENABLE_QA_SELECTORS
diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml
index 8c0ebd55b0b4b..6c9ec56319b61 100644
--- a/.gitlab/ci/rails.gitlab-ci.yml
+++ b/.gitlab/ci/rails.gitlab-ci.yml
@@ -1290,44 +1290,6 @@ rspec-ee fail-fast:
   variables:
     MATCHING_TESTS_PATH: "${RSPEC_MATCHING_TESTS_EE_PATH}"
 
-rspec-foss-impact:pipeline-generate:
-  extends:
-    - .rails:rules:rspec-foss-impact
-  stage: prepare
-  needs: ["detect-tests", "retrieve-tests-metadata"]
-  script:
-    - scripts/generate_rspec_pipeline.rb -f "${RSPEC_MATCHING_TESTS_FOSS_PATH}" -t "${RSPEC_FOSS_IMPACT_PIPELINE_TEMPLATE_YML}" -k "${KNAPSACK_RSPEC_SUITE_REPORT_PATH}"
-    - cat "${RSPEC_FOSS_IMPACT_PIPELINE_TEMPLATE_YML}.yml"
-  artifacts:
-    expire_in: 1 day
-    paths:
-      - "${RSPEC_FOSS_IMPACT_PIPELINE_TEMPLATE_YML}.yml"
-
-rspec-foss-impact:trigger:
-  extends:
-    - .rails:rules:rspec-foss-impact
-  stage: test
-  needs:
-    - job: "setup-test-env"
-      artifacts: false
-    - job: "retrieve-tests-metadata"
-      artifacts: false
-    - job: "compile-test-assets as-if-foss"
-      artifacts: false
-    - job: "rspec-foss-impact:pipeline-generate"
-      artifacts: true
-  variables:
-    PARENT_PIPELINE_ID: $CI_PIPELINE_ID
-    GIT_STRATEGY: $GIT_STRATEGY
-  trigger:
-    strategy: depend
-    forward:
-      yaml_variables: true
-      pipeline_variables: true
-    include:
-      - artifact: "${RSPEC_FOSS_IMPACT_PIPELINE_TEMPLATE_YML}.yml"
-        job: rspec-foss-impact:pipeline-generate
-
 fail-pipeline-early:
   extends:
     - .rails:rules:fail-pipeline-early
diff --git a/.gitlab/ci/rails/rspec-foss-impact.gitlab-ci.yml.erb b/.gitlab/ci/rails/rspec-foss-impact.gitlab-ci.yml.erb
deleted file mode 100644
index e9c66a49f7298..0000000000000
--- a/.gitlab/ci/rails/rspec-foss-impact.gitlab-ci.yml.erb
+++ /dev/null
@@ -1,89 +0,0 @@
-# RSpec FOSS impact pipeline loaded dynamically by script: scripts/generate_rspec_pipeline.rb
-
-include:
-  - local: .gitlab/ci/rails/shared.gitlab-ci.yml
-
-default:
-  image: $DEFAULT_CI_IMAGE
-  tags:
-    - gitlab-org
-  # Default job timeout set to 90m https://gitlab.com/gitlab-com/gl-infra/infrastructure/-/issues/10520
-  timeout: 90m
-  interruptible: true
-
-stages:
-  - test
-
-dont-interrupt-me:
-  extends: .rules:dont-interrupt
-  stage: .pre
-  interruptible: false
-  script:
-    - echo "This jobs makes sure this pipeline won't be interrupted! See https://docs.gitlab.com/ee/ci/yaml/#interruptible."
-
-.base-rspec-foss-impact:
-  extends: .rspec-base-pg14-as-if-foss
-  needs:
-  <% if repo_from_artifacts %>
-    - pipeline: $PARENT_PIPELINE_ID
-      job: clone-gitlab-repo
-  <% end %>
-    - pipeline: $PARENT_PIPELINE_ID
-      job: detect-tests
-    - pipeline: $PARENT_PIPELINE_ID
-      job: setup-test-env
-    - pipeline: $PARENT_PIPELINE_ID
-      job: retrieve-tests-metadata
-    - pipeline: $PARENT_PIPELINE_ID
-      job: compile-test-assets as-if-foss
-  rules:
-    - when: always
-  variables:
-    RSPEC_TESTS_FILTER_FILE: "${RSPEC_MATCHING_TESTS_FOSS_PATH}"
-    RSPEC_TESTS_MAPPING_ENABLED: "true"
-  script:
-    - !reference [.base-script, script]
-    - rspec_parallelized_job "--fail-fast=${RSPEC_FAIL_FAST_THRESHOLD} --tag ~quarantine --tag ~level:background_migration --tag ~zoekt --tag ~click_house"
-
-<% if rspec_files_per_test_level[:migration][:files].size > 0 %>
-rspec migration foss-impact:
-  extends: .base-rspec-foss-impact
-<% if rspec_files_per_test_level[:migration][:parallelization] > 1 %>
-  parallel: <%= rspec_files_per_test_level[:migration][:parallelization] %>
-<% end %>
-  script:
-    - !reference [.base-script, script]
-    - rspec_parallelized_job "--fail-fast=${RSPEC_FAIL_FAST_THRESHOLD} --tag ~quarantine --tag ~zoekt --tag ~click_house"
-<% end %>
-
-<% if rspec_files_per_test_level[:background_migration][:files].size > 0 %>
-rspec background_migration foss-impact:
-  extends: .base-rspec-foss-impact
-<% if rspec_files_per_test_level[:background_migration][:parallelization] > 1 %>
-  parallel: <%= rspec_files_per_test_level[:background_migration][:parallelization] %>
-<% end %>
-<% end %>
-
-<% if rspec_files_per_test_level[:unit][:files].size > 0 %>
-rspec unit foss-impact:
-  extends: .base-rspec-foss-impact
-<% if rspec_files_per_test_level[:unit][:parallelization] > 1 %>
-  parallel: <%= rspec_files_per_test_level[:unit][:parallelization] %>
-<% end %>
-<% end %>
-
-<% if rspec_files_per_test_level[:integration][:files].size > 0 %>
-rspec integration foss-impact:
-  extends: .base-rspec-foss-impact
-<% if rspec_files_per_test_level[:integration][:parallelization] > 1 %>
-  parallel: <%= rspec_files_per_test_level[:integration][:parallelization] %>
-<% end %>
-<% end %>
-
-<% if rspec_files_per_test_level[:system][:files].size > 0 %>
-rspec system foss-impact:
-  extends: .base-rspec-foss-impact
-<% if rspec_files_per_test_level[:system][:parallelization] > 1 %>
-  parallel: <%= rspec_files_per_test_level[:system][:parallelization] %>
-<% end %>
-<% end %>
diff --git a/.gitlab/ci/rules.gitlab-ci.yml b/.gitlab/ci/rules.gitlab-ci.yml
index 715c485658c7d..557885950fe24 100644
--- a/.gitlab/ci/rules.gitlab-ci.yml
+++ b/.gitlab/ci/rules.gitlab-ci.yml
@@ -2056,6 +2056,7 @@
       changes: *code-backstage-patterns
     - <<: *if-merge-request-labels-pipeline-expedite
       when: never
+    - if: '$ENABLE_RSPEC_PREDICTIVE_PIPELINE_GENERATE == "true"'
     - <<: *if-merge-request
       changes: *db-patterns
     - <<: *if-merge-request
@@ -2085,6 +2086,7 @@
       changes: *code-backstage-patterns
     - <<: *if-merge-request-labels-pipeline-expedite
       when: never
+    - if: '$ENABLE_RSPEC_PREDICTIVE_TRIGGER == "true"'
     - <<: *if-merge-request-labels-run-all-rspec
       when: never
     - <<: *if-merge-request
@@ -2112,6 +2114,7 @@
       when: never
     - <<: *if-merge-request-labels-pipeline-expedite
       when: never
+    - if: '$ENABLE_RSPEC_PREDICTIVE_TRIGGER_SINGLE_DB == "true"'
     - <<: *if-merge-request-approved
       when: never
     - <<: *if-merge-request
@@ -2130,6 +2133,7 @@
       when: never
     - <<: *if-merge-request-labels-pipeline-expedite
       when: never
+    - if: '$ENABLE_RSPEC_PREDICTIVE_TRIGGER_SINGLE_DB_CI_CONNECTION == "true"'
     - <<: *if-merge-request-approved
       when: never
     - <<: *if-merge-request
@@ -2371,19 +2375,6 @@
   rules:
     - !reference [".rails:rules:previous-failed-tests-default-rules", rules]
 
-.rails:rules:rspec-foss-impact:
-  rules:
-    - <<: *if-not-ee
-      when: never
-    - <<: *if-merge-request-labels-as-if-foss
-      when: never
-    - <<: *if-merge-request-labels-pipeline-expedite
-      when: never
-    - <<: *if-security-merge-request
-      changes: *code-backstage-patterns
-    - <<: *if-dot-com-gitlab-org-merge-request
-      changes: *code-backstage-patterns
-
 .rails:rules:rspec fail-fast:
   rules:
     - <<: *if-not-ee
diff --git a/danger/specs/Dangerfile b/danger/specs/Dangerfile
index ff766cf688ee9..bfc8a4b234442 100644
--- a/danger/specs/Dangerfile
+++ b/danger/specs/Dangerfile
@@ -19,7 +19,7 @@ This could be a sign that you're testing an EE-specific behavior in a FOSS test.
 Please make sure the spec files pass in AS-IF-FOSS mode either:
 
 1. Locally with `FOSS_ONLY=1 bin/rspec -- %<spec_files>s`.
-1. In the MR pipeline by verifying that the `rspec foss-impact` job has passed.
+1. In the MR pipeline by verifying that the `rspec:predictive:trigger` job has passed in the as-if-foss cross project downstream pipeline.
 1. In the MR pipelines by setting the ~"pipeline:run-as-if-foss" label on the MR (you can do it with the `/label ~"pipeline:run-as-if-foss"` quick action) and start a new MR pipeline.
 
 MSG
diff --git a/ee/spec/policies/group_policy_spec.rb b/ee/spec/policies/group_policy_spec.rb
index f6529266b4cc3..240296f1a6f7f 100644
--- a/ee/spec/policies/group_policy_spec.rb
+++ b/ee/spec/policies/group_policy_spec.rb
@@ -14,7 +14,7 @@ def stub_group_saml_config(enabled)
 
   include_context 'GroupPolicy context'
   # Can't move to GroupPolicy context because auditor trait is not present
-  # outside of EE context and foss-impact will fail on this
+  # outside of EE context and FOSS will fail on this
   let_it_be(:auditor) { create(:user, :auditor) }
 
   let(:epic_rules) do
diff --git a/scripts/setup/generate-as-if-foss-env.rb b/scripts/setup/generate-as-if-foss-env.rb
index 3048b9967e818..5ec3fd8d77afa 100755
--- a/scripts/setup/generate-as-if-foss-env.rb
+++ b/scripts/setup/generate-as-if-foss-env.rb
@@ -6,6 +6,7 @@
 require 'set' # rubocop:disable Lint/RedundantRequireStatement -- Ruby 3.1 and earlier needs this. Drop this line after Ruby 3.2+ is only supported.
 
 class GenerateAsIfFossEnv
+  # rubocop:disable Style/WordArray -- Probably a bug? It already is
   FOSS_JOBS = Set.new(%w[
     build-assets-image
     build-qa-image
@@ -16,11 +17,16 @@ class GenerateAsIfFossEnv
     eslint
     generate-apollo-graphql-schema
     graphql-schema-dump
+    rspec-predictive:pipeline-generate
+    rspec:predictive:trigger
+    rspec:predictive:trigger\ single-db
+    rspec:predictive:trigger\ single-db-ci-connection
     rubocop
     qa:internal
     qa:selectors
     static-analysis
   ]).freeze
+  # rubocop:enable Style/WordArray
 
   def initialize
     @client = Gitlab.client(
@@ -62,8 +68,10 @@ def scan_jobs
   end
 
   def each_job
-    client.pipeline_jobs(ENV['CI_PROJECT_ID'], ENV['CI_PIPELINE_ID']).auto_paginate do |job|
-      yield(job)
+    %i[pipeline_jobs pipeline_bridges].each do |kind|
+      client.public_send(kind, ENV['CI_PROJECT_ID'], ENV['CI_PIPELINE_ID']).auto_paginate do |job| # rubocop:disable GitlabSecurity/PublicSend -- We're sending with static values, no concerns
+        yield(job)
+      end
     end
   end
 
diff --git a/spec/dot_gitlab_ci/rules_spec.rb b/spec/dot_gitlab_ci/rules_spec.rb
index 4ee3939593a6e..1076ddd9f4dc2 100644
--- a/spec/dot_gitlab_ci/rules_spec.rb
+++ b/spec/dot_gitlab_ci/rules_spec.rb
@@ -102,6 +102,9 @@
 
           if base['when'] == 'never'
             expect(derived).to eq(base.except('when'))
+          elsif base['if'].start_with?('$ENABLE_')
+            derived_if = base['if'].sub('RSPEC', 'RSPEC_PREDICTIVE_TRIGGER')
+            expect(derived).to eq(base.merge('if' => derived_if))
           elsif base['when'].nil?
             expect(derived).to eq(base.merge('when' => 'never'))
           end
@@ -119,7 +122,13 @@
     it_behaves_like 'predictive is inverse of non-predictive' do
       let(:base_rules) { config.dig('.rails:rules:ee-and-foss-default-rules', 'rules') }
 
-      let(:derived_rules) { config.dig('.rails:rules:rspec-predictive', 'rules') }
+      let(:derived_rules) do
+        config.dig('.rails:rules:rspec-predictive', 'rules').reject do |rule|
+          # This happens in each specific rspec,
+          # which is outside of .rails:rules:ee-and-foss-default-rules
+          rule['if'].start_with?('$ENABLE_')
+        end
+      end
 
       it 'contains an additional allow rule about code-backstage-patterns not present in the base' do
         expected_rule = {
@@ -135,24 +144,14 @@
 
   describe '.rails:rules:single-db' do
     it_behaves_like 'predictive is inverse of non-predictive' do
-      let(:base_rules) do
-        config.dig('.rails:rules:single-db', 'rules').reject do |rule|
-          rule['if'].start_with?('$ENABLE_') # We don't handle this yet
-        end
-      end
-
+      let(:base_rules) { config.dig('.rails:rules:single-db', 'rules') }
       let(:derived_rules) { config.dig('.rails:rules:rspec-predictive:single-db', 'rules') }
     end
   end
 
   describe '.rails:rules:single-db-ci-connection' do
     it_behaves_like 'predictive is inverse of non-predictive' do
-      let(:base_rules) do
-        config.dig('.rails:rules:single-db-ci-connection', 'rules').reject do |rule|
-          rule['if'].start_with?('$ENABLE_') # We don't handle this yet
-        end
-      end
-
+      let(:base_rules) { config.dig('.rails:rules:single-db-ci-connection', 'rules') }
       let(:derived_rules) { config.dig('.rails:rules:rspec-predictive:single-db-ci-connection', 'rules') }
     end
   end
diff --git a/spec/scripts/setup/generate_as_if_foss_env_spec.rb b/spec/scripts/setup/generate_as_if_foss_env_spec.rb
index 768cd754cd16a..14212424d888e 100644
--- a/spec/scripts/setup/generate_as_if_foss_env_spec.rb
+++ b/spec/scripts/setup/generate_as_if_foss_env_spec.rb
@@ -53,15 +53,32 @@
       ]
     end
 
+    let(:bridges) do
+      [
+        'rspec-predictive:pipeline-generate',
+        'rspec:predictive:trigger',
+        'rspec:predictive:trigger single-db',
+        'rspec:predictive:trigger single-db-ci-connection'
+      ]
+    end
+
+    # rubocop:disable RSpec/VerifiedDoubles -- As explained at the top of this file, we do not load the Gitlab client
     before do
-      messages = receive_message_chain(:client, :pipeline_jobs, :auto_paginate)
+      client = double
+      allow(Gitlab).to receive(:client).and_return(client)
 
-      yield_jobs = jobs.inject(messages) do |stub, job|
-        stub.and_yield(double(name: job)) # rubocop:disable RSpec/VerifiedDoubles -- As explained at the top of this file, we do not load the Gitlab client
-      end
+      allow(client).to yield_jobs(:pipeline_jobs, jobs)
+      allow(client).to yield_jobs(:pipeline_bridges, bridges)
+    end
 
-      allow(Gitlab).to yield_jobs
+    def yield_jobs(api_method, jobs)
+      messages = receive_message_chain(api_method, :auto_paginate)
+
+      jobs.inject(messages) do |stub, job_name|
+        stub.and_yield(double(name: job_name))
+      end
     end
+    # rubocop:enable RSpec/VerifiedDoubles
   end
 
   describe '#variables' do
@@ -100,7 +117,11 @@
         ENABLE_RUBOCOP: 'true',
         ENABLE_QA_INTERNAL: 'true',
         ENABLE_QA_SELECTORS: 'true',
-        ENABLE_STATIC_ANALYSIS: 'true'
+        ENABLE_STATIC_ANALYSIS: 'true',
+        ENABLE_RSPEC_PREDICTIVE_PIPELINE_GENERATE: 'true',
+        ENABLE_RSPEC_PREDICTIVE_TRIGGER: 'true',
+        ENABLE_RSPEC_PREDICTIVE_TRIGGER_SINGLE_DB: 'true',
+        ENABLE_RSPEC_PREDICTIVE_TRIGGER_SINGLE_DB_CI_CONNECTION: 'true'
       })
     end
   end
@@ -142,6 +163,10 @@
         ENABLE_QA_INTERNAL=true
         ENABLE_QA_SELECTORS=true
         ENABLE_STATIC_ANALYSIS=true
+        ENABLE_RSPEC_PREDICTIVE_PIPELINE_GENERATE=true
+        ENABLE_RSPEC_PREDICTIVE_TRIGGER=true
+        ENABLE_RSPEC_PREDICTIVE_TRIGGER_SINGLE_DB=true
+        ENABLE_RSPEC_PREDICTIVE_TRIGGER_SINGLE_DB_CI_CONNECTION=true
       ENV
     end
   end
-- 
GitLab