Merge branch 'add-downstream-pipeline-rate-limit' into 'master'

Introduce downstream pipeline trigger rate limit

See merge request https://gitlab.com/gitlab-org/gitlab/-/merge_requests/142869



Merged-by: drew stachon <730684-drew@users.noreply.gitlab.com>
Approved-by: Furkan Ayhan <furkanayhn@gmail.com>
Approved-by: drew stachon <730684-drew@users.noreply.gitlab.com>
Reviewed-by: Furkan Ayhan <furkanayhn@gmail.com>
Co-authored-by: lma-git <lma@gitlab.com>

app/models/ci/bridge.rb

@@ -32,10 +32,8 @@ class Bridge < Ci::Processable
 
     state_machine :status do
       after_transition [:created, :manual, :waiting_for_resource] => :pending do |bridge|
-        next unless bridge.triggers_downstream_pipeline?
-
         bridge.run_after_commit do
-          ::Ci::CreateDownstreamPipelineWorker.perform_async(bridge.id)
+          Ci::TriggerDownstreamPipelineService.new(bridge).execute # rubocop: disable CodeReuse/ServiceClass
         end
       end
 

app/models/concerns/enums/ci/commit_status.rb

@@ -41,7 +41,8 @@ def self.failure_reasons
           secrets_provider_not_found: 1_008,
           reached_max_descendant_pipelines_depth: 1_009,
           ip_restriction_failure: 1_010,
-          reached_max_pipeline_hierarchy_size: 1_011
+          reached_max_pipeline_hierarchy_size: 1_011,
+          reached_downstream_pipeline_trigger_rate_limit: 1_012
         }
       end
     end

app/presenters/commit_status_presenter.rb

@@ -37,7 +37,8 @@ class CommitStatusPresenter < Gitlab::View::Presenter::Delegated
     environment_creation_failure: 'This job could not be executed because it would create an environment with an invalid parameter.',
     deployment_rejected: 'This deployment job was rejected.',
     ip_restriction_failure: "This job could not be executed because group IP address restrictions are enabled, and the runner's IP address is not in the allowed range.",
-    failed_outdated_deployment_job: 'The deployment job is older than the latest deployment, and therefore failed.'
+    failed_outdated_deployment_job: 'The deployment job is older than the latest deployment, and therefore failed.',
+    reached_downstream_pipeline_trigger_rate_limit: 'Too many downstream pipelines triggered in the last minute. Try again later.'
   }.freeze
 
   TROUBLESHOOTING_DOC = {

app/services/ci/trigger_downstream_pipeline_service.rb

+# frozen_string_literal: true
+
+module Ci
+  # Enqueues the downstream pipeline worker.
+  class TriggerDownstreamPipelineService
+    # This is a temporary constant. It may be converted into an application setting
+    # in the future. See https://gitlab.com/gitlab-org/gitlab/-/issues/425941.
+    DOWNSTREAM_PIPELINE_TRIGGER_LIMIT_PER_PROJECT_USER_SHA = 50
+
+    def initialize(bridge)
+      @bridge = bridge
+      @current_user = bridge.user
+      @project = bridge.project
+      @pipeline = bridge.pipeline
+    end
+
+    def execute
+      unless bridge.triggers_downstream_pipeline?
+        return ServiceResponse.success(message: 'Does not trigger a downstream pipeline')
+      end
+
+      if rate_limit_throttled? && enforce_rate_limit?
+        bridge.drop!(:reached_downstream_pipeline_trigger_rate_limit)
+
+        return ServiceResponse.error(message: 'Reached downstream pipeline trigger rate limit')
+      end
+
+      CreateDownstreamPipelineWorker.perform_async(bridge.id)
+
+      ServiceResponse.success(message: 'Downstream pipeline enqueued')
+    end
+
+    private
+
+    attr_reader :bridge, :current_user, :project, :pipeline
+
+    def rate_limit_throttled?
+      scope = [project, current_user, pipeline.sha]
+
+      ::Gitlab::ApplicationRateLimiter.throttled?(:downstream_pipeline_trigger, scope: scope).tap do |throttled|
+        create_throttled_log_entry if throttled
+      end
+    end
+
+    def create_throttled_log_entry
+      ::Gitlab::AppJsonLogger.info(
+        class: self.class.name,
+        project_id: project.id,
+        current_user_id: current_user.id,
+        pipeline_sha: pipeline.sha,
+        subscription_plan: project.actual_plan_name,
+        downstream_type: bridge.triggers_child_pipeline? ? 'child' : 'multi-project',
+        message: 'Activated downstream pipeline trigger rate limit'
+      )
+    end
+
+    def enforce_rate_limit?
+      ::Feature.enabled?(:ci_rate_limit_downstream_pipelines, project, type: :gitlab_com_derisk)
+    end
+  end
+end

config/feature_flags/gitlab_com_derisk/ci_rate_limit_downstream_pipelines.yml

+---
+name: ci_rate_limit_downstream_pipelines
+feature_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/425941
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/142869
+rollout_issue_url: https://gitlab.com/gitlab-com/gl-infra/production/-/issues/17471
+milestone: '16.9'
+group: group::pipeline authoring
+type: gitlab_com_derisk
+default_enabled: false

doc/api/graphql/reference/index.md

@@ -29835,6 +29835,7 @@ Values for sorting inherited variables.
 | <a id="cijobfailurereasonpipeline_loop_detected"></a>`PIPELINE_LOOP_DETECTED` | A job that failed due to pipeline loop detected. |
 | <a id="cijobfailurereasonproject_deleted"></a>`PROJECT_DELETED` | A job that failed due to project deleted. |
 | <a id="cijobfailurereasonprotected_environment_failure"></a>`PROTECTED_ENVIRONMENT_FAILURE` | A job that failed due to protected environment failure. |
+| <a id="cijobfailurereasonreached_downstream_pipeline_trigger_rate_limit"></a>`REACHED_DOWNSTREAM_PIPELINE_TRIGGER_RATE_LIMIT` | A job that failed due to reached downstream pipeline trigger rate limit. |
 | <a id="cijobfailurereasonreached_max_descendant_pipelines_depth"></a>`REACHED_MAX_DESCENDANT_PIPELINES_DEPTH` | A job that failed due to reached max descendant pipelines depth. |
 | <a id="cijobfailurereasonreached_max_pipeline_hierarchy_size"></a>`REACHED_MAX_PIPELINE_HIERARCHY_SIZE` | A job that failed due to reached max pipeline hierarchy size. |
 | <a id="cijobfailurereasonrunner_system_failure"></a>`RUNNER_SYSTEM_FAILURE` | A job that failed due to runner system failure. |

lib/gitlab/application_rate_limiter.rb

@@ -65,6 +65,9 @@ def rate_limits # rubocop:disable Metrics/AbcSize
           bulk_import: { threshold: 6, interval: 1.minute },
           projects_api_rate_limit_unauthenticated: {
             threshold: -> { application_settings.projects_api_rate_limit_unauthenticated }, interval: 10.minutes
+          },
+          downstream_pipeline_trigger: {
+            threshold: -> { ::Ci::TriggerDownstreamPipelineService::DOWNSTREAM_PIPELINE_TRIGGER_LIMIT_PER_PROJECT_USER_SHA }, interval: 1.minute
           }
         }.freeze
       end

lib/gitlab/ci/status/build/failed.rb

@@ -5,6 +5,7 @@ module Ci
     module Status
       module Build
         class Failed < Status::Extended
+          # rubocop: disable Layout/LineLength -- Long error messages
           REASONS = {
             unknown_failure: 'unknown failure',
             script_failure: 'script failure',
@@ -41,8 +42,10 @@ class Failed < Status::Extended
             environment_creation_failure: 'environment creation failure',
             deployment_rejected: 'deployment rejected',
             ip_restriction_failure: 'IP address restriction failure',
-            failed_outdated_deployment_job: 'failed outdated deployment job'
+            failed_outdated_deployment_job: 'failed outdated deployment job',
+            reached_downstream_pipeline_trigger_rate_limit: 'Too many downstream pipelines triggered in the last minute. Try again later.'
           }.freeze
+          # rubocop: enable Layout/LineLength
 
           private_constant :REASONS
 

spec/services/ci/trigger_downstream_pipeline_service_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Ci::TriggerDownstreamPipelineService, feature_category: :continuous_integration do
+  let_it_be(:project) { create(:project, :repository) }
+  let_it_be(:user) { project.first_owner }
+  let_it_be(:pipeline) { create(:ci_pipeline, project: project, user: user) }
+
+  let(:bridge) do
+    create(:ci_bridge, status: :created, options: bridge_options, pipeline: pipeline, user: user)
+  end
+
+  let(:bridge_options) { { trigger: { project: 'my/project' } } }
+  let(:service) { described_class.new(bridge) }
+
+  describe '#execute' do
+    subject(:execute) { service.execute }
+
+    context 'when the bridge does not trigger a downstream pipeline' do
+      let(:bridge_options) { { trigger: {} } }
+
+      it 'returns a success response' do
+        expect(execute).to be_success
+        expect(execute.message).to eq('Does not trigger a downstream pipeline')
+      end
+    end
+
+    # In these tests, we execute the service twice in succession
+    describe 'rate limiting', :freeze_time, :clean_gitlab_redis_rate_limiting do
+      shared_examples 'creates a log entry' do |downstream_type = 'multi-project'|
+        it do
+          service.execute
+
+          expect(Gitlab::AppJsonLogger).to receive(:info).with(
+            a_hash_including(
+              class: described_class.name,
+              project_id: project.id,
+              current_user_id: user.id,
+              pipeline_sha: pipeline.sha,
+              subscription_plan: project.actual_plan_name,
+              downstream_type: downstream_type,
+              message: 'Activated downstream pipeline trigger rate limit'
+            )
+          )
+
+          execute
+        end
+      end
+
+      context 'when the limit is exceeded' do
+        before do
+          stub_const("#{described_class.name}::DOWNSTREAM_PIPELINE_TRIGGER_LIMIT_PER_PROJECT_USER_SHA", 1)
+        end
+
+        it 'drops the bridge and does not schedule the downstream pipeline worker', :aggregate_failures do
+          service.execute
+
+          expect { execute }.not_to change { ::Ci::CreateDownstreamPipelineWorker.jobs.size }
+          expect(bridge).to be_failed
+          expect(bridge.failure_reason).to eq('reached_downstream_pipeline_trigger_rate_limit')
+          expect(execute).to be_error
+          expect(execute.message).to eq('Reached downstream pipeline trigger rate limit')
+        end
+
+        it_behaves_like 'creates a log entry'
+
+        context 'with a child pipeline' do
+          let(:bridge_options) { { trigger: { include: 'my_child_config.yml' } } }
+
+          it 'drops the bridge and does not schedule the downstream pipeline worker', :aggregate_failures do
+            service.execute
+
+            expect { execute }.not_to change { ::Ci::CreateDownstreamPipelineWorker.jobs.size }
+            expect(bridge).to be_failed
+            expect(bridge.failure_reason).to eq('reached_downstream_pipeline_trigger_rate_limit')
+            expect(execute).to be_error
+            expect(execute.message).to eq('Reached downstream pipeline trigger rate limit')
+          end
+
+          it_behaves_like 'creates a log entry', 'child'
+        end
+
+        context 'when FF `ci_rate_limit_downstream_pipelines` is disabled' do
+          before do
+            stub_feature_flags(ci_rate_limit_downstream_pipelines: false)
+          end
+
+          it 'schedules the downstream pipeline worker' do
+            service.execute
+
+            expect { execute }.to change { ::Ci::CreateDownstreamPipelineWorker.jobs.size }.by(1)
+            expect(bridge).not_to be_failed
+            expect(execute).to be_success
+            expect(execute.message).to eq('Downstream pipeline enqueued')
+          end
+
+          it_behaves_like 'creates a log entry'
+        end
+      end
+
+      context 'when the limit is not exceeded' do
+        it 'schedules the downstream pipeline worker' do
+          service.execute
+
+          expect { execute }.to change { ::Ci::CreateDownstreamPipelineWorker.jobs.size }.by(1)
+          expect(bridge).not_to be_failed
+          expect(execute).to be_success
+          expect(execute.message).to eq('Downstream pipeline enqueued')
+        end
+
+        it 'does not create a log entry' do
+          service.execute
+
+          expect(Gitlab::AppJsonLogger).not_to receive(:info)
+
+          execute
+        end
+      end
+    end
+  end
+end