diff --git a/app/models/ci/bridge.rb b/app/models/ci/bridge.rb
index 8db80cd05dc0c6953b4d31a48ada86a5a3bd21bd..d62bdfa4c8701f93236d6e0f9b68f06a1650a7af 100644
--- a/app/models/ci/bridge.rb
+++ b/app/models/ci/bridge.rb
@@ -32,10 +32,8 @@ class Bridge < Ci::Processable
state_machine :status do
after_transition [:created, :manual, :waiting_for_resource] => :pending do |bridge|
- next unless bridge.triggers_downstream_pipeline?
-
bridge.run_after_commit do
- ::Ci::CreateDownstreamPipelineWorker.perform_async(bridge.id)
+ Ci::TriggerDownstreamPipelineService.new(bridge).execute # rubocop: disable CodeReuse/ServiceClass
end
end
diff --git a/app/models/concerns/enums/ci/commit_status.rb b/app/models/concerns/enums/ci/commit_status.rb
index 9de2da5aac3cb136a3b70a73d0a02c63153c5bc8..b6eabd46de3d0758d29081286bf4cd2227ebd940 100644
--- a/app/models/concerns/enums/ci/commit_status.rb
+++ b/app/models/concerns/enums/ci/commit_status.rb
@@ -41,7 +41,8 @@ def self.failure_reasons
secrets_provider_not_found: 1_008,
reached_max_descendant_pipelines_depth: 1_009,
ip_restriction_failure: 1_010,
- reached_max_pipeline_hierarchy_size: 1_011
+ reached_max_pipeline_hierarchy_size: 1_011,
+ reached_downstream_pipeline_trigger_rate_limit: 1_012
}
end
end
diff --git a/app/presenters/commit_status_presenter.rb b/app/presenters/commit_status_presenter.rb
index 38469be572ac93b489e486db05a85e00a7aec6d6..28656b0ccc4b875707b466b904baa4d4ea175ae2 100644
--- a/app/presenters/commit_status_presenter.rb
+++ b/app/presenters/commit_status_presenter.rb
@@ -37,7 +37,8 @@ class CommitStatusPresenter < Gitlab::View::Presenter::Delegated
environment_creation_failure: 'This job could not be executed because it would create an environment with an invalid parameter.',
deployment_rejected: 'This deployment job was rejected.',
ip_restriction_failure: "This job could not be executed because group IP address restrictions are enabled, and the runner's IP address is not in the allowed range.",
- failed_outdated_deployment_job: 'The deployment job is older than the latest deployment, and therefore failed.'
+ failed_outdated_deployment_job: 'The deployment job is older than the latest deployment, and therefore failed.',
+ reached_downstream_pipeline_trigger_rate_limit: 'Too many downstream pipelines triggered in the last minute. Try again later.'
}.freeze
TROUBLESHOOTING_DOC = {
diff --git a/app/services/ci/trigger_downstream_pipeline_service.rb b/app/services/ci/trigger_downstream_pipeline_service.rb
new file mode 100644
index 0000000000000000000000000000000000000000..87f1c075f0ef4947de4f35f646598b0d0b31b1a6
--- /dev/null
+++ b/app/services/ci/trigger_downstream_pipeline_service.rb
@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Ci
+ # Enqueues the downstream pipeline worker.
+ class TriggerDownstreamPipelineService
+ # This is a temporary constant. It may be converted into an application setting
+ # in the future. See https://gitlab.com/gitlab-org/gitlab/-/issues/425941.
+ DOWNSTREAM_PIPELINE_TRIGGER_LIMIT_PER_PROJECT_USER_SHA = 50
+
+ def initialize(bridge)
+ @bridge = bridge
+ @current_user = bridge.user
+ @project = bridge.project
+ @pipeline = bridge.pipeline
+ end
+
+ def execute
+ unless bridge.triggers_downstream_pipeline?
+ return ServiceResponse.success(message: 'Does not trigger a downstream pipeline')
+ end
+
+ if rate_limit_throttled? && enforce_rate_limit?
+ bridge.drop!(:reached_downstream_pipeline_trigger_rate_limit)
+
+ return ServiceResponse.error(message: 'Reached downstream pipeline trigger rate limit')
+ end
+
+ CreateDownstreamPipelineWorker.perform_async(bridge.id)
+
+ ServiceResponse.success(message: 'Downstream pipeline enqueued')
+ end
+
+ private
+
+ attr_reader :bridge, :current_user, :project, :pipeline
+
+ def rate_limit_throttled?
+ scope = [project, current_user, pipeline.sha]
+
+ ::Gitlab::ApplicationRateLimiter.throttled?(:downstream_pipeline_trigger, scope: scope).tap do |throttled|
+ create_throttled_log_entry if throttled
+ end
+ end
+
+ def create_throttled_log_entry
+ ::Gitlab::AppJsonLogger.info(
+ class: self.class.name,
+ project_id: project.id,
+ current_user_id: current_user.id,
+ pipeline_sha: pipeline.sha,
+ subscription_plan: project.actual_plan_name,
+ downstream_type: bridge.triggers_child_pipeline? ? 'child' : 'multi-project',
+ message: 'Activated downstream pipeline trigger rate limit'
+ )
+ end
+
+ def enforce_rate_limit?
+ ::Feature.enabled?(:ci_rate_limit_downstream_pipelines, project, type: :gitlab_com_derisk)
+ end
+ end
+end
diff --git a/config/feature_flags/gitlab_com_derisk/ci_rate_limit_downstream_pipelines.yml b/config/feature_flags/gitlab_com_derisk/ci_rate_limit_downstream_pipelines.yml
new file mode 100644
index 0000000000000000000000000000000000000000..8f4fa323f7be72353495d64473dd54a5aef5039f
--- /dev/null
+++ b/config/feature_flags/gitlab_com_derisk/ci_rate_limit_downstream_pipelines.yml
@@ -0,0 +1,9 @@
+---
+name: ci_rate_limit_downstream_pipelines
+feature_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/425941
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/142869
+rollout_issue_url: https://gitlab.com/gitlab-com/gl-infra/production/-/issues/17471
+milestone: '16.9'
+group: group::pipeline authoring
+type: gitlab_com_derisk
+default_enabled: false
diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md
index e961f86bc6cc66327d47aa9a3a7e47915489a8ef..d7c2a8da78286415c810dbd3d9878af4da0970d4 100644
--- a/doc/api/graphql/reference/index.md
+++ b/doc/api/graphql/reference/index.md
@@ -29835,6 +29835,7 @@ Values for sorting inherited variables.
| <a id="cijobfailurereasonpipeline_loop_detected"></a>`PIPELINE_LOOP_DETECTED` | A job that failed due to pipeline loop detected. |
| <a id="cijobfailurereasonproject_deleted"></a>`PROJECT_DELETED` | A job that failed due to project deleted. |
| <a id="cijobfailurereasonprotected_environment_failure"></a>`PROTECTED_ENVIRONMENT_FAILURE` | A job that failed due to protected environment failure. |
+| <a id="cijobfailurereasonreached_downstream_pipeline_trigger_rate_limit"></a>`REACHED_DOWNSTREAM_PIPELINE_TRIGGER_RATE_LIMIT` | A job that failed due to reached downstream pipeline trigger rate limit. |
| <a id="cijobfailurereasonreached_max_descendant_pipelines_depth"></a>`REACHED_MAX_DESCENDANT_PIPELINES_DEPTH` | A job that failed due to reached max descendant pipelines depth. |
| <a id="cijobfailurereasonreached_max_pipeline_hierarchy_size"></a>`REACHED_MAX_PIPELINE_HIERARCHY_SIZE` | A job that failed due to reached max pipeline hierarchy size. |
| <a id="cijobfailurereasonrunner_system_failure"></a>`RUNNER_SYSTEM_FAILURE` | A job that failed due to runner system failure. |
diff --git a/lib/gitlab/application_rate_limiter.rb b/lib/gitlab/application_rate_limiter.rb
index 5a2881e6c966362301db8af38564b944afda2171..2e992e38a44c70f9896174f2415f400ee6215f31 100644
--- a/lib/gitlab/application_rate_limiter.rb
+++ b/lib/gitlab/application_rate_limiter.rb
@@ -65,6 +65,9 @@ def rate_limits # rubocop:disable Metrics/AbcSize
bulk_import: { threshold: 6, interval: 1.minute },
projects_api_rate_limit_unauthenticated: {
threshold: -> { application_settings.projects_api_rate_limit_unauthenticated }, interval: 10.minutes
+ },
+ downstream_pipeline_trigger: {
+ threshold: -> { ::Ci::TriggerDownstreamPipelineService::DOWNSTREAM_PIPELINE_TRIGGER_LIMIT_PER_PROJECT_USER_SHA }, interval: 1.minute
}
}.freeze
end
diff --git a/lib/gitlab/ci/status/build/failed.rb b/lib/gitlab/ci/status/build/failed.rb
index a136044c124995127deb6ca8c97f9cae6c43efb2..caaa4139f38edef9e584524c22d512079ba60a90 100644
--- a/lib/gitlab/ci/status/build/failed.rb
+++ b/lib/gitlab/ci/status/build/failed.rb
@@ -5,6 +5,7 @@ module Ci
module Status
module Build
class Failed < Status::Extended
+ # rubocop: disable Layout/LineLength -- Long error messages
REASONS = {
unknown_failure: 'unknown failure',
script_failure: 'script failure',
@@ -41,8 +42,10 @@ class Failed < Status::Extended
environment_creation_failure: 'environment creation failure',
deployment_rejected: 'deployment rejected',
ip_restriction_failure: 'IP address restriction failure',
- failed_outdated_deployment_job: 'failed outdated deployment job'
+ failed_outdated_deployment_job: 'failed outdated deployment job',
+ reached_downstream_pipeline_trigger_rate_limit: 'Too many downstream pipelines triggered in the last minute. Try again later.'
}.freeze
+ # rubocop: enable Layout/LineLength
private_constant :REASONS
diff --git a/spec/services/ci/trigger_downstream_pipeline_service_spec.rb b/spec/services/ci/trigger_downstream_pipeline_service_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..71d6931658925faa279856864c9bdf5c9edb03d9
--- /dev/null
+++ b/spec/services/ci/trigger_downstream_pipeline_service_spec.rb
@@ -0,0 +1,122 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Ci::TriggerDownstreamPipelineService, feature_category: :continuous_integration do
+ let_it_be(:project) { create(:project, :repository) }
+ let_it_be(:user) { project.first_owner }
+ let_it_be(:pipeline) { create(:ci_pipeline, project: project, user: user) }
+
+ let(:bridge) do
+ create(:ci_bridge, status: :created, options: bridge_options, pipeline: pipeline, user: user)
+ end
+
+ let(:bridge_options) { { trigger: { project: 'my/project' } } }
+ let(:service) { described_class.new(bridge) }
+
+ describe '#execute' do
+ subject(:execute) { service.execute }
+
+ context 'when the bridge does not trigger a downstream pipeline' do
+ let(:bridge_options) { { trigger: {} } }
+
+ it 'returns a success response' do
+ expect(execute).to be_success
+ expect(execute.message).to eq('Does not trigger a downstream pipeline')
+ end
+ end
+
+ # In these tests, we execute the service twice in succession
+ describe 'rate limiting', :freeze_time, :clean_gitlab_redis_rate_limiting do
+ shared_examples 'creates a log entry' do |downstream_type = 'multi-project'|
+ it do
+ service.execute
+
+ expect(Gitlab::AppJsonLogger).to receive(:info).with(
+ a_hash_including(
+ class: described_class.name,
+ project_id: project.id,
+ current_user_id: user.id,
+ pipeline_sha: pipeline.sha,
+ subscription_plan: project.actual_plan_name,
+ downstream_type: downstream_type,
+ message: 'Activated downstream pipeline trigger rate limit'
+ )
+ )
+
+ execute
+ end
+ end
+
+ context 'when the limit is exceeded' do
+ before do
+ stub_const("#{described_class.name}::DOWNSTREAM_PIPELINE_TRIGGER_LIMIT_PER_PROJECT_USER_SHA", 1)
+ end
+
+ it 'drops the bridge and does not schedule the downstream pipeline worker', :aggregate_failures do
+ service.execute
+
+ expect { execute }.not_to change { ::Ci::CreateDownstreamPipelineWorker.jobs.size }
+ expect(bridge).to be_failed
+ expect(bridge.failure_reason).to eq('reached_downstream_pipeline_trigger_rate_limit')
+ expect(execute).to be_error
+ expect(execute.message).to eq('Reached downstream pipeline trigger rate limit')
+ end
+
+ it_behaves_like 'creates a log entry'
+
+ context 'with a child pipeline' do
+ let(:bridge_options) { { trigger: { include: 'my_child_config.yml' } } }
+
+ it 'drops the bridge and does not schedule the downstream pipeline worker', :aggregate_failures do
+ service.execute
+
+ expect { execute }.not_to change { ::Ci::CreateDownstreamPipelineWorker.jobs.size }
+ expect(bridge).to be_failed
+ expect(bridge.failure_reason).to eq('reached_downstream_pipeline_trigger_rate_limit')
+ expect(execute).to be_error
+ expect(execute.message).to eq('Reached downstream pipeline trigger rate limit')
+ end
+
+ it_behaves_like 'creates a log entry', 'child'
+ end
+
+ context 'when FF `ci_rate_limit_downstream_pipelines` is disabled' do
+ before do
+ stub_feature_flags(ci_rate_limit_downstream_pipelines: false)
+ end
+
+ it 'schedules the downstream pipeline worker' do
+ service.execute
+
+ expect { execute }.to change { ::Ci::CreateDownstreamPipelineWorker.jobs.size }.by(1)
+ expect(bridge).not_to be_failed
+ expect(execute).to be_success
+ expect(execute.message).to eq('Downstream pipeline enqueued')
+ end
+
+ it_behaves_like 'creates a log entry'
+ end
+ end
+
+ context 'when the limit is not exceeded' do
+ it 'schedules the downstream pipeline worker' do
+ service.execute
+
+ expect { execute }.to change { ::Ci::CreateDownstreamPipelineWorker.jobs.size }.by(1)
+ expect(bridge).not_to be_failed
+ expect(execute).to be_success
+ expect(execute.message).to eq('Downstream pipeline enqueued')
+ end
+
+ it 'does not create a log entry' do
+ service.execute
+
+ expect(Gitlab::AppJsonLogger).not_to receive(:info)
+
+ execute
+ end
+ end
+ end
+ end
+end