diff --git a/app/services/ci/cancel_pipeline_service.rb b/app/services/ci/cancel_pipeline_service.rb
index 92eead3fdd14c46dfa576abc4406e4621b7af3be..68e893c8a6dfd4593216eb24846a854334a683a6 100644
--- a/app/services/ci/cancel_pipeline_service.rb
+++ b/app/services/ci/cancel_pipeline_service.rb
@@ -43,8 +43,7 @@ def force_execute
end
log_pipeline_being_canceled
-
- pipeline.update_column(:auto_canceled_by_id, @auto_canceled_by_pipeline.id) if @auto_canceled_by_pipeline
+ update_auto_canceled_pipeline_attributes
if @safe_cancellation
# Only build and bridge (trigger) jobs can be interruptible.
@@ -61,7 +60,7 @@ def force_execute
private
- attr_reader :pipeline, :current_user
+ attr_reader :pipeline, :current_user, :auto_canceled_by_pipeline
def log_pipeline_being_canceled
Gitlab::AppJsonLogger.info(
@@ -74,6 +73,15 @@ def log_pipeline_being_canceled
)
end
+ def update_auto_canceled_pipeline_attributes
+ return unless auto_canceled_by_pipeline
+
+ pipeline.update_columns(
+ auto_canceled_by_id: auto_canceled_by_pipeline.id,
+ auto_canceled_by_partition_id: auto_canceled_by_pipeline.partition_id
+ )
+ end
+
def cascade_to_children?
@cascade_to_children
end
diff --git a/db/docs/batched_background_migrations/queue_backfill_autocancel_partition_id_on_ci_pipelines.yml b/db/docs/batched_background_migrations/queue_backfill_autocancel_partition_id_on_ci_pipelines.yml
new file mode 100644
index 0000000000000000000000000000000000000000..49e97b5d0263e39ce73d090f2a6222d8383b3317
--- /dev/null
+++ b/db/docs/batched_background_migrations/queue_backfill_autocancel_partition_id_on_ci_pipelines.yml
@@ -0,0 +1,10 @@
+---
+migration_job_name: QueueBackfillAutocancelPartitionIdOnCiPipelines
+description: It backfills the autocancel_by_partition_id to prepare for composite primary key on ci_pipelines
+feature_category: ci_scaling
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/157619
+milestone: '17.2'
+queued_migration_version: 20240627122810
+# Replace with the approximate date you think it's best to ensure the completion of this BBM.
+finalize_after: '2024-07-22'
+finalized_by: # version of the migration that finalized this BBM
diff --git a/db/migrate/20240627121919_add_auto_canceled_by_partition_id_to_ci_pipelines.rb b/db/migrate/20240627121919_add_auto_canceled_by_partition_id_to_ci_pipelines.rb
new file mode 100644
index 0000000000000000000000000000000000000000..768ed463cc45b974a39f09960dbc6aa6e1fe79d2
--- /dev/null
+++ b/db/migrate/20240627121919_add_auto_canceled_by_partition_id_to_ci_pipelines.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class AddAutoCanceledByPartitionIdToCiPipelines < Gitlab::Database::Migration[2.2]
+ milestone '17.2'
+
+ def change
+ add_column :ci_pipelines, :auto_canceled_by_partition_id, :bigint
+ end
+end
diff --git a/db/post_migrate/20240627122810_queue_queue_backfill_autocancel_partition_id_on_ci_pipelines.rb b/db/post_migrate/20240627122810_queue_queue_backfill_autocancel_partition_id_on_ci_pipelines.rb
new file mode 100644
index 0000000000000000000000000000000000000000..9e707400eec97c4ab59771f73c6f18e779f4f594
--- /dev/null
+++ b/db/post_migrate/20240627122810_queue_queue_backfill_autocancel_partition_id_on_ci_pipelines.rb
@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+class QueueQueueBackfillAutocancelPartitionIdOnCiPipelines < Gitlab::Database::Migration[2.2]
+ milestone '17.2'
+ restrict_gitlab_migration gitlab_schema: :gitlab_ci
+
+ MIGRATION = "QueueBackfillAutocancelPartitionIdOnCiPipelines"
+ DELAY_INTERVAL = 2.minutes
+ BATCH_SIZE = 5000
+ SUB_BATCH_SIZE = 250
+ GITLAB_OPTIMIZED_BATCH_SIZE = 75_000
+ GITLAB_OPTIMIZED_SUB_BATCH_SIZE = 750
+
+ def up
+ queue_batched_background_migration(
+ MIGRATION,
+ :ci_pipelines,
+ :id,
+ job_interval: DELAY_INTERVAL,
+ **batch_sizes
+ )
+ end
+
+ def down
+ delete_batched_background_migration(MIGRATION, :ci_pipelines, :id, [])
+ end
+
+ private
+
+ def batch_sizes
+ if Gitlab.com_except_jh?
+ {
+ batch_size: GITLAB_OPTIMIZED_BATCH_SIZE,
+ sub_batch_size: GITLAB_OPTIMIZED_SUB_BATCH_SIZE
+ }
+ else
+ {
+ batch_size: BATCH_SIZE,
+ sub_batch_size: SUB_BATCH_SIZE
+ }
+ end
+ end
+end
diff --git a/db/schema_migrations/20240627121919 b/db/schema_migrations/20240627121919
new file mode 100644
index 0000000000000000000000000000000000000000..439705b35319260aa659a53f1629527d255ccac4
--- /dev/null
+++ b/db/schema_migrations/20240627121919
@@ -0,0 +1 @@
+80de104335055d247a08dcf0094fdaf85b573cd5161e572429141dea7b353720
\ No newline at end of file
diff --git a/db/schema_migrations/20240627122810 b/db/schema_migrations/20240627122810
new file mode 100644
index 0000000000000000000000000000000000000000..a74ae76915fe3ad34d59bbd535629cacc1225fd3
--- /dev/null
+++ b/db/schema_migrations/20240627122810
@@ -0,0 +1 @@
+8970d19294a69126387be4c728025c55d50866f31173935e7b0f2db772fe13c1
\ No newline at end of file
diff --git a/db/structure.sql b/db/structure.sql
index a393f6b9209b173a75d4a76d9e38427bddb9f866..fd3da17e46406559d43ce6ffdf554fa90a8523cd 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -7997,6 +7997,7 @@ CREATE TABLE ci_pipelines (
partition_id bigint NOT NULL,
id bigint NOT NULL,
auto_canceled_by_id bigint,
+ auto_canceled_by_partition_id bigint,
CONSTRAINT check_d7e99a025e CHECK ((lock_version IS NOT NULL))
);
diff --git a/lib/gitlab/background_migration/queue_backfill_autocancel_partition_id_on_ci_pipelines.rb b/lib/gitlab/background_migration/queue_backfill_autocancel_partition_id_on_ci_pipelines.rb
new file mode 100644
index 0000000000000000000000000000000000000000..4c19f3767fcdb5a47b13690789bc42b4f66e88e7
--- /dev/null
+++ b/lib/gitlab/background_migration/queue_backfill_autocancel_partition_id_on_ci_pipelines.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ class QueueBackfillAutocancelPartitionIdOnCiPipelines < BatchedMigrationJob
+ operation_name :update_all
+ feature_category :ci_scaling
+
+ # rubocop:disable Layout/LineLength -- Improve readability
+ def perform
+ each_sub_batch do |sub_batch|
+ sub_batch
+ .where.not(auto_canceled_by_id: nil)
+ .where('ci_pipelines.auto_canceled_by_id = canceling_pipelines.id')
+ .update_all('auto_canceled_by_partition_id = canceling_pipelines.partition_id FROM ci_pipelines as canceling_pipelines')
+ end
+ end
+ # rubocop:enable Layout/LineLength
+ end
+ end
+end
diff --git a/spec/db/schema_spec.rb b/spec/db/schema_spec.rb
index bbc1b5184400d343dee42e888bef61b07dfb5b01..be831ee8c11f307379a9cfd1406aac0506f0b843 100644
--- a/spec/db/schema_spec.rb
+++ b/spec/db/schema_spec.rb
@@ -65,7 +65,7 @@
ci_pipeline_messages: %w[partition_id],
ci_pipeline_metadata: %w[partition_id],
ci_pipeline_variables: %w[partition_id],
- ci_pipelines: %w[partition_id],
+ ci_pipelines: %w[partition_id auto_canceled_by_partition_id],
ci_runner_projects: %w[runner_id],
ci_sources_pipelines: %w[partition_id source_partition_id source_job_id],
ci_sources_projects: %w[partition_id],
diff --git a/spec/lib/gitlab/background_migration/queue_backfill_autocancel_partition_id_on_ci_pipelines_spec.rb b/spec/lib/gitlab/background_migration/queue_backfill_autocancel_partition_id_on_ci_pipelines_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..27581b315a78049962d109b12edeca727d81489e
--- /dev/null
+++ b/spec/lib/gitlab/background_migration/queue_backfill_autocancel_partition_id_on_ci_pipelines_spec.rb
@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::QueueBackfillAutocancelPartitionIdOnCiPipelines, feature_category: :ci_scaling do
+ let(:ci_pipelines_table) { table(:ci_pipelines, database: :ci) }
+
+ let!(:pipeline_1) { ci_pipelines_table.create!(partition_id: 100) }
+ let!(:pipeline_3) { ci_pipelines_table.create!(partition_id: 101) }
+ let!(:pipeline_2) { ci_pipelines_table.create!(partition_id: 100, auto_canceled_by_id: pipeline_3.id) }
+ let!(:pipeline_4) { ci_pipelines_table.create!(partition_id: 101, auto_canceled_by_id: pipeline_2.id) }
+
+ let(:migration_attrs) do
+ {
+ start_id: ci_pipelines_table.minimum(:id),
+ end_id: ci_pipelines_table.maximum(:id),
+ batch_table: :ci_pipelines,
+ batch_column: :id,
+ sub_batch_size: 2,
+ pause_ms: 0,
+ connection: Ci::ApplicationRecord.connection
+ }
+ end
+
+ let!(:migration) { described_class.new(**migration_attrs) }
+
+ describe '#perform' do
+ it 'backfills auto_canceled_by_partition_id' do
+ expect { migration.perform }
+ .to not_change { pipeline_1.reload.auto_canceled_by_partition_id }
+ .and not_change { pipeline_3.reload.auto_canceled_by_partition_id }
+ .and change { pipeline_2.reload.auto_canceled_by_partition_id }.from(nil).to(101)
+ .and change { pipeline_4.reload.auto_canceled_by_partition_id }.from(nil).to(100)
+ end
+ end
+end
diff --git a/spec/migrations/20240627122810_queue_queue_backfill_autocancel_partition_id_on_ci_pipelines_spec.rb b/spec/migrations/20240627122810_queue_queue_backfill_autocancel_partition_id_on_ci_pipelines_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..0255cce06438192a66090e81dcf8da0bd2b5aa6f
--- /dev/null
+++ b/spec/migrations/20240627122810_queue_queue_backfill_autocancel_partition_id_on_ci_pipelines_spec.rb
@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe QueueQueueBackfillAutocancelPartitionIdOnCiPipelines, migration: :gitlab_ci, feature_category: :ci_scaling do
+ let!(:batched_migration) { described_class::MIGRATION }
+
+ it 'schedules a new batched migration' do
+ reversible_migration do |migration|
+ migration.before -> {
+ expect(batched_migration).not_to have_scheduled_batched_migration
+ }
+
+ migration.after -> {
+ expect(batched_migration).to have_scheduled_batched_migration(
+ gitlab_schema: :gitlab_ci,
+ table_name: :ci_pipelines,
+ column_name: :id,
+ interval: described_class::DELAY_INTERVAL,
+ batch_size: described_class::BATCH_SIZE,
+ sub_batch_size: described_class::SUB_BATCH_SIZE
+ )
+ }
+ end
+ end
+
+ context 'when executed on .com' do
+ before do
+ allow(Gitlab).to receive(:com_except_jh?).and_return(true)
+ end
+
+ it 'schedules a new batched migration' do
+ reversible_migration do |migration|
+ migration.before -> {
+ expect(batched_migration).not_to have_scheduled_batched_migration
+ }
+
+ migration.after -> {
+ expect(batched_migration).to have_scheduled_batched_migration(
+ gitlab_schema: :gitlab_ci,
+ table_name: :ci_pipelines,
+ column_name: :id,
+ interval: described_class::DELAY_INTERVAL,
+ batch_size: described_class::GITLAB_OPTIMIZED_BATCH_SIZE,
+ sub_batch_size: described_class::GITLAB_OPTIMIZED_SUB_BATCH_SIZE
+ )
+ }
+ end
+ end
+ end
+end
diff --git a/spec/services/ci/cancel_pipeline_service_spec.rb b/spec/services/ci/cancel_pipeline_service_spec.rb
index 6051485c4df8c88cab7182cc0bb054a56f5a36a3..d6312aa7d2fa8de0ce460aa2f3f8530ce16ac0d4 100644
--- a/spec/services/ci/cancel_pipeline_service_spec.rb
+++ b/spec/services/ci/cancel_pipeline_service_spec.rb
@@ -80,6 +80,7 @@
subject
expect(pipeline.auto_canceled_by_id).to eq(auto_canceled_by_pipeline.id)
+ expect(pipeline.auto_canceled_by_partition_id).to eq(auto_canceled_by_pipeline.partition_id)
expect(pipeline.all_jobs.canceled.pluck(:auto_canceled_by_id).uniq)
.to eq([auto_canceled_by_pipeline.id])