diff --git a/db/docs/batched_background_migrations/delete_orphaned_pipeline_variable_records.yml b/db/docs/batched_background_migrations/delete_orphaned_pipeline_variable_records.yml new file mode 100644 index 0000000000000000000000000000000000000000..ee0cd4e838e71b8eb9224bf1b39578fe6e9dddd4 --- /dev/null +++ b/db/docs/batched_background_migrations/delete_orphaned_pipeline_variable_records.yml @@ -0,0 +1,8 @@ +--- +migration_job_name: DeleteOrphanedPipelineVariableRecords +description: Deletes corrupted rows from p_ci_pipeline_variables table +feature_category: continuous_integration +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/169441 +milestone: '17.6' +queued_migration_version: 20241016131601 +finalized_by: # version of the migration that finalized this BBM diff --git a/db/post_migrate/20241016131601_queue_delete_orphaned_pipeline_variable_records.rb b/db/post_migrate/20241016131601_queue_delete_orphaned_pipeline_variable_records.rb new file mode 100644 index 0000000000000000000000000000000000000000..9aa441eb04e9d575f0a0872d90d00ac54e503701 --- /dev/null +++ b/db/post_migrate/20241016131601_queue_delete_orphaned_pipeline_variable_records.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class QueueDeleteOrphanedPipelineVariableRecords < Gitlab::Database::Migration[2.2] + milestone '17.6' + restrict_gitlab_migration gitlab_schema: :gitlab_ci + + MIGRATION = "DeleteOrphanedPipelineVariableRecords" + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 100 + + def up + queue_batched_background_migration( + MIGRATION, + :p_ci_pipeline_variables, + :pipeline_id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + batch_class_name: 'LooseIndexScanBatchingStrategy', + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration(MIGRATION, :p_ci_pipeline_variables, :pipeline_id, []) + end +end diff --git a/db/schema_migrations/20241016131601 b/db/schema_migrations/20241016131601 new file mode 100644 index 0000000000000000000000000000000000000000..0239a33506dcba6942c34fae9244adcf4cb1fef5 --- /dev/null +++ b/db/schema_migrations/20241016131601 @@ -0,0 +1 @@ +41127c2fcc61d93b91ac88a9f21c40ed503b18a25140c7ef6a30ec37d83f1f54 \ No newline at end of file diff --git a/lib/gitlab/background_migration/delete_orphaned_pipeline_variable_records.rb b/lib/gitlab/background_migration/delete_orphaned_pipeline_variable_records.rb new file mode 100644 index 0000000000000000000000000000000000000000..597a129e9b5c4a4d7fcd8f5a25f9ecd2c7d92dbd --- /dev/null +++ b/lib/gitlab/background_migration/delete_orphaned_pipeline_variable_records.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class DeleteOrphanedPipelineVariableRecords < BatchedMigrationJob + operation_name :delete_orphaned_pipeline_variable_records + feature_category :continuous_integration + + class CiPipeline < ::Ci::ApplicationRecord + self.table_name = :p_ci_pipelines + self.primary_key = :id + end + + def perform + distinct_each_batch do |batch| + pipeline_ids = batch.pluck(batch_column) + pipelines_query = CiPipeline + .where('p_ci_pipeline_variables.pipeline_id = p_ci_pipelines.id') + .where('p_ci_pipeline_variables.partition_id = p_ci_pipelines.partition_id') + .select(1) + + base_relation + .where(batch_column => pipeline_ids) + .where('NOT EXISTS (?)', pipelines_query) + .delete_all + end + end + + private + + def base_relation + define_batchable_model(batch_table, connection: connection, primary_key: :id) + .where(batch_column => start_id..end_id) + end + end + end +end diff --git a/spec/lib/gitlab/background_migration/delete_orphaned_pipeline_variable_records_spec.rb b/spec/lib/gitlab/background_migration/delete_orphaned_pipeline_variable_records_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..53599dc4fc3cb7b6edf70b19993dd64c1b336766 --- /dev/null +++ b/spec/lib/gitlab/background_migration/delete_orphaned_pipeline_variable_records_spec.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::DeleteOrphanedPipelineVariableRecords, + feature_category: :continuous_integration, migration: :gitlab_ci do + let(:pipelines_table) { table(:p_ci_pipelines, database: :ci, primary_key: :id) } + let(:variables_table) { table(:p_ci_pipeline_variables, database: :ci, primary_key: :id) } + + let(:default_attributes) { { project_id: 600, partition_id: 100 } } + let!(:regular_pipeline) { pipelines_table.create!(default_attributes) } + let!(:deleted_pipeline) { pipelines_table.create!(default_attributes) } + let!(:other_pipeline) { pipelines_table.create!(default_attributes) } + + let!(:regular_variable) do + variables_table.create!(pipeline_id: regular_pipeline.id, key: :key1, **default_attributes) + end + + let!(:orphaned_variable) do + variables_table.create!(pipeline_id: deleted_pipeline.id, key: :key2, **default_attributes) + end + + let(:connection) { Ci::ApplicationRecord.connection } + + around do |example| + connection.transaction do + connection.execute(<<~SQL) + ALTER TABLE ci_pipelines DISABLE TRIGGER ALL; + SQL + + example.run + + connection.execute(<<~SQL) + ALTER TABLE ci_pipelines ENABLE TRIGGER ALL; + SQL + end + end + + describe '#perform' do + subject(:migration) do + described_class.new( + start_id: variables_table.minimum(:pipeline_id), + end_id: variables_table.maximum(:pipeline_id), + batch_table: :p_ci_pipeline_variables, + batch_column: :pipeline_id, + sub_batch_size: 100, + pause_ms: 0, + connection: connection + ) + end + + it 'deletes from p_ci_pipeline_variables where pipeline_id has no related', :aggregate_failures do + expect { deleted_pipeline.delete }.to not_change { variables_table.count } + + expect { migration.perform }.to change { variables_table.count }.from(2).to(1) + + expect(regular_variable.reload).to be_persisted + expect { orphaned_variable.reload }.to raise_error(ActiveRecord::RecordNotFound) + end + end +end diff --git a/spec/migrations/20241016131601_queue_delete_orphaned_pipeline_variable_records_spec.rb b/spec/migrations/20241016131601_queue_delete_orphaned_pipeline_variable_records_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..b56d257cd867560be74083592cd0196e27e2086d --- /dev/null +++ b/spec/migrations/20241016131601_queue_delete_orphaned_pipeline_variable_records_spec.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueDeleteOrphanedPipelineVariableRecords, migration: :gitlab_ci, feature_category: :continuous_integration do + let!(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + table_name: :p_ci_pipeline_variables, + column_name: :pipeline_id, + interval: described_class::DELAY_INTERVAL, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE, + gitlab_schema: :gitlab_ci + ) + } + end + end +end