Merge branch '369930-update-batched-migration-batch-sizes' into 'master'

Update task system note background migration batch sizes

See merge request https://gitlab.com/gitlab-org/gitlab/-/merge_requests/100457



Merged-by: Diogo Frazão <dfrazao@gitlab.com>
Approved-by: Diogo Frazão <dfrazao@gitlab.com>
Co-authored-by: Mario Celi <mcelicalderon@gitlab.com>

db/post_migrate/20221006172302_adjust_task_note_rename_background_migration_values.rb

+# frozen_string_literal: true
+
+class AdjustTaskNoteRenameBackgroundMigrationValues < Gitlab::Database::Migration[2.0]
+  restrict_gitlab_migration gitlab_schema: :gitlab_main
+
+  JOB_CLASS_NAME = 'RenameTaskSystemNoteToChecklistItem'
+  MIGRATION_FAILED_STATUS = 4
+  MIGRATION_FINISHED_STATUS = 3
+  MIGRATION_ACTIVE_STATUS = 1
+  JOB_FAILED_STATUS = 2
+
+  OLD_BATCH_SIZE = 10_000
+  NEW_BATCH_SIZE = 5_000
+
+  OLD_SUB_BATCH_SIZE = 100
+  NEW_SUB_BATCH_SIZE = 10
+
+  class InlineBatchedMigration < MigrationRecord
+    self.table_name = :batched_background_migrations
+
+    scope :for_configuration, ->(job_class_name, table_name, column_name, job_arguments) do
+      where(job_class_name: job_class_name, table_name: table_name, column_name: column_name)
+        .where("job_arguments = ?", job_arguments.to_json) # rubocop:disable Rails/WhereEquals
+    end
+  end
+
+  class InlineBatchedJob < MigrationRecord
+    include EachBatch
+    self.table_name = :batched_background_migration_jobs
+  end
+
+  def up
+    migration = InlineBatchedMigration.for_configuration(
+      JOB_CLASS_NAME,
+      :system_note_metadata,
+      :id,
+      []
+    ).first
+    return if migration.blank? || migration.status == MIGRATION_FINISHED_STATUS
+
+    InlineBatchedJob.where(
+      batched_background_migration_id: migration.id,
+      status: JOB_FAILED_STATUS
+    ).each_batch(of: 100) do |batch|
+      batch.update_all(attempts: 0, sub_batch_size: NEW_SUB_BATCH_SIZE)
+    end
+
+    update_params = { batch_size: NEW_BATCH_SIZE, sub_batch_size: NEW_SUB_BATCH_SIZE }
+
+    if migration.status == MIGRATION_FAILED_STATUS
+      update_params[:status] = MIGRATION_ACTIVE_STATUS
+      update_params[:started_at] = Time.zone.now if migration.respond_to?(:started_at)
+    end
+
+    migration.update!(**update_params)
+  end
+
+  def down
+    migration = InlineBatchedMigration.for_configuration(
+      JOB_CLASS_NAME,
+      :system_note_metadata,
+      :id,
+      []
+    ).first
+    return if migration.blank?
+
+    migration.update!(
+      batch_size: OLD_BATCH_SIZE,
+      sub_batch_size: OLD_SUB_BATCH_SIZE
+    )
+  end
+end

db/schema_migrations/20221006172302

+19e5ca6f9716fd41bfe8a103dab8a1dc37107f99503abedbdcb8175b699283f2
\ No newline at end of file

spec/migrations/adjust_task_note_rename_background_migration_values_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe AdjustTaskNoteRenameBackgroundMigrationValues, :migration do
+  let(:finished_status) { 3 }
+  let(:failed_status) { described_class::MIGRATION_FAILED_STATUS }
+  let(:active_status) { described_class::MIGRATION_ACTIVE_STATUS }
+
+  shared_examples 'task note migration with failing batches' do
+    it 'updates batch sizes and resets failed batches' do
+      migration = create_background_migration(status: initial_status)
+      batches = []
+
+      batches << create_failed_batched_job(migration)
+      batches << create_failed_batched_job(migration)
+
+      migrate!
+
+      expect(described_class::JOB_CLASS_NAME).to have_scheduled_batched_migration(
+        table_name: :system_note_metadata,
+        column_name: :id,
+        interval: 2.minutes,
+        batch_size: described_class::NEW_BATCH_SIZE,
+        max_batch_size: 20_000,
+        sub_batch_size: described_class::NEW_SUB_BATCH_SIZE
+      )
+      expect(migration.reload.status).to eq(active_status)
+
+      updated_batches = batches.map { |b| b.reload.attributes.slice('attempts', 'sub_batch_size') }
+      expect(updated_batches).to all(eq("attempts" => 0, "sub_batch_size" => 10))
+    end
+  end
+
+  describe '#up' do
+    context 'when migration was already finished' do
+      it 'does not update batch sizes' do
+        create_background_migration(status: finished_status)
+
+        migrate!
+
+        expect(described_class::JOB_CLASS_NAME).to have_scheduled_batched_migration(
+          table_name: :system_note_metadata,
+          column_name: :id,
+          interval: 2.minutes,
+          batch_size: described_class::OLD_BATCH_SIZE,
+          max_batch_size: 20_000,
+          sub_batch_size: described_class::OLD_SUB_BATCH_SIZE
+        )
+      end
+    end
+
+    context 'when the migration had failing batches' do
+      context 'when migration had a failed status' do
+        it_behaves_like 'task note migration with failing batches' do
+          let(:initial_status) { failed_status }
+        end
+
+        it 'updates started_at timestamp' do
+          migration = create_background_migration(status: failed_status)
+          now = Time.zone.now
+
+          travel_to now do
+            migrate!
+            migration.reload
+          end
+
+          expect(migration.started_at).to be_like_time(now)
+        end
+      end
+
+      context 'when migration had an active status' do
+        it_behaves_like 'task note migration with failing batches' do
+          let(:initial_status) { active_status }
+        end
+
+        it 'does not update started_at timestamp' do
+          migration = create_background_migration(status: active_status)
+          original_time = migration.started_at
+
+          migrate!
+          migration.reload
+
+          expect(migration.started_at).to be_like_time(original_time)
+        end
+      end
+    end
+  end
+
+  describe '#down' do
+    it 'reverts to old batch sizes' do
+      create_background_migration(status: finished_status)
+
+      migrate!
+      schema_migrate_down!
+
+      expect(described_class::JOB_CLASS_NAME).to have_scheduled_batched_migration(
+        table_name: :system_note_metadata,
+        column_name: :id,
+        interval: 2.minutes,
+        batch_size: described_class::OLD_BATCH_SIZE,
+        max_batch_size: 20_000,
+        sub_batch_size: described_class::OLD_SUB_BATCH_SIZE
+      )
+    end
+  end
+
+  def create_failed_batched_job(migration)
+    table(:batched_background_migration_jobs).create!(
+      batched_background_migration_id: migration.id,
+      status: described_class::JOB_FAILED_STATUS,
+      min_value: 1,
+      max_value: 10,
+      attempts: 3,
+      batch_size: described_class::OLD_BATCH_SIZE,
+      sub_batch_size: described_class::OLD_SUB_BATCH_SIZE
+    )
+  end
+
+  def create_background_migration(status:)
+    migrations_table = table(:batched_background_migrations)
+    # make sure we only have on migration with that job class name in the specs
+    migrations_table.where(job_class_name: described_class::JOB_CLASS_NAME).delete_all
+
+    migrations_table.create!(
+      job_class_name: described_class::JOB_CLASS_NAME,
+      status: status,
+      max_value: 10,
+      max_batch_size: 20_000,
+      batch_size: described_class::OLD_BATCH_SIZE,
+      sub_batch_size: described_class::OLD_SUB_BATCH_SIZE,
+      interval: 2.minutes,
+      table_name: :system_note_metadata,
+      column_name: :id,
+      total_tuple_count: 100_000,
+      pause_ms: 100,
+      gitlab_schema: :gitlab_main,
+      job_arguments: [],
+      started_at: 2.days.ago
+    )
+  end
+end