Merge branch...

Merge branch 'backfill-multiple-desired-sharding-key-small-table-bulk_import_export_uploads' into 'master' 

Add & backfill sharding keys for bulk_import_export_uploads

See merge request https://gitlab.com/gitlab-org/gitlab/-/merge_requests/175723



Merged-by: Adam Hegyi <ahegyi@gitlab.com>
Approved-by: Adam Hegyi <ahegyi@gitlab.com>
Co-authored-by: Shubham Kumar <shukumar@gitlab.com>

db/docs/batched_background_migrations/backfill_bulk_import_export_uploads_group_id.yml

+---
+migration_job_name: BackfillBulkImportExportUploadsGroupId
+description: Backfills sharding key `bulk_import_export_uploads.group_id` from `bulk_import_exports`.
+feature_category: importers
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/175723
+milestone: '17.8'
+queued_migration_version: 20241213150054
+finalized_by: # version of the migration that finalized this BBM

db/docs/batched_background_migrations/backfill_bulk_import_export_uploads_project_id.yml

+---
+migration_job_name: BackfillBulkImportExportUploadsProjectId
+description: Backfills sharding key `bulk_import_export_uploads.project_id` from `bulk_import_exports`.
+feature_category: importers
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/175723
+milestone: '17.8'
+queued_migration_version: 20241213150049
+finalized_by: # version of the migration that finalized this BBM

db/docs/bulk_import_export_uploads.yml

@@ -27,3 +27,6 @@ desired_sharding_key:
         sharding_key: group_id
         belongs_to: export
 table_size: small
+desired_sharding_key_migration_job_name:
+- BackfillBulkImportExportUploadsProjectId
+- BackfillBulkImportExportUploadsGroupId

db/migrate/20241213150045_add_project_id_to_bulk_import_export_uploads.rb

+# frozen_string_literal: true
+
+class AddProjectIdToBulkImportExportUploads < Gitlab::Database::Migration[2.2]
+  milestone '17.8'
+
+  def change
+    add_column :bulk_import_export_uploads, :project_id, :bigint
+  end
+end

db/migrate/20241213150050_add_group_id_to_bulk_import_export_uploads.rb

+# frozen_string_literal: true
+
+class AddGroupIdToBulkImportExportUploads < Gitlab::Database::Migration[2.2]
+  milestone '17.8'
+
+  def change
+    add_column :bulk_import_export_uploads, :group_id, :bigint
+  end
+end

db/post_migrate/20241213150046_index_bulk_import_export_uploads_on_project_id.rb

+# frozen_string_literal: true
+
+class IndexBulkImportExportUploadsOnProjectId < Gitlab::Database::Migration[2.2]
+  milestone '17.8'
+  disable_ddl_transaction!
+
+  INDEX_NAME = 'index_bulk_import_export_uploads_on_project_id'
+
+  def up
+    add_concurrent_index :bulk_import_export_uploads, :project_id, name: INDEX_NAME
+  end
+
+  def down
+    remove_concurrent_index_by_name :bulk_import_export_uploads, INDEX_NAME
+  end
+end

db/post_migrate/20241213150047_add_bulk_import_export_uploads_project_id_fk.rb

+# frozen_string_literal: true
+
+class AddBulkImportExportUploadsProjectIdFk < Gitlab::Database::Migration[2.2]
+  milestone '17.8'
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_foreign_key :bulk_import_export_uploads, :projects, column: :project_id, on_delete: :cascade
+  end
+
+  def down
+    with_lock_retries do
+      remove_foreign_key :bulk_import_export_uploads, column: :project_id
+    end
+  end
+end

db/post_migrate/20241213150048_add_bulk_import_export_uploads_project_id_trigger.rb

+# frozen_string_literal: true
+
+class AddBulkImportExportUploadsProjectIdTrigger < Gitlab::Database::Migration[2.2]
+  milestone '17.8'
+
+  def up
+    install_sharding_key_assignment_trigger(
+      table: :bulk_import_export_uploads,
+      sharding_key: :project_id,
+      parent_table: :bulk_import_exports,
+      parent_sharding_key: :project_id,
+      foreign_key: :export_id
+    )
+  end
+
+  def down
+    remove_sharding_key_assignment_trigger(
+      table: :bulk_import_export_uploads,
+      sharding_key: :project_id,
+      parent_table: :bulk_import_exports,
+      parent_sharding_key: :project_id,
+      foreign_key: :export_id
+    )
+  end
+end

db/post_migrate/20241213150049_queue_backfill_bulk_import_export_uploads_project_id.rb

+# frozen_string_literal: true
+
+class QueueBackfillBulkImportExportUploadsProjectId < Gitlab::Database::Migration[2.2]
+  milestone '17.8'
+  restrict_gitlab_migration gitlab_schema: :gitlab_main_cell
+
+  MIGRATION = "BackfillBulkImportExportUploadsProjectId"
+  DELAY_INTERVAL = 2.minutes
+  BATCH_SIZE = 1000
+  SUB_BATCH_SIZE = 100
+
+  def up
+    queue_batched_background_migration(
+      MIGRATION,
+      :bulk_import_export_uploads,
+      :id,
+      :project_id,
+      :bulk_import_exports,
+      :project_id,
+      :export_id,
+      job_interval: DELAY_INTERVAL,
+      batch_size: BATCH_SIZE,
+      sub_batch_size: SUB_BATCH_SIZE
+    )
+  end
+
+  def down
+    delete_batched_background_migration(
+      MIGRATION,
+      :bulk_import_export_uploads,
+      :id,
+      [
+        :project_id,
+        :bulk_import_exports,
+        :project_id,
+        :export_id
+      ]
+    )
+  end
+end

db/post_migrate/20241213150051_index_bulk_import_export_uploads_on_group_id.rb

+# frozen_string_literal: true
+
+class IndexBulkImportExportUploadsOnGroupId < Gitlab::Database::Migration[2.2]
+  milestone '17.8'
+  disable_ddl_transaction!
+
+  INDEX_NAME = 'index_bulk_import_export_uploads_on_group_id'
+
+  def up
+    add_concurrent_index :bulk_import_export_uploads, :group_id, name: INDEX_NAME
+  end
+
+  def down
+    remove_concurrent_index_by_name :bulk_import_export_uploads, INDEX_NAME
+  end
+end

db/post_migrate/20241213150052_add_bulk_import_export_uploads_group_id_fk.rb

+# frozen_string_literal: true
+
+class AddBulkImportExportUploadsGroupIdFk < Gitlab::Database::Migration[2.2]
+  milestone '17.8'
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_foreign_key :bulk_import_export_uploads, :namespaces, column: :group_id, on_delete: :cascade
+  end
+
+  def down
+    with_lock_retries do
+      remove_foreign_key :bulk_import_export_uploads, column: :group_id
+    end
+  end
+end

db/post_migrate/20241213150053_add_bulk_import_export_uploads_group_id_trigger.rb

+# frozen_string_literal: true
+
+class AddBulkImportExportUploadsGroupIdTrigger < Gitlab::Database::Migration[2.2]
+  milestone '17.8'
+
+  def up
+    install_sharding_key_assignment_trigger(
+      table: :bulk_import_export_uploads,
+      sharding_key: :group_id,
+      parent_table: :bulk_import_exports,
+      parent_sharding_key: :group_id,
+      foreign_key: :export_id
+    )
+  end
+
+  def down
+    remove_sharding_key_assignment_trigger(
+      table: :bulk_import_export_uploads,
+      sharding_key: :group_id,
+      parent_table: :bulk_import_exports,
+      parent_sharding_key: :group_id,
+      foreign_key: :export_id
+    )
+  end
+end

db/post_migrate/20241213150054_queue_backfill_bulk_import_export_uploads_group_id.rb

+# frozen_string_literal: true
+
+class QueueBackfillBulkImportExportUploadsGroupId < Gitlab::Database::Migration[2.2]
+  milestone '17.8'
+  restrict_gitlab_migration gitlab_schema: :gitlab_main_cell
+
+  MIGRATION = "BackfillBulkImportExportUploadsGroupId"
+  DELAY_INTERVAL = 2.minutes
+  BATCH_SIZE = 1000
+  SUB_BATCH_SIZE = 100
+
+  def up
+    queue_batched_background_migration(
+      MIGRATION,
+      :bulk_import_export_uploads,
+      :id,
+      :group_id,
+      :bulk_import_exports,
+      :group_id,
+      :export_id,
+      job_interval: DELAY_INTERVAL,
+      batch_size: BATCH_SIZE,
+      sub_batch_size: SUB_BATCH_SIZE
+    )
+  end
+
+  def down
+    delete_batched_background_migration(
+      MIGRATION,
+      :bulk_import_export_uploads,
+      :id,
+      [
+        :group_id,
+        :bulk_import_exports,
+        :group_id,
+        :export_id
+      ]
+    )
+  end
+end

db/schema_migrations/20241213150045

+1f05ebd14030efcc79a88a09a16c954d30f30032fce39068e23e8d9bf45b5ba7
\ No newline at end of file

db/schema_migrations/20241213150046

+94d04e41dd8d681dea88a6828bf4f27c5812938dbf4436ae5ad81478b1c23fc4
\ No newline at end of file

db/schema_migrations/20241213150047

+cd31480390b905bba738eef4ac28e1fb392881a3097f6c69fa427dc85b7bc93a
\ No newline at end of file

db/schema_migrations/20241213150048

+8c3064d90d9fc6cf33aa8685c634ff8c37934453075870b151a372f09aee8ee4
\ No newline at end of file

db/schema_migrations/20241213150049

+c343e40451fe18fd57ce80a450a79bd8e7fb4d4e2b4166bcd260835d2dfa7579
\ No newline at end of file

db/schema_migrations/20241213150050

+7baaad2cc55cc1922708e1cf5cd0bcf465e4fe34f2d067e555966aa8c1a62107
\ No newline at end of file

db/schema_migrations/20241213150051

+d93e4a954c0ca17c77f713fc4548fd9e18a249b1e3a63ba84f3670b6060ee638
\ No newline at end of file