diff --git a/db/docs/batched_background_migrations/deduplicate_lfs_objects_projects.yml b/db/docs/batched_background_migrations/deduplicate_lfs_objects_projects.yml
index d3f5e9e1e2c9d81edaf77f96433bc53fbeb1a1aa..cee3650c9c42bbe45b35fa513730faf3271604b9 100644
--- a/db/docs/batched_background_migrations/deduplicate_lfs_objects_projects.yml
+++ b/db/docs/batched_background_migrations/deduplicate_lfs_objects_projects.yml
@@ -5,9 +5,13 @@ description: >-
After the migration is finalized, we need to add a unique index on all three columns to ensure
data consistency since the unique validation already exists at the model level.
- Originally introduced by https://gitlab.com/gitlab-org/gitlab/-/merge_requests/154323
+ 1) Originally introduced by https://gitlab.com/gitlab-org/gitlab/-/merge_requests/154323
Re-enqueued because we found some duplicates were still being introduced
+
+ 2) Originally re-enqueued in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/167468
+ Re-enqueued a second time since we found some duplicates when repository_type is NULL
feature_category: source_code_management
-introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/164127
-milestone: '17.4'
-queued_migration_version: 20240827204855
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/167468
+milestone: '17.5'
+queued_migration_version: 20240927202948
+finalize_after: '2024-10-17' # required upgrade stop due date
diff --git a/db/post_migrate/20240827204855_reenqueue_deduplicate_lfs_objects_projects.rb b/db/post_migrate/20240827204855_reenqueue_deduplicate_lfs_objects_projects.rb
index 915117cf25bc06897577595f45828551311e62ee..871c2af4e070fc1d509a600a6ee2f51ee981de78 100644
--- a/db/post_migrate/20240827204855_reenqueue_deduplicate_lfs_objects_projects.rb
+++ b/db/post_migrate/20240827204855_reenqueue_deduplicate_lfs_objects_projects.rb
@@ -13,20 +13,13 @@ class ReenqueueDeduplicateLfsObjectsProjects < Gitlab::Database::Migration[2.2]
restrict_gitlab_migration gitlab_schema: :gitlab_main
+ # To be re-enqueued by:
+ # db/post_migrate/20240927202948_reenqueue_deduplicate_lfs_objects_projects_with_null_repository_types.rb
def up
- delete_batched_background_migration(MIGRATION, :lfs_objects_projects, :id, [])
-
- queue_batched_background_migration(
- MIGRATION,
- :lfs_objects_projects,
- :id,
- job_interval: DELAY_INTERVAL,
- batch_size: BATCH_SIZE,
- sub_batch_size: SUB_BATCH_SIZE
- )
+ # no-op
end
def down
- delete_batched_background_migration(MIGRATION, :lfs_objects_projects, :id, [])
+ # no-op
end
end
diff --git a/db/post_migrate/20240927202948_reenqueue_deduplicate_lfs_objects_projects_with_null_repository_types.rb b/db/post_migrate/20240927202948_reenqueue_deduplicate_lfs_objects_projects_with_null_repository_types.rb
new file mode 100644
index 0000000000000000000000000000000000000000..2e34779de6ae2be46f01aa6e75f65ff5e645af85
--- /dev/null
+++ b/db/post_migrate/20240927202948_reenqueue_deduplicate_lfs_objects_projects_with_null_repository_types.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+class ReenqueueDeduplicateLfsObjectsProjectsWithNullRepositoryTypes < Gitlab::Database::Migration[2.2]
+ milestone '17.5'
+
+ MIGRATION = 'DeduplicateLfsObjectsProjects'
+ TABLE_NAME = :lfs_objects_projects
+ DELAY_INTERVAL = 100
+ BATCH_SIZE = 10_000
+ SUB_BATCH_SIZE = 2_500
+
+ disable_ddl_transaction!
+
+ restrict_gitlab_migration gitlab_schema: :gitlab_main
+
+ def up
+ delete_batched_background_migration(MIGRATION, :lfs_objects_projects, :id, [])
+
+ queue_batched_background_migration(
+ MIGRATION,
+ :lfs_objects_projects,
+ :id,
+ job_interval: DELAY_INTERVAL,
+ batch_size: BATCH_SIZE,
+ sub_batch_size: SUB_BATCH_SIZE
+ )
+ end
+
+ def down
+ delete_batched_background_migration(MIGRATION, :lfs_objects_projects, :id, [])
+ end
+end
diff --git a/db/schema_migrations/20240927202948 b/db/schema_migrations/20240927202948
new file mode 100644
index 0000000000000000000000000000000000000000..2d44e02e6c2a41187a968735a4a280165273535d
--- /dev/null
+++ b/db/schema_migrations/20240927202948
@@ -0,0 +1 @@
+b693c1b11d18d4dae7281af0e5784c91bd285d1e366baae96c00d22f3d0e05d8
\ No newline at end of file
diff --git a/lib/gitlab/background_migration/deduplicate_lfs_objects_projects.rb b/lib/gitlab/background_migration/deduplicate_lfs_objects_projects.rb
index e4191946ffd69f2743752937fb4cceead5f22592..ee3012bfce431cdcc835495cd3250428bdba5f70 100644
--- a/lib/gitlab/background_migration/deduplicate_lfs_objects_projects.rb
+++ b/lib/gitlab/background_migration/deduplicate_lfs_objects_projects.rb
@@ -34,7 +34,7 @@ def perform
INNER JOIN (#{subquery.to_sql}) AS duplicates
ON lfs_objects_projects.project_id = duplicates.project_id
AND lfs_objects_projects.lfs_object_id = duplicates.lfs_object_id
- AND lfs_objects_projects.repository_type = duplicates.repository_type
+ AND lfs_objects_projects.repository_type IS NOT DISTINCT FROM duplicates.repository_type
SQL
duplicated_lfs_objects_projects = LfsObjectsProject.joins(join_query).where.not(
diff --git a/spec/lib/gitlab/background_migration/deduplicate_lfs_objects_projects_spec.rb b/spec/lib/gitlab/background_migration/deduplicate_lfs_objects_projects_spec.rb
index 51fb55ea520c39e9136e6977d9eab8e21259aab7..4c60620a9af805e0d632d15cf3401231a60c10c7 100644
--- a/spec/lib/gitlab/background_migration/deduplicate_lfs_objects_projects_spec.rb
+++ b/spec/lib/gitlab/background_migration/deduplicate_lfs_objects_projects_spec.rb
@@ -68,15 +68,15 @@
end
let!(:duplicated_lfs_objects_project3) do
- lfs_objects_projects_table.create!(project_id: project2.id, lfs_object_id: lfs_object4.id, repository_type: 0)
+ lfs_objects_projects_table.create!(project_id: project2.id, lfs_object_id: lfs_object4.id, repository_type: nil)
end
let!(:duplicated_lfs_objects_project4) do
- lfs_objects_projects_table.create!(project_id: project2.id, lfs_object_id: lfs_object4.id, repository_type: 0)
+ lfs_objects_projects_table.create!(project_id: project2.id, lfs_object_id: lfs_object4.id, repository_type: nil)
end
let!(:lfs_objects_project5) do
- lfs_objects_projects_table.create!(project_id: project2.id, lfs_object_id: lfs_object4.id, repository_type: 0)
+ lfs_objects_projects_table.create!(project_id: project2.id, lfs_object_id: lfs_object4.id, repository_type: nil)
end
let!(:lfs_objects_project6) do
diff --git a/spec/migrations/20240827204855_reenqueue_deduplicate_lfs_objects_projects_spec.rb b/spec/migrations/20240827204855_reenqueue_deduplicate_lfs_objects_projects_spec.rb
index 4aa781478ef9862d6a0f3181b7271868dbeaba3c..a91071cb4e65f23292c24d492a7ce6be3cdcb5db 100644
--- a/spec/migrations/20240827204855_reenqueue_deduplicate_lfs_objects_projects_spec.rb
+++ b/spec/migrations/20240827204855_reenqueue_deduplicate_lfs_objects_projects_spec.rb
@@ -7,20 +7,7 @@
let(:batched_migration) { described_class::MIGRATION }
it 'schedules a new batched migration' do
- reversible_migration do |migration|
- migration.before -> {
- expect(batched_migration).not_to have_scheduled_batched_migration
- }
-
- migration.after -> {
- expect(batched_migration).to have_scheduled_batched_migration(
- table_name: :lfs_objects_projects,
- column_name: :id,
- batch_class_name: described_class::BATCH_CLASS_NAME,
- batch_size: described_class::BATCH_SIZE,
- sub_batch_size: described_class::SUB_BATCH_SIZE
- )
- }
- end
+ migrate!
+ expect(batched_migration).not_to have_scheduled_batched_migration
end
end
diff --git a/spec/migrations/20240927202948_reenqueue_deduplicate_lfs_objects_projects_with_null_repository_types_spec.rb b/spec/migrations/20240927202948_reenqueue_deduplicate_lfs_objects_projects_with_null_repository_types_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..015a7ea4dda3360670fb2e04a8fae5ef0a0194b8
--- /dev/null
+++ b/spec/migrations/20240927202948_reenqueue_deduplicate_lfs_objects_projects_with_null_repository_types_spec.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe ReenqueueDeduplicateLfsObjectsProjectsWithNullRepositoryTypes, feature_category: :source_code_management do
+ let(:batched_migration) { described_class::MIGRATION }
+
+ it 'schedules a new batched migration' do
+ reversible_migration do |migration|
+ migration.before -> {
+ expect(batched_migration).not_to have_scheduled_batched_migration
+ }
+
+ migration.after -> {
+ expect(batched_migration).to have_scheduled_batched_migration(
+ table_name: :lfs_objects_projects,
+ column_name: :id,
+ batch_class_name: described_class::BATCH_CLASS_NAME,
+ batch_size: described_class::BATCH_SIZE,
+ sub_batch_size: described_class::SUB_BATCH_SIZE
+ )
+ }
+ end
+ end
+end