From 15d02673bf96ac5c3dbd539eeae49ef1ae214a28 Mon Sep 17 00:00:00 2001
From: Aakriti Gupta <agupta@gitlab.com>
Date: Fri, 21 Apr 2023 07:32:30 +0000
Subject: [PATCH] Backfill design_management_repositories table
Changelog: other
EE: true
---
app/models/design_management/repository.rb | 2 +-
...ackfill_design_management_repositories.yml | 6 ++
...backfill_design_management_repositories.rb | 26 +++++++
db/schema_migrations/20230406121544 | 1 +
...backfill_design_management_repositories.rb | 29 ++++++++
.../import_export/project/import_export.yml | 1 +
.../import_export/project/relation_factory.rb | 1 +
.../gitlab/import_export/complex/project.json | 6 ++
.../design_management_repository.ndjson | 1 +
...ill_design_management_repositories_spec.rb | 68 +++++++++++++++++++
.../import_export/safe_model_attributes.yml | 5 ++
...ill_design_management_repositories_spec.rb | 26 +++++++
12 files changed, 171 insertions(+), 1 deletion(-)
create mode 100644 db/docs/batched_background_migrations/backfill_design_management_repositories.yml
create mode 100644 db/post_migrate/20230406121544_queue_backfill_design_management_repositories.rb
create mode 100644 db/schema_migrations/20230406121544
create mode 100644 lib/gitlab/background_migration/backfill_design_management_repositories.rb
create mode 100644 spec/fixtures/lib/gitlab/import_export/complex/tree/project/design_management_repository.ndjson
create mode 100644 spec/lib/gitlab/background_migration/backfill_design_management_repositories_spec.rb
create mode 100644 spec/migrations/20230406121544_queue_backfill_design_management_repositories_spec.rb
diff --git a/app/models/design_management/repository.rb b/app/models/design_management/repository.rb
index e6790bc3253c0..c1b14a875ac08 100644
--- a/app/models/design_management/repository.rb
+++ b/app/models/design_management/repository.rb
@@ -17,7 +17,7 @@ class Repository < ApplicationRecord
delegate_missing_to :git_repo
def git_repo
- GitRepository.new(project)
+ project ? GitRepository.new(project) : nil
end
strong_memoize_attr :git_repo
end
diff --git a/db/docs/batched_background_migrations/backfill_design_management_repositories.yml b/db/docs/batched_background_migrations/backfill_design_management_repositories.yml
new file mode 100644
index 0000000000000..95ff87869c335
--- /dev/null
+++ b/db/docs/batched_background_migrations/backfill_design_management_repositories.yml
@@ -0,0 +1,6 @@
+---
+migration_job_name: BackfillDesignManagementRepositories
+description: Backfills the new table with project ids
+feature_category: geo_replication
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/116975
+milestone: 15.11
diff --git a/db/post_migrate/20230406121544_queue_backfill_design_management_repositories.rb b/db/post_migrate/20230406121544_queue_backfill_design_management_repositories.rb
new file mode 100644
index 0000000000000..22ff4c8df4581
--- /dev/null
+++ b/db/post_migrate/20230406121544_queue_backfill_design_management_repositories.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+class QueueBackfillDesignManagementRepositories < Gitlab::Database::Migration[2.1]
+ MIGRATION = "BackfillDesignManagementRepositories"
+
+ DELAY_INTERVAL = 2.minutes
+ BATCH_SIZE = 1000
+ SUB_BATCH_SIZE = 100
+
+ restrict_gitlab_migration gitlab_schema: :gitlab_main
+
+ def up
+ queue_batched_background_migration(
+ MIGRATION,
+ :projects,
+ :id,
+ job_interval: DELAY_INTERVAL,
+ batch_size: BATCH_SIZE,
+ sub_batch_size: SUB_BATCH_SIZE
+ )
+ end
+
+ def down
+ delete_batched_background_migration(MIGRATION, :projects, :id, [])
+ end
+end
diff --git a/db/schema_migrations/20230406121544 b/db/schema_migrations/20230406121544
new file mode 100644
index 0000000000000..3039b8b297f65
--- /dev/null
+++ b/db/schema_migrations/20230406121544
@@ -0,0 +1 @@
+62541ed3a4cd0d62c227c29403d9685bec1779226a00c9c96c13352eed10f1b4
\ No newline at end of file
diff --git a/lib/gitlab/background_migration/backfill_design_management_repositories.rb b/lib/gitlab/background_migration/backfill_design_management_repositories.rb
new file mode 100644
index 0000000000000..fe57767a6932f
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_design_management_repositories.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ # Backfill design_management_repositories table for a range of projects
+ class BackfillDesignManagementRepositories < BatchedMigrationJob
+ operation_name :backfill_design_management_repositories
+ feature_category :geo_replication
+
+ def perform
+ each_sub_batch do |sub_batch|
+ backfill_design_management_repositories(sub_batch)
+ end
+ end
+
+ def backfill_design_management_repositories(relation)
+ connection.execute(
+ <<~SQL
+ INSERT INTO design_management_repositories (project_id, created_at, updated_at)
+ SELECT projects.id, now(), now()
+ FROM projects
+ WHERE projects.id IN(#{relation.select(:id).to_sql})
+ ON CONFLICT (project_id) DO NOTHING;
+ SQL
+ )
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml
index 56cbc5f1bb4ad..36a3c73271b6c 100644
--- a/lib/gitlab/import_export/project/import_export.yml
+++ b/lib/gitlab/import_export/project/import_export.yml
@@ -121,6 +121,7 @@ tree:
- label:
- :priorities
- :service_desk_setting
+ - :design_management_repository
group_members:
- :user
diff --git a/lib/gitlab/import_export/project/relation_factory.rb b/lib/gitlab/import_export/project/relation_factory.rb
index 9afa7cc1daebe..5d7e3ea9ed738 100644
--- a/lib/gitlab/import_export/project/relation_factory.rb
+++ b/lib/gitlab/import_export/project/relation_factory.rb
@@ -22,6 +22,7 @@ class RelationFactory < Base::RelationFactory
create_access_levels: 'ProtectedTag::CreateAccessLevel',
design: 'DesignManagement::Design',
designs: 'DesignManagement::Design',
+ design_management_repository: 'DesignManagement::Repository',
design_versions: 'DesignManagement::Version',
actions: 'DesignManagement::Action',
labels: :project_labels,
diff --git a/spec/fixtures/lib/gitlab/import_export/complex/project.json b/spec/fixtures/lib/gitlab/import_export/complex/project.json
index cdf9395fbe615..8a307af1ca7b2 100644
--- a/spec/fixtures/lib/gitlab/import_export/complex/project.json
+++ b/spec/fixtures/lib/gitlab/import_export/complex/project.json
@@ -45,6 +45,12 @@
]
}
],
+ "design_management_repository": {
+ "id": 500,
+ "project_id": 30,
+ "created_at": "2019-08-07T03:57:55.007Z",
+ "updated_at": "2019-08-07T03:57:55.007Z"
+ },
"issues": [
{
"id": 40,
diff --git a/spec/fixtures/lib/gitlab/import_export/complex/tree/project/design_management_repository.ndjson b/spec/fixtures/lib/gitlab/import_export/complex/tree/project/design_management_repository.ndjson
new file mode 100644
index 0000000000000..c1676157e686d
--- /dev/null
+++ b/spec/fixtures/lib/gitlab/import_export/complex/tree/project/design_management_repository.ndjson
@@ -0,0 +1 @@
+{"id":500, "project_id":30, "created_at":"2019-08-07T03:57:55.007Z", "updated_at":"2019-08-07T03:57:55.007Z"}
\ No newline at end of file
diff --git a/spec/lib/gitlab/background_migration/backfill_design_management_repositories_spec.rb b/spec/lib/gitlab/background_migration/backfill_design_management_repositories_spec.rb
new file mode 100644
index 0000000000000..0cabdc78db836
--- /dev/null
+++ b/spec/lib/gitlab/background_migration/backfill_design_management_repositories_spec.rb
@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe(
+ Gitlab::BackgroundMigration::BackfillDesignManagementRepositories,
+ schema: 20230406121544,
+ feature_category: :geo_replication
+) do
+ let!(:namespaces) { table(:namespaces) }
+ let!(:projects) { table(:projects) }
+ let!(:design_management_repositories) { table(:design_management_repositories) }
+
+ subject(:migration) do
+ described_class.new(
+ start_id: projects.minimum(:id),
+ end_id: projects.maximum(:id),
+ batch_table: :projects,
+ batch_column: :id,
+ sub_batch_size: 2,
+ pause_ms: 0,
+ connection: ActiveRecord::Base.connection
+ )
+ end
+
+ describe '#perform' do
+ it 'creates design_management_repositories entries for all projects in range' do
+ namespace1 = create_namespace('test1')
+ namespace2 = create_namespace('test2')
+ project1 = create_project(namespace1, 'test1')
+ project2 = create_project(namespace2, 'test2')
+ design_management_repositories.create!(project_id: project2.id)
+
+ expect { migration.perform }
+ .to change { design_management_repositories.pluck(:project_id) }
+ .from([project2.id])
+ .to match_array([project1.id, project2.id])
+ end
+
+ context 'when project_id already exists in design_management_repositories' do
+ it "doesn't duplicate project_id" do
+ namespace = create_namespace('test1')
+ project = create_project(namespace, 'test1')
+ design_management_repositories.create!(project_id: project.id)
+
+ expect { migration.perform }
+ .not_to change { design_management_repositories.pluck(:project_id) }
+ end
+ end
+
+ def create_namespace(name)
+ namespaces.create!(
+ name: name,
+ path: name,
+ type: 'Project'
+ )
+ end
+
+ def create_project(namespace, name)
+ projects.create!(
+ namespace_id: namespace.id,
+ project_namespace_id: namespace.id,
+ name: name,
+ path: name
+ )
+ end
+ end
+end
diff --git a/spec/lib/gitlab/import_export/safe_model_attributes.yml b/spec/lib/gitlab/import_export/safe_model_attributes.yml
index 854909fd59250..a2087695d3150 100644
--- a/spec/lib/gitlab/import_export/safe_model_attributes.yml
+++ b/spec/lib/gitlab/import_export/safe_model_attributes.yml
@@ -927,6 +927,11 @@ DesignManagement::Version:
- created_at
- sha
- author_id
+DesignManagement::Repository:
+- id
+- project_id
+- created_at
+- updated_at
ZoomMeeting:
- id
- project_id
diff --git a/spec/migrations/20230406121544_queue_backfill_design_management_repositories_spec.rb b/spec/migrations/20230406121544_queue_backfill_design_management_repositories_spec.rb
new file mode 100644
index 0000000000000..39ef769fd1191
--- /dev/null
+++ b/spec/migrations/20230406121544_queue_backfill_design_management_repositories_spec.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe QueueBackfillDesignManagementRepositories, feature_category: :geo_replication do
+ let!(:batched_migration) { described_class::MIGRATION }
+
+ it 'schedules a new batched migration' do
+ reversible_migration do |migration|
+ migration.before -> {
+ expect(batched_migration).not_to have_scheduled_batched_migration
+ }
+
+ migration.after -> {
+ expect(batched_migration).to have_scheduled_batched_migration(
+ table_name: :projects,
+ column_name: :id,
+ interval: described_class::DELAY_INTERVAL,
+ batch_size: described_class::BATCH_SIZE,
+ sub_batch_size: described_class::SUB_BATCH_SIZE
+ )
+ }
+ end
+ end
+end
--
GitLab