Merge branch...

Merge branch 'backfill-desired-sharding-key-small-table-design_management_repositories' into 'master' 

Add and backfill namespace_id for design_management_repositories

See merge request https://gitlab.com/gitlab-org/gitlab/-/merge_requests/153150



Merged-by: Adam Hegyi <ahegyi@gitlab.com>
Approved-by: Manoj M J <mmj@gitlab.com>
Approved-by: Adam Hegyi <ahegyi@gitlab.com>
Reviewed-by: Huzaifa Iftikhar <hiftikhar@gitlab.com>
Co-authored-by: Abhilash Kotte <akotte@gitlab.com>

db/docs/batched_background_migrations/backfill_design_management_repositories_namespace_id.yml

+---
+migration_job_name: BackfillDesignManagementRepositoriesNamespaceId
+description: Backfills sharding key `design_management_repositories.namespace_id` from `projects`.
+feature_category: design_management
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/153150
+milestone: '17.1'
+queued_migration_version: 20240515155723
+finalize_after: '2024-06-22'
+finalized_by: # version of the migration that finalized this BBM

db/docs/design_management_repositories.yml

@@ -23,3 +23,4 @@ desired_sharding_key:
         table: projects
         sharding_key: namespace_id
         belongs_to: project
+desired_sharding_key_migration_job_name: BackfillDesignManagementRepositoriesNamespaceId

db/migrate/20240515155719_add_namespace_id_to_design_management_repositories.rb

+# frozen_string_literal: true
+
+class AddNamespaceIdToDesignManagementRepositories < Gitlab::Database::Migration[2.2]
+  milestone '17.1'
+
+  def change
+    add_column :design_management_repositories, :namespace_id, :bigint
+  end
+end

db/post_migrate/20240515155720_index_design_management_repositories_on_namespace_id.rb

+# frozen_string_literal: true
+
+class IndexDesignManagementRepositoriesOnNamespaceId < Gitlab::Database::Migration[2.2]
+  milestone '17.1'
+  disable_ddl_transaction!
+
+  INDEX_NAME = 'index_design_management_repositories_on_namespace_id'
+
+  def up
+    add_concurrent_index :design_management_repositories, :namespace_id, name: INDEX_NAME
+  end
+
+  def down
+    remove_concurrent_index_by_name :design_management_repositories, INDEX_NAME
+  end
+end

db/post_migrate/20240515155721_add_design_management_repositories_namespace_id_fk.rb

+# frozen_string_literal: true
+
+class AddDesignManagementRepositoriesNamespaceIdFk < Gitlab::Database::Migration[2.2]
+  milestone '17.1'
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_foreign_key :design_management_repositories, :namespaces, column: :namespace_id, on_delete: :cascade
+  end
+
+  def down
+    with_lock_retries do
+      remove_foreign_key :design_management_repositories, column: :namespace_id
+    end
+  end
+end

db/post_migrate/20240515155722_add_design_management_repositories_namespace_id_trigger.rb

+# frozen_string_literal: true
+
+class AddDesignManagementRepositoriesNamespaceIdTrigger < Gitlab::Database::Migration[2.2]
+  milestone '17.1'
+
+  def up
+    install_sharding_key_assignment_trigger(
+      table: :design_management_repositories,
+      sharding_key: :namespace_id,
+      parent_table: :projects,
+      parent_sharding_key: :namespace_id,
+      foreign_key: :project_id
+    )
+  end
+
+  def down
+    remove_sharding_key_assignment_trigger(
+      table: :design_management_repositories,
+      sharding_key: :namespace_id,
+      parent_table: :projects,
+      parent_sharding_key: :namespace_id,
+      foreign_key: :project_id
+    )
+  end
+end

db/post_migrate/20240515155723_queue_backfill_design_management_repositories_namespace_id.rb

+# frozen_string_literal: true
+
+class QueueBackfillDesignManagementRepositoriesNamespaceId < Gitlab::Database::Migration[2.2]
+  milestone '17.1'
+  restrict_gitlab_migration gitlab_schema: :gitlab_main_cell
+
+  MIGRATION = "BackfillDesignManagementRepositoriesNamespaceId"
+  DELAY_INTERVAL = 2.minutes
+  BATCH_SIZE = 1000
+  SUB_BATCH_SIZE = 100
+
+  def up
+    queue_batched_background_migration(
+      MIGRATION,
+      :design_management_repositories,
+      :id,
+      :namespace_id,
+      :projects,
+      :namespace_id,
+      :project_id,
+      job_interval: DELAY_INTERVAL,
+      batch_size: BATCH_SIZE,
+      sub_batch_size: SUB_BATCH_SIZE
+    )
+  end
+
+  def down
+    delete_batched_background_migration(
+      MIGRATION,
+      :design_management_repositories,
+      :id,
+      [
+        :namespace_id,
+        :projects,
+        :namespace_id,
+        :project_id
+      ]
+    )
+  end
+end

db/schema_migrations/20240515155719

+b60f74cd823f86acd4ac7a0f50c2de2b366c78622f0b7baa92da7211d50de8df
\ No newline at end of file

db/schema_migrations/20240515155720

+00b8a22695a3d87a6c081511203515b304f91f2df09dfbaca580d41c71292667
\ No newline at end of file

db/schema_migrations/20240515155721

+374b03647d2a1bdc3d9cab6fef927efcf78f7cf9884780d90fb9b5ec4682491a
\ No newline at end of file

db/schema_migrations/20240515155722

+64d891e6d08b3ae175c83ba40d86ea72a5ce6a6232e1412e290c8ca53eeed323
\ No newline at end of file

db/schema_migrations/20240515155723

+61a43adbe0aa1a50946b6637611ce10104966ad7108a956b52cc4120b719c434
\ No newline at end of file

db/structure.sql

@@ -897,6 +897,22 @@ RETURN NEW;
 END
 $$;
+CREATE FUNCTION trigger_8ac78f164b2d() RETURNS trigger
+    LANGUAGE plpgsql
+    AS $$
+BEGIN
+IF NEW."namespace_id" IS NULL THEN
+  SELECT "namespace_id"
+  INTO NEW."namespace_id"
+  FROM "projects"
+  WHERE "projects"."id" = NEW."project_id";
+END IF;
+
+RETURN NEW;
+
+END
+$$;
+
 CREATE FUNCTION trigger_94514aeadc50() RETURNS trigger
     LANGUAGE plpgsql
     AS $$
@@ -8695,7 +8711,8 @@ CREATE TABLE design_management_repositories (
     id bigint NOT NULL,
     project_id bigint NOT NULL,
     created_at timestamp with time zone NOT NULL,
-    updated_at timestamp with time zone NOT NULL
+    updated_at timestamp with time zone NOT NULL,
+    namespace_id bigint
 );
 CREATE SEQUENCE design_management_repositories_id_seq
@@ -25739,6 +25756,8 @@ CREATE INDEX index_design_management_designs_versions_on_event ON design_managem
 CREATE INDEX index_design_management_designs_versions_on_version_id ON design_management_designs_versions USING btree (version_id);
+CREATE INDEX index_design_management_repositories_on_namespace_id ON design_management_repositories USING btree (namespace_id);
+
 CREATE UNIQUE INDEX index_design_management_repositories_on_project_id ON design_management_repositories USING btree (project_id);
 CREATE INDEX index_design_management_repository_states_failed_verification ON design_management_repository_states USING btree (verification_retry_at NULLS FIRST) WHERE (verification_state = 3);
@@ -30207,6 +30226,8 @@ CREATE TRIGGER trigger_8e66b994e8f0 BEFORE INSERT OR UPDATE ON audit_events_stre
 CREATE TRIGGER trigger_8fbb044c64ad BEFORE INSERT OR UPDATE ON design_management_designs FOR EACH ROW EXECUTE FUNCTION trigger_8fbb044c64ad();
+CREATE TRIGGER trigger_8ac78f164b2d BEFORE INSERT OR UPDATE ON design_management_repositories FOR EACH ROW EXECUTE FUNCTION trigger_8ac78f164b2d();
+
 CREATE TRIGGER trigger_94514aeadc50 BEFORE INSERT OR UPDATE ON deployment_approvals FOR EACH ROW EXECUTE FUNCTION trigger_94514aeadc50();
 CREATE TRIGGER trigger_b4520c29ea74 BEFORE INSERT OR UPDATE ON approval_merge_request_rule_sources FOR EACH ROW EXECUTE FUNCTION trigger_b4520c29ea74();
@@ -30508,6 +30529,9 @@ ALTER TABLE ONLY approvals
 ALTER TABLE ONLY namespaces
     ADD CONSTRAINT fk_319256d87a FOREIGN KEY (file_template_project_id) REFERENCES projects(id) ON DELETE SET NULL;
+ALTER TABLE ONLY design_management_repositories
+    ADD CONSTRAINT fk_335d4698e2 FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY issue_tracker_data
     ADD CONSTRAINT fk_33921c0ee1 FOREIGN KEY (integration_id) REFERENCES integrations(id) ON DELETE CASCADE;

lib/gitlab/background_migration/backfill_design_management_repositories_namespace_id.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # rubocop: disable Migration/BackgroundMigrationBaseClass -- BackfillDesiredShardingKeyJob inherits from BatchedMigrationJob.
+    class BackfillDesignManagementRepositoriesNamespaceId < BackfillDesiredShardingKeyJob
+      operation_name :backfill_design_management_repositories_namespace_id
+      feature_category :design_management
+    end
+    # rubocop: enable Migration/BackgroundMigrationBaseClass
+  end
+end

spec/lib/gitlab/background_migration/backfill_design_management_repositories_namespace_id_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::BackfillDesignManagementRepositoriesNamespaceId,
+  feature_category: :design_management,
+  schema: 20240515155719 do
+  include_examples 'desired sharding key backfill job' do
+    let(:batch_table) { :design_management_repositories }
+    let(:backfill_column) { :namespace_id }
+    let(:backfill_via_table) { :projects }
+    let(:backfill_via_column) { :namespace_id }
+    let(:backfill_via_foreign_key) { :project_id }
+  end
+end

spec/migrations/20240515155723_queue_backfill_design_management_repositories_namespace_id_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe QueueBackfillDesignManagementRepositoriesNamespaceId, feature_category: :design_management do
+  let!(:batched_migration) { described_class::MIGRATION }
+
+  it 'schedules a new batched migration' do
+    reversible_migration do |migration|
+      migration.before -> {
+        expect(batched_migration).not_to have_scheduled_batched_migration
+      }
+
+      migration.after -> {
+        expect(batched_migration).to have_scheduled_batched_migration(
+          table_name: :design_management_repositories,
+          column_name: :id,
+          interval: described_class::DELAY_INTERVAL,
+          batch_size: described_class::BATCH_SIZE,
+          sub_batch_size: described_class::SUB_BATCH_SIZE,
+          gitlab_schema: :gitlab_main_cell,
+          job_arguments: [
+            :namespace_id,
+            :projects,
+            :namespace_id,
+            :project_id
+          ]
+        )
+      }
+    end
+  end
+end