diff --git a/db/docs/batched_background_migrations/backfill_design_management_repositories_namespace_id.yml b/db/docs/batched_background_migrations/backfill_design_management_repositories_namespace_id.yml new file mode 100644 index 0000000000000000000000000000000000000000..ce61879111d80ec06de5e9e59efead03c46852b9 --- /dev/null +++ b/db/docs/batched_background_migrations/backfill_design_management_repositories_namespace_id.yml @@ -0,0 +1,9 @@ +--- +migration_job_name: BackfillDesignManagementRepositoriesNamespaceId +description: Backfills sharding key `design_management_repositories.namespace_id` from `projects`. +feature_category: design_management +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/153150 +milestone: '17.1' +queued_migration_version: 20240515155723 +finalize_after: '2024-06-22' +finalized_by: # version of the migration that finalized this BBM diff --git a/db/docs/design_management_repositories.yml b/db/docs/design_management_repositories.yml index 798032b8b073f0b391300622aba6a4412678426a..4c02a74a7bf58a244f1d47d2d6ffeedca48826a5 100644 --- a/db/docs/design_management_repositories.yml +++ b/db/docs/design_management_repositories.yml @@ -23,3 +23,4 @@ desired_sharding_key: table: projects sharding_key: namespace_id belongs_to: project +desired_sharding_key_migration_job_name: BackfillDesignManagementRepositoriesNamespaceId diff --git a/db/migrate/20240515155719_add_namespace_id_to_design_management_repositories.rb b/db/migrate/20240515155719_add_namespace_id_to_design_management_repositories.rb new file mode 100644 index 0000000000000000000000000000000000000000..4677ed2e237c7c60b2468611195ebda35efa6a03 --- /dev/null +++ b/db/migrate/20240515155719_add_namespace_id_to_design_management_repositories.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class AddNamespaceIdToDesignManagementRepositories < Gitlab::Database::Migration[2.2] + milestone '17.1' + + def change + add_column :design_management_repositories, :namespace_id, :bigint + end +end diff --git a/db/post_migrate/20240515155720_index_design_management_repositories_on_namespace_id.rb b/db/post_migrate/20240515155720_index_design_management_repositories_on_namespace_id.rb new file mode 100644 index 0000000000000000000000000000000000000000..150753283ab9616e56546cb04f5181e7e9951cfe --- /dev/null +++ b/db/post_migrate/20240515155720_index_design_management_repositories_on_namespace_id.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class IndexDesignManagementRepositoriesOnNamespaceId < Gitlab::Database::Migration[2.2] + milestone '17.1' + disable_ddl_transaction! + + INDEX_NAME = 'index_design_management_repositories_on_namespace_id' + + def up + add_concurrent_index :design_management_repositories, :namespace_id, name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :design_management_repositories, INDEX_NAME + end +end diff --git a/db/post_migrate/20240515155721_add_design_management_repositories_namespace_id_fk.rb b/db/post_migrate/20240515155721_add_design_management_repositories_namespace_id_fk.rb new file mode 100644 index 0000000000000000000000000000000000000000..854b3a94c35f9c02af0d94fbee7f9342605fac53 --- /dev/null +++ b/db/post_migrate/20240515155721_add_design_management_repositories_namespace_id_fk.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class AddDesignManagementRepositoriesNamespaceIdFk < Gitlab::Database::Migration[2.2] + milestone '17.1' + disable_ddl_transaction! + + def up + add_concurrent_foreign_key :design_management_repositories, :namespaces, column: :namespace_id, on_delete: :cascade + end + + def down + with_lock_retries do + remove_foreign_key :design_management_repositories, column: :namespace_id + end + end +end diff --git a/db/post_migrate/20240515155722_add_design_management_repositories_namespace_id_trigger.rb b/db/post_migrate/20240515155722_add_design_management_repositories_namespace_id_trigger.rb new file mode 100644 index 0000000000000000000000000000000000000000..e4021c317c63664945b93b77f1d535c2adbf47f5 --- /dev/null +++ b/db/post_migrate/20240515155722_add_design_management_repositories_namespace_id_trigger.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class AddDesignManagementRepositoriesNamespaceIdTrigger < Gitlab::Database::Migration[2.2] + milestone '17.1' + + def up + install_sharding_key_assignment_trigger( + table: :design_management_repositories, + sharding_key: :namespace_id, + parent_table: :projects, + parent_sharding_key: :namespace_id, + foreign_key: :project_id + ) + end + + def down + remove_sharding_key_assignment_trigger( + table: :design_management_repositories, + sharding_key: :namespace_id, + parent_table: :projects, + parent_sharding_key: :namespace_id, + foreign_key: :project_id + ) + end +end diff --git a/db/post_migrate/20240515155723_queue_backfill_design_management_repositories_namespace_id.rb b/db/post_migrate/20240515155723_queue_backfill_design_management_repositories_namespace_id.rb new file mode 100644 index 0000000000000000000000000000000000000000..ed604392ab62e0e5099418effd9a72460da0ad8f --- /dev/null +++ b/db/post_migrate/20240515155723_queue_backfill_design_management_repositories_namespace_id.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +class QueueBackfillDesignManagementRepositoriesNamespaceId < Gitlab::Database::Migration[2.2] + milestone '17.1' + restrict_gitlab_migration gitlab_schema: :gitlab_main_cell + + MIGRATION = "BackfillDesignManagementRepositoriesNamespaceId" + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 100 + + def up + queue_batched_background_migration( + MIGRATION, + :design_management_repositories, + :id, + :namespace_id, + :projects, + :namespace_id, + :project_id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration( + MIGRATION, + :design_management_repositories, + :id, + [ + :namespace_id, + :projects, + :namespace_id, + :project_id + ] + ) + end +end diff --git a/db/schema_migrations/20240515155719 b/db/schema_migrations/20240515155719 new file mode 100644 index 0000000000000000000000000000000000000000..81b2dcf28f0ca97fac87ee823f1492867b397c55 --- /dev/null +++ b/db/schema_migrations/20240515155719 @@ -0,0 +1 @@ +b60f74cd823f86acd4ac7a0f50c2de2b366c78622f0b7baa92da7211d50de8df \ No newline at end of file diff --git a/db/schema_migrations/20240515155720 b/db/schema_migrations/20240515155720 new file mode 100644 index 0000000000000000000000000000000000000000..90c5beb6de30fa8dca17fdd35c3393e29c1bc560 --- /dev/null +++ b/db/schema_migrations/20240515155720 @@ -0,0 +1 @@ +00b8a22695a3d87a6c081511203515b304f91f2df09dfbaca580d41c71292667 \ No newline at end of file diff --git a/db/schema_migrations/20240515155721 b/db/schema_migrations/20240515155721 new file mode 100644 index 0000000000000000000000000000000000000000..6a93674832c8164024f95e6efddc37e815f1c431 --- /dev/null +++ b/db/schema_migrations/20240515155721 @@ -0,0 +1 @@ +374b03647d2a1bdc3d9cab6fef927efcf78f7cf9884780d90fb9b5ec4682491a \ No newline at end of file diff --git a/db/schema_migrations/20240515155722 b/db/schema_migrations/20240515155722 new file mode 100644 index 0000000000000000000000000000000000000000..6670c20aa8c50a6c86b18190f76553e99e2123c0 --- /dev/null +++ b/db/schema_migrations/20240515155722 @@ -0,0 +1 @@ +64d891e6d08b3ae175c83ba40d86ea72a5ce6a6232e1412e290c8ca53eeed323 \ No newline at end of file diff --git a/db/schema_migrations/20240515155723 b/db/schema_migrations/20240515155723 new file mode 100644 index 0000000000000000000000000000000000000000..8a7700670e81ca425c98bbad5a1d0f3ed6ffe3b2 --- /dev/null +++ b/db/schema_migrations/20240515155723 @@ -0,0 +1 @@ +61a43adbe0aa1a50946b6637611ce10104966ad7108a956b52cc4120b719c434 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index bbdd935afda6725cb7a91c594f936064f92784f1..37e328ebba5556f37d1d20c3d40f90c9b6c99c47 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -897,6 +897,22 @@ RETURN NEW; END $$; +CREATE FUNCTION trigger_8ac78f164b2d() RETURNS trigger + LANGUAGE plpgsql + AS $$ +BEGIN +IF NEW."namespace_id" IS NULL THEN + SELECT "namespace_id" + INTO NEW."namespace_id" + FROM "projects" + WHERE "projects"."id" = NEW."project_id"; +END IF; + +RETURN NEW; + +END +$$; + CREATE FUNCTION trigger_94514aeadc50() RETURNS trigger LANGUAGE plpgsql AS $$ @@ -8695,7 +8711,8 @@ CREATE TABLE design_management_repositories ( id bigint NOT NULL, project_id bigint NOT NULL, created_at timestamp with time zone NOT NULL, - updated_at timestamp with time zone NOT NULL + updated_at timestamp with time zone NOT NULL, + namespace_id bigint ); CREATE SEQUENCE design_management_repositories_id_seq @@ -25739,6 +25756,8 @@ CREATE INDEX index_design_management_designs_versions_on_event ON design_managem CREATE INDEX index_design_management_designs_versions_on_version_id ON design_management_designs_versions USING btree (version_id); +CREATE INDEX index_design_management_repositories_on_namespace_id ON design_management_repositories USING btree (namespace_id); + CREATE UNIQUE INDEX index_design_management_repositories_on_project_id ON design_management_repositories USING btree (project_id); CREATE INDEX index_design_management_repository_states_failed_verification ON design_management_repository_states USING btree (verification_retry_at NULLS FIRST) WHERE (verification_state = 3); @@ -30207,6 +30226,8 @@ CREATE TRIGGER trigger_8e66b994e8f0 BEFORE INSERT OR UPDATE ON audit_events_stre CREATE TRIGGER trigger_8fbb044c64ad BEFORE INSERT OR UPDATE ON design_management_designs FOR EACH ROW EXECUTE FUNCTION trigger_8fbb044c64ad(); +CREATE TRIGGER trigger_8ac78f164b2d BEFORE INSERT OR UPDATE ON design_management_repositories FOR EACH ROW EXECUTE FUNCTION trigger_8ac78f164b2d(); + CREATE TRIGGER trigger_94514aeadc50 BEFORE INSERT OR UPDATE ON deployment_approvals FOR EACH ROW EXECUTE FUNCTION trigger_94514aeadc50(); CREATE TRIGGER trigger_b4520c29ea74 BEFORE INSERT OR UPDATE ON approval_merge_request_rule_sources FOR EACH ROW EXECUTE FUNCTION trigger_b4520c29ea74(); @@ -30508,6 +30529,9 @@ ALTER TABLE ONLY approvals ALTER TABLE ONLY namespaces ADD CONSTRAINT fk_319256d87a FOREIGN KEY (file_template_project_id) REFERENCES projects(id) ON DELETE SET NULL; +ALTER TABLE ONLY design_management_repositories + ADD CONSTRAINT fk_335d4698e2 FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE; + ALTER TABLE ONLY issue_tracker_data ADD CONSTRAINT fk_33921c0ee1 FOREIGN KEY (integration_id) REFERENCES integrations(id) ON DELETE CASCADE; diff --git a/lib/gitlab/background_migration/backfill_design_management_repositories_namespace_id.rb b/lib/gitlab/background_migration/backfill_design_management_repositories_namespace_id.rb new file mode 100644 index 0000000000000000000000000000000000000000..7a1435b55b0cd4ef673fe8d5f4b9c225c3cc29d9 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_design_management_repositories_namespace_id.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # rubocop: disable Migration/BackgroundMigrationBaseClass -- BackfillDesiredShardingKeyJob inherits from BatchedMigrationJob. + class BackfillDesignManagementRepositoriesNamespaceId < BackfillDesiredShardingKeyJob + operation_name :backfill_design_management_repositories_namespace_id + feature_category :design_management + end + # rubocop: enable Migration/BackgroundMigrationBaseClass + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_design_management_repositories_namespace_id_spec.rb b/spec/lib/gitlab/background_migration/backfill_design_management_repositories_namespace_id_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..002f1b1078e998dafffeaf522dfbab0ce6c73dec --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_design_management_repositories_namespace_id_spec.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillDesignManagementRepositoriesNamespaceId, + feature_category: :design_management, + schema: 20240515155719 do + include_examples 'desired sharding key backfill job' do + let(:batch_table) { :design_management_repositories } + let(:backfill_column) { :namespace_id } + let(:backfill_via_table) { :projects } + let(:backfill_via_column) { :namespace_id } + let(:backfill_via_foreign_key) { :project_id } + end +end diff --git a/spec/migrations/20240515155723_queue_backfill_design_management_repositories_namespace_id_spec.rb b/spec/migrations/20240515155723_queue_backfill_design_management_repositories_namespace_id_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..19a23dade6b7b004d6d36372df11cdbd5080eb6e --- /dev/null +++ b/spec/migrations/20240515155723_queue_backfill_design_management_repositories_namespace_id_spec.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueBackfillDesignManagementRepositoriesNamespaceId, feature_category: :design_management do + let!(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + table_name: :design_management_repositories, + column_name: :id, + interval: described_class::DELAY_INTERVAL, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE, + gitlab_schema: :gitlab_main_cell, + job_arguments: [ + :namespace_id, + :projects, + :namespace_id, + :project_id + ] + ) + } + end + end +end