diff --git a/db/docs/batched_background_migrations/backfill_packages_debian_group_components_group_id.yml b/db/docs/batched_background_migrations/backfill_packages_debian_group_components_group_id.yml new file mode 100644 index 0000000000000000000000000000000000000000..5d2b95767104cdb3538fc67b7a24a554cc88443e --- /dev/null +++ b/db/docs/batched_background_migrations/backfill_packages_debian_group_components_group_id.yml @@ -0,0 +1,9 @@ +--- +migration_job_name: BackfillPackagesDebianGroupComponentsGroupId +description: Backfills sharding key `packages_debian_group_components.group_id` from `packages_debian_group_distributions`. +feature_category: package_registry +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/157905 +milestone: '17.2' +queued_migration_version: 20240701074852 +finalize_after: '2024-08-22' +finalized_by: # version of the migration that finalized this BBM diff --git a/db/docs/packages_debian_group_components.yml b/db/docs/packages_debian_group_components.yml index a01681b67506b0331ff2417357f07acc9fa7f459..ff3e6718d493f3fa85e6e5ef6e2227b9cb7749f1 100644 --- a/db/docs/packages_debian_group_components.yml +++ b/db/docs/packages_debian_group_components.yml @@ -19,3 +19,4 @@ desired_sharding_key: table: packages_debian_group_distributions sharding_key: group_id belongs_to: distribution +desired_sharding_key_migration_job_name: BackfillPackagesDebianGroupComponentsGroupId diff --git a/db/migrate/20240701074848_add_group_id_to_packages_debian_group_components.rb b/db/migrate/20240701074848_add_group_id_to_packages_debian_group_components.rb new file mode 100644 index 0000000000000000000000000000000000000000..787227dcab43bdd2d0aef7bd5f929ddb710b41ed --- /dev/null +++ b/db/migrate/20240701074848_add_group_id_to_packages_debian_group_components.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class AddGroupIdToPackagesDebianGroupComponents < Gitlab::Database::Migration[2.2] + milestone '17.2' + + def change + add_column :packages_debian_group_components, :group_id, :bigint + end +end diff --git a/db/post_migrate/20240701074849_index_packages_debian_group_components_on_group_id.rb b/db/post_migrate/20240701074849_index_packages_debian_group_components_on_group_id.rb new file mode 100644 index 0000000000000000000000000000000000000000..429d977475e172b3174be988ee2fb09f4b9025e1 --- /dev/null +++ b/db/post_migrate/20240701074849_index_packages_debian_group_components_on_group_id.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class IndexPackagesDebianGroupComponentsOnGroupId < Gitlab::Database::Migration[2.2] + milestone '17.2' + disable_ddl_transaction! + + INDEX_NAME = 'index_packages_debian_group_components_on_group_id' + + def up + add_concurrent_index :packages_debian_group_components, :group_id, name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :packages_debian_group_components, INDEX_NAME + end +end diff --git a/db/post_migrate/20240701074850_add_packages_debian_group_components_group_id_fk.rb b/db/post_migrate/20240701074850_add_packages_debian_group_components_group_id_fk.rb new file mode 100644 index 0000000000000000000000000000000000000000..b42e6781d338d0982bf17670f739b84def8a1b7b --- /dev/null +++ b/db/post_migrate/20240701074850_add_packages_debian_group_components_group_id_fk.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class AddPackagesDebianGroupComponentsGroupIdFk < Gitlab::Database::Migration[2.2] + milestone '17.2' + disable_ddl_transaction! + + def up + add_concurrent_foreign_key :packages_debian_group_components, :namespaces, column: :group_id, on_delete: :cascade + end + + def down + with_lock_retries do + remove_foreign_key :packages_debian_group_components, column: :group_id + end + end +end diff --git a/db/post_migrate/20240701074851_add_packages_debian_group_components_group_id_trigger.rb b/db/post_migrate/20240701074851_add_packages_debian_group_components_group_id_trigger.rb new file mode 100644 index 0000000000000000000000000000000000000000..e173adf8f048204ee1e771a3b35930119149bac9 --- /dev/null +++ b/db/post_migrate/20240701074851_add_packages_debian_group_components_group_id_trigger.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class AddPackagesDebianGroupComponentsGroupIdTrigger < Gitlab::Database::Migration[2.2] + milestone '17.2' + + def up + install_sharding_key_assignment_trigger( + table: :packages_debian_group_components, + sharding_key: :group_id, + parent_table: :packages_debian_group_distributions, + parent_sharding_key: :group_id, + foreign_key: :distribution_id + ) + end + + def down + remove_sharding_key_assignment_trigger( + table: :packages_debian_group_components, + sharding_key: :group_id, + parent_table: :packages_debian_group_distributions, + parent_sharding_key: :group_id, + foreign_key: :distribution_id + ) + end +end diff --git a/db/post_migrate/20240701074852_queue_backfill_packages_debian_group_components_group_id.rb b/db/post_migrate/20240701074852_queue_backfill_packages_debian_group_components_group_id.rb new file mode 100644 index 0000000000000000000000000000000000000000..3a1585e7b8fcaa8ec6adc23db6e972848006316d --- /dev/null +++ b/db/post_migrate/20240701074852_queue_backfill_packages_debian_group_components_group_id.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +class QueueBackfillPackagesDebianGroupComponentsGroupId < Gitlab::Database::Migration[2.2] + milestone '17.2' + restrict_gitlab_migration gitlab_schema: :gitlab_main_cell + + MIGRATION = "BackfillPackagesDebianGroupComponentsGroupId" + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 100 + + def up + queue_batched_background_migration( + MIGRATION, + :packages_debian_group_components, + :id, + :group_id, + :packages_debian_group_distributions, + :group_id, + :distribution_id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration( + MIGRATION, + :packages_debian_group_components, + :id, + [ + :group_id, + :packages_debian_group_distributions, + :group_id, + :distribution_id + ] + ) + end +end diff --git a/db/schema_migrations/20240701074848 b/db/schema_migrations/20240701074848 new file mode 100644 index 0000000000000000000000000000000000000000..ae4b9e521d759591613984955e0dc4df6e1cfe45 --- /dev/null +++ b/db/schema_migrations/20240701074848 @@ -0,0 +1 @@ +581c79b6e0faa4a2e17211a1732514ed3e0fab959a8ac541cc1431bafc461a73 \ No newline at end of file diff --git a/db/schema_migrations/20240701074849 b/db/schema_migrations/20240701074849 new file mode 100644 index 0000000000000000000000000000000000000000..1eacf7c8c6ee64646a237fb475fe504669a2087f --- /dev/null +++ b/db/schema_migrations/20240701074849 @@ -0,0 +1 @@ +0021e3a5f6b67e11435a04c2d585abb37f7026dd647d0814c9b3b11f6d334a69 \ No newline at end of file diff --git a/db/schema_migrations/20240701074850 b/db/schema_migrations/20240701074850 new file mode 100644 index 0000000000000000000000000000000000000000..14368c329346083d65ce3997115f72729c863497 --- /dev/null +++ b/db/schema_migrations/20240701074850 @@ -0,0 +1 @@ +3642e9ba86c2060b2938eaa81b19b76a7000ce9a76eab03bd4b5093b170804c4 \ No newline at end of file diff --git a/db/schema_migrations/20240701074851 b/db/schema_migrations/20240701074851 new file mode 100644 index 0000000000000000000000000000000000000000..bcf8a79430f713da92d5ffa179690f981d8d09a0 --- /dev/null +++ b/db/schema_migrations/20240701074851 @@ -0,0 +1 @@ +2d77cf72555aa598191c1764ebe14f2be6da97737761d3adb9131207589eea85 \ No newline at end of file diff --git a/db/schema_migrations/20240701074852 b/db/schema_migrations/20240701074852 new file mode 100644 index 0000000000000000000000000000000000000000..75bd768fa4b222e2c08d4664858bfa2ba4dc9741 --- /dev/null +++ b/db/schema_migrations/20240701074852 @@ -0,0 +1 @@ +40ebb133d35e36ea503f1ac50770d863d8d9d433f4c45ac64501c169021a37aa \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 5ad25ea556585f034558a940fcfe642260f2be85..9f5539520812437f5445c70ff79fd74fd94fa573 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -1329,6 +1329,22 @@ RETURN NEW; END $$; +CREATE FUNCTION trigger_8d002f38bdef() RETURNS trigger + LANGUAGE plpgsql + AS $$ +BEGIN +IF NEW."group_id" IS NULL THEN + SELECT "group_id" + INTO NEW."group_id" + FROM "packages_debian_group_distributions" + WHERE "packages_debian_group_distributions"."id" = NEW."distribution_id"; +END IF; + +RETURN NEW; + +END +$$; + CREATE FUNCTION trigger_8d17725116fe() RETURNS trigger LANGUAGE plpgsql AS $$ @@ -13955,6 +13971,7 @@ CREATE TABLE packages_debian_group_components ( updated_at timestamp with time zone NOT NULL, distribution_id bigint NOT NULL, name text NOT NULL, + group_id bigint, CONSTRAINT check_a9bc7d85be CHECK ((char_length(name) <= 255)) ); @@ -28112,6 +28129,8 @@ CREATE INDEX index_packages_conan_metadata_on_project_id ON packages_conan_metad CREATE INDEX index_packages_debian_group_component_files_on_component_id ON packages_debian_group_component_files USING btree (component_id); +CREATE INDEX index_packages_debian_group_components_on_group_id ON packages_debian_group_components USING btree (group_id); + CREATE INDEX index_packages_debian_group_distribution_keys_on_group_id ON packages_debian_group_distribution_keys USING btree (group_id); CREATE INDEX index_packages_debian_group_distributions_on_creator_id ON packages_debian_group_distributions USING btree (creator_id); @@ -31530,6 +31549,8 @@ CREATE TRIGGER trigger_8a38ce2327de BEFORE INSERT OR UPDATE ON boards_epic_user_ CREATE TRIGGER trigger_8ac78f164b2d BEFORE INSERT OR UPDATE ON design_management_repositories FOR EACH ROW EXECUTE FUNCTION trigger_8ac78f164b2d(); +CREATE TRIGGER trigger_8d002f38bdef BEFORE INSERT OR UPDATE ON packages_debian_group_components FOR EACH ROW EXECUTE FUNCTION trigger_8d002f38bdef(); + CREATE TRIGGER trigger_8d17725116fe BEFORE INSERT OR UPDATE ON merge_request_reviewers FOR EACH ROW EXECUTE FUNCTION trigger_8d17725116fe(); CREATE TRIGGER trigger_8e66b994e8f0 BEFORE INSERT OR UPDATE ON audit_events_streaming_event_type_filters FOR EACH ROW EXECUTE FUNCTION trigger_8e66b994e8f0(); @@ -32937,6 +32958,9 @@ ALTER TABLE ONLY abuse_events ALTER TABLE ONLY user_preferences ADD CONSTRAINT fk_e5e029c10b FOREIGN KEY (home_organization_id) REFERENCES organizations(id) ON DELETE SET NULL; +ALTER TABLE ONLY packages_debian_group_components + ADD CONSTRAINT fk_e63e8ee3b1 FOREIGN KEY (group_id) REFERENCES namespaces(id) ON DELETE CASCADE; + ALTER TABLE ONLY merge_requests ADD CONSTRAINT fk_e719a85f8a FOREIGN KEY (author_id) REFERENCES users(id) ON DELETE SET NULL; diff --git a/lib/gitlab/background_migration/backfill_packages_debian_group_components_group_id.rb b/lib/gitlab/background_migration/backfill_packages_debian_group_components_group_id.rb new file mode 100644 index 0000000000000000000000000000000000000000..28f229fd0fde83810c7a656fba5518d33e52bdb1 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_packages_debian_group_components_group_id.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class BackfillPackagesDebianGroupComponentsGroupId < BackfillDesiredShardingKeyJob + operation_name :backfill_packages_debian_group_components_group_id + feature_category :package_registry + end + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_packages_debian_group_components_group_id_spec.rb b/spec/lib/gitlab/background_migration/backfill_packages_debian_group_components_group_id_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..bbac7a70a16200770864588057d407dc762d7440 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_packages_debian_group_components_group_id_spec.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillPackagesDebianGroupComponentsGroupId, + feature_category: :package_registry, + schema: 20240701074848 do + include_examples 'desired sharding key backfill job' do + let(:batch_table) { :packages_debian_group_components } + let(:backfill_column) { :group_id } + let(:backfill_via_table) { :packages_debian_group_distributions } + let(:backfill_via_column) { :group_id } + let(:backfill_via_foreign_key) { :distribution_id } + end +end diff --git a/spec/migrations/20240701074852_queue_backfill_packages_debian_group_components_group_id_spec.rb b/spec/migrations/20240701074852_queue_backfill_packages_debian_group_components_group_id_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..d7bf5e85e36e7295aecf8368a9abb25ae11ae47a --- /dev/null +++ b/spec/migrations/20240701074852_queue_backfill_packages_debian_group_components_group_id_spec.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueBackfillPackagesDebianGroupComponentsGroupId, feature_category: :package_registry do + let!(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + table_name: :packages_debian_group_components, + column_name: :id, + interval: described_class::DELAY_INTERVAL, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE, + gitlab_schema: :gitlab_main_cell, + job_arguments: [ + :group_id, + :packages_debian_group_distributions, + :group_id, + :distribution_id + ] + ) + } + end + end +end