diff --git a/db/docs/batched_background_migrations/backfill_issuable_slas_namespace_id.yml b/db/docs/batched_background_migrations/backfill_issuable_slas_namespace_id.yml new file mode 100644 index 0000000000000000000000000000000000000000..169c91f2e007098fb0e7bcb664542e94ad8b2c45 --- /dev/null +++ b/db/docs/batched_background_migrations/backfill_issuable_slas_namespace_id.yml @@ -0,0 +1,8 @@ +--- +migration_job_name: BackfillIssuableSlasNamespaceId +description: Backfills sharding key `issuable_slas.namespace_id` from `issues`. +feature_category: incident_management +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/175719 +milestone: '17.8' +queued_migration_version: 20241213142263 +finalized_by: # version of the migration that finalized this BBM diff --git a/db/docs/issuable_slas.yml b/db/docs/issuable_slas.yml index 7df1d2a5a06335935a9544b57b80b7cf604c8723..c157c08f46a72312cf5c5b662e8f203c8dd8f505 100644 --- a/db/docs/issuable_slas.yml +++ b/db/docs/issuable_slas.yml @@ -18,3 +18,4 @@ desired_sharding_key: sharding_key: namespace_id belongs_to: issue table_size: small +desired_sharding_key_migration_job_name: BackfillIssuableSlasNamespaceId diff --git a/db/migrate/20241213142259_add_namespace_id_to_issuable_slas.rb b/db/migrate/20241213142259_add_namespace_id_to_issuable_slas.rb new file mode 100644 index 0000000000000000000000000000000000000000..f3d333619e26c214dea299cfffd22ba78ac20fcb --- /dev/null +++ b/db/migrate/20241213142259_add_namespace_id_to_issuable_slas.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class AddNamespaceIdToIssuableSlas < Gitlab::Database::Migration[2.2] + milestone '17.8' + + def change + add_column :issuable_slas, :namespace_id, :bigint + end +end diff --git a/db/post_migrate/20241213142260_index_issuable_slas_on_namespace_id.rb b/db/post_migrate/20241213142260_index_issuable_slas_on_namespace_id.rb new file mode 100644 index 0000000000000000000000000000000000000000..fecd58060aec06f045b3f353c8de88190e9caa6e --- /dev/null +++ b/db/post_migrate/20241213142260_index_issuable_slas_on_namespace_id.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class IndexIssuableSlasOnNamespaceId < Gitlab::Database::Migration[2.2] + milestone '17.8' + disable_ddl_transaction! + + INDEX_NAME = 'index_issuable_slas_on_namespace_id' + + def up + add_concurrent_index :issuable_slas, :namespace_id, name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :issuable_slas, INDEX_NAME + end +end diff --git a/db/post_migrate/20241213142261_add_issuable_slas_namespace_id_fk.rb b/db/post_migrate/20241213142261_add_issuable_slas_namespace_id_fk.rb new file mode 100644 index 0000000000000000000000000000000000000000..22216f29a4d7ee72a955fefa4dfd2bd2dcb2d026 --- /dev/null +++ b/db/post_migrate/20241213142261_add_issuable_slas_namespace_id_fk.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class AddIssuableSlasNamespaceIdFk < Gitlab::Database::Migration[2.2] + milestone '17.8' + disable_ddl_transaction! + + def up + add_concurrent_foreign_key :issuable_slas, :namespaces, column: :namespace_id, on_delete: :cascade + end + + def down + with_lock_retries do + remove_foreign_key :issuable_slas, column: :namespace_id + end + end +end diff --git a/db/post_migrate/20241213142262_add_issuable_slas_namespace_id_trigger.rb b/db/post_migrate/20241213142262_add_issuable_slas_namespace_id_trigger.rb new file mode 100644 index 0000000000000000000000000000000000000000..88b146c77b1ed6c9ca032d325044d491f2a813b1 --- /dev/null +++ b/db/post_migrate/20241213142262_add_issuable_slas_namespace_id_trigger.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class AddIssuableSlasNamespaceIdTrigger < Gitlab::Database::Migration[2.2] + milestone '17.8' + + def up + install_sharding_key_assignment_trigger( + table: :issuable_slas, + sharding_key: :namespace_id, + parent_table: :issues, + parent_sharding_key: :namespace_id, + foreign_key: :issue_id + ) + end + + def down + remove_sharding_key_assignment_trigger( + table: :issuable_slas, + sharding_key: :namespace_id, + parent_table: :issues, + parent_sharding_key: :namespace_id, + foreign_key: :issue_id + ) + end +end diff --git a/db/post_migrate/20241213142263_queue_backfill_issuable_slas_namespace_id.rb b/db/post_migrate/20241213142263_queue_backfill_issuable_slas_namespace_id.rb new file mode 100644 index 0000000000000000000000000000000000000000..0ae839d1ad8448adfb67293742712b92a68fae27 --- /dev/null +++ b/db/post_migrate/20241213142263_queue_backfill_issuable_slas_namespace_id.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +class QueueBackfillIssuableSlasNamespaceId < Gitlab::Database::Migration[2.2] + milestone '17.8' + restrict_gitlab_migration gitlab_schema: :gitlab_main_cell + + MIGRATION = "BackfillIssuableSlasNamespaceId" + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 100 + + def up + queue_batched_background_migration( + MIGRATION, + :issuable_slas, + :id, + :namespace_id, + :issues, + :namespace_id, + :issue_id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration( + MIGRATION, + :issuable_slas, + :id, + [ + :namespace_id, + :issues, + :namespace_id, + :issue_id + ] + ) + end +end diff --git a/db/schema_migrations/20241213142259 b/db/schema_migrations/20241213142259 new file mode 100644 index 0000000000000000000000000000000000000000..f7d0a9339fa5d7576f5a24cf3afcadf49748cd60 --- /dev/null +++ b/db/schema_migrations/20241213142259 @@ -0,0 +1 @@ +974aaa995416645bb17f250224e2eda3327dc94c9f2194e11219624f4cf37efc \ No newline at end of file diff --git a/db/schema_migrations/20241213142260 b/db/schema_migrations/20241213142260 new file mode 100644 index 0000000000000000000000000000000000000000..ac289713a72642b30a4e662ad02ad20162956d48 --- /dev/null +++ b/db/schema_migrations/20241213142260 @@ -0,0 +1 @@ +3c0f16d01dc521e3032111746b0b2469378548b8d5c80e5e9c3355cec6ad1fed \ No newline at end of file diff --git a/db/schema_migrations/20241213142261 b/db/schema_migrations/20241213142261 new file mode 100644 index 0000000000000000000000000000000000000000..a3bb3107256bc385ba05f2468a3d218c87cac045 --- /dev/null +++ b/db/schema_migrations/20241213142261 @@ -0,0 +1 @@ +7e19b2234f7bfb386d2e39d5da33de45bd51d3d9d4fb560f70e3eedaa687163b \ No newline at end of file diff --git a/db/schema_migrations/20241213142262 b/db/schema_migrations/20241213142262 new file mode 100644 index 0000000000000000000000000000000000000000..551218b1d51379f1b96c0bda4cca92407f3dbfdb --- /dev/null +++ b/db/schema_migrations/20241213142262 @@ -0,0 +1 @@ +a79a7f0b9972a57e5a73e19fa46457f6e84a00e5c81ba22d2691c88de95bea13 \ No newline at end of file diff --git a/db/schema_migrations/20241213142263 b/db/schema_migrations/20241213142263 new file mode 100644 index 0000000000000000000000000000000000000000..948adf836fa4a3d5f36c8905d96183de1c6dc181 --- /dev/null +++ b/db/schema_migrations/20241213142263 @@ -0,0 +1 @@ +7c9d3234710dcccae97cd6a15eb1777fedb687eec2e8123b059d63db6e5b99af \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 4cd1ef0559a46bdc6a33449e8434220257096d30..07bb978abc9ec23daa8f715ad5b13ddd0572d2b1 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -2838,6 +2838,22 @@ RETURN NEW; END $$; +CREATE FUNCTION trigger_cd50823537a3() RETURNS trigger + LANGUAGE plpgsql + AS $$ +BEGIN +IF NEW."namespace_id" IS NULL THEN + SELECT "namespace_id" + INTO NEW."namespace_id" + FROM "issues" + WHERE "issues"."id" = NEW."issue_id"; +END IF; + +RETURN NEW; + +END +$$; + CREATE FUNCTION trigger_cf646a118cbb() RETURNS trigger LANGUAGE plpgsql AS $$ @@ -13901,7 +13917,8 @@ CREATE TABLE issuable_slas ( issue_id bigint NOT NULL, due_at timestamp with time zone NOT NULL, label_applied boolean DEFAULT false NOT NULL, - issuable_closed boolean DEFAULT false NOT NULL + issuable_closed boolean DEFAULT false NOT NULL, + namespace_id bigint ); CREATE SEQUENCE issuable_slas_id_seq @@ -31054,6 +31071,8 @@ CREATE INDEX index_issuable_slas_on_due_at_id_label_applied_issuable_closed ON i CREATE UNIQUE INDEX index_issuable_slas_on_issue_id ON issuable_slas USING btree (issue_id); +CREATE INDEX index_issuable_slas_on_namespace_id ON issuable_slas USING btree (namespace_id); + CREATE INDEX index_issue_assignees_on_user_id_and_issue_id ON issue_assignees USING btree (user_id, issue_id); CREATE INDEX index_issue_assignment_events_on_namespace_id ON issue_assignment_events USING btree (namespace_id); @@ -35918,6 +35937,8 @@ CREATE TRIGGER trigger_cac7c0698291 BEFORE INSERT OR UPDATE ON evidences FOR EAC CREATE TRIGGER trigger_catalog_resource_sync_event_on_project_update AFTER UPDATE ON projects FOR EACH ROW WHEN ((((old.name)::text IS DISTINCT FROM (new.name)::text) OR (old.description IS DISTINCT FROM new.description) OR (old.visibility_level IS DISTINCT FROM new.visibility_level))) EXECUTE FUNCTION insert_catalog_resource_sync_event(); +CREATE TRIGGER trigger_cd50823537a3 BEFORE INSERT OR UPDATE ON issuable_slas FOR EACH ROW EXECUTE FUNCTION trigger_cd50823537a3(); + CREATE TRIGGER trigger_cf646a118cbb BEFORE INSERT OR UPDATE ON milestone_releases FOR EACH ROW EXECUTE FUNCTION trigger_cf646a118cbb(); CREATE TRIGGER trigger_d4487a75bd44 BEFORE INSERT OR UPDATE ON terraform_state_versions FOR EACH ROW EXECUTE FUNCTION trigger_d4487a75bd44(); @@ -36300,6 +36321,9 @@ ALTER TABLE ONLY user_namespace_callouts ALTER TABLE ONLY user_details ADD CONSTRAINT fk_27ac767d6a FOREIGN KEY (bot_namespace_id) REFERENCES namespaces(id) ON DELETE SET NULL; +ALTER TABLE ONLY issuable_slas + ADD CONSTRAINT fk_282ef683a5 FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE; + ALTER TABLE ONLY work_item_dates_sources ADD CONSTRAINT fk_283fb4ad36 FOREIGN KEY (start_date_sourcing_milestone_id) REFERENCES milestones(id) ON DELETE SET NULL; diff --git a/lib/gitlab/background_migration/backfill_issuable_slas_namespace_id.rb b/lib/gitlab/background_migration/backfill_issuable_slas_namespace_id.rb new file mode 100644 index 0000000000000000000000000000000000000000..2a07d8131fc594663e4b434e1181814c01f49f83 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_issuable_slas_namespace_id.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class BackfillIssuableSlasNamespaceId < BackfillDesiredShardingKeyJob + operation_name :backfill_issuable_slas_namespace_id + feature_category :incident_management + end + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_issuable_slas_namespace_id_spec.rb b/spec/lib/gitlab/background_migration/backfill_issuable_slas_namespace_id_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..95fc64afe746e7a71f01ac1c2fae09ed0fb065b7 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_issuable_slas_namespace_id_spec.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillIssuableSlasNamespaceId, + feature_category: :incident_management, + schema: 20241213142259 do + include_examples 'desired sharding key backfill job' do + let(:batch_table) { :issuable_slas } + let(:backfill_column) { :namespace_id } + let(:backfill_via_table) { :issues } + let(:backfill_via_column) { :namespace_id } + let(:backfill_via_foreign_key) { :issue_id } + end +end diff --git a/spec/migrations/20241213142263_queue_backfill_issuable_slas_namespace_id_spec.rb b/spec/migrations/20241213142263_queue_backfill_issuable_slas_namespace_id_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..625c46ee2a360aeb337380c82bb7115d56326639 --- /dev/null +++ b/spec/migrations/20241213142263_queue_backfill_issuable_slas_namespace_id_spec.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueBackfillIssuableSlasNamespaceId, feature_category: :incident_management do + let!(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + table_name: :issuable_slas, + column_name: :id, + interval: described_class::DELAY_INTERVAL, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE, + gitlab_schema: :gitlab_main_cell, + job_arguments: [ + :namespace_id, + :issues, + :namespace_id, + :issue_id + ] + ) + } + end + end +end