diff --git a/db/docs/batched_background_migrations/backfill_packages_nuget_metadata_project_id.yml b/db/docs/batched_background_migrations/backfill_packages_nuget_metadata_project_id.yml new file mode 100644 index 0000000000000000000000000000000000000000..5b38828db285b90e401c67572caec0e3da26ff34 --- /dev/null +++ b/db/docs/batched_background_migrations/backfill_packages_nuget_metadata_project_id.yml @@ -0,0 +1,9 @@ +--- +migration_job_name: BackfillPackagesNugetMetadataProjectId +description: Backfills sharding key `packages_nuget_metadata.project_id` from `packages_packages`. +feature_category: package_registry +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/167607 +milestone: '17.5' +queued_migration_version: 20240930121136 +finalize_after: '2024-10-22' +finalized_by: # version of the migration that finalized this BBM diff --git a/db/docs/packages_nuget_metadata.yml b/db/docs/packages_nuget_metadata.yml index 639539e9b1351057903503a5478258cb906c5bb4..28f64be60419624f97469ad9db2e405189da1802 100644 --- a/db/docs/packages_nuget_metadata.yml +++ b/db/docs/packages_nuget_metadata.yml @@ -19,3 +19,4 @@ desired_sharding_key: table: packages_packages sharding_key: project_id belongs_to: package +desired_sharding_key_migration_job_name: BackfillPackagesNugetMetadataProjectId diff --git a/db/migrate/20240930121132_add_project_id_to_packages_nuget_metadata.rb b/db/migrate/20240930121132_add_project_id_to_packages_nuget_metadata.rb new file mode 100644 index 0000000000000000000000000000000000000000..4df4ef3d197edba4df978957a7c4fa58dc706c68 --- /dev/null +++ b/db/migrate/20240930121132_add_project_id_to_packages_nuget_metadata.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class AddProjectIdToPackagesNugetMetadata < Gitlab::Database::Migration[2.2] + milestone '17.5' + + def change + add_column :packages_nuget_metadata, :project_id, :bigint + end +end diff --git a/db/post_migrate/20240930121133_index_packages_nuget_metadata_on_project_id.rb b/db/post_migrate/20240930121133_index_packages_nuget_metadata_on_project_id.rb new file mode 100644 index 0000000000000000000000000000000000000000..8d51bad269733cb9482e9bcd47ce7d6215f78dd8 --- /dev/null +++ b/db/post_migrate/20240930121133_index_packages_nuget_metadata_on_project_id.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class IndexPackagesNugetMetadataOnProjectId < Gitlab::Database::Migration[2.2] + milestone '17.5' + disable_ddl_transaction! + + INDEX_NAME = 'index_packages_nuget_metadata_on_project_id' + + def up + add_concurrent_index :packages_nuget_metadata, :project_id, name: INDEX_NAME + end + + def down + remove_concurrent_index_by_name :packages_nuget_metadata, INDEX_NAME + end +end diff --git a/db/post_migrate/20240930121134_add_packages_nuget_metadata_project_id_fk.rb b/db/post_migrate/20240930121134_add_packages_nuget_metadata_project_id_fk.rb new file mode 100644 index 0000000000000000000000000000000000000000..d9c95cea180c9e88b53fc962753984d9cc4e4851 --- /dev/null +++ b/db/post_migrate/20240930121134_add_packages_nuget_metadata_project_id_fk.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +class AddPackagesNugetMetadataProjectIdFk < Gitlab::Database::Migration[2.2] + milestone '17.5' + disable_ddl_transaction! + + def up + add_concurrent_foreign_key :packages_nuget_metadata, :projects, column: :project_id, on_delete: :cascade + end + + def down + with_lock_retries do + remove_foreign_key :packages_nuget_metadata, column: :project_id + end + end +end diff --git a/db/post_migrate/20240930121135_add_packages_nuget_metadata_project_id_trigger.rb b/db/post_migrate/20240930121135_add_packages_nuget_metadata_project_id_trigger.rb new file mode 100644 index 0000000000000000000000000000000000000000..4f16cefec7a89d34b2bf46d038393a8d343eb51f --- /dev/null +++ b/db/post_migrate/20240930121135_add_packages_nuget_metadata_project_id_trigger.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class AddPackagesNugetMetadataProjectIdTrigger < Gitlab::Database::Migration[2.2] + milestone '17.5' + + def up + install_sharding_key_assignment_trigger( + table: :packages_nuget_metadata, + sharding_key: :project_id, + parent_table: :packages_packages, + parent_sharding_key: :project_id, + foreign_key: :package_id + ) + end + + def down + remove_sharding_key_assignment_trigger( + table: :packages_nuget_metadata, + sharding_key: :project_id, + parent_table: :packages_packages, + parent_sharding_key: :project_id, + foreign_key: :package_id + ) + end +end diff --git a/db/post_migrate/20240930121136_queue_backfill_packages_nuget_metadata_project_id.rb b/db/post_migrate/20240930121136_queue_backfill_packages_nuget_metadata_project_id.rb new file mode 100644 index 0000000000000000000000000000000000000000..c055bee7b2211814ee70a78eebe828c921a6d7a7 --- /dev/null +++ b/db/post_migrate/20240930121136_queue_backfill_packages_nuget_metadata_project_id.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +class QueueBackfillPackagesNugetMetadataProjectId < Gitlab::Database::Migration[2.2] + milestone '17.5' + restrict_gitlab_migration gitlab_schema: :gitlab_main_cell + + MIGRATION = "BackfillPackagesNugetMetadataProjectId" + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 1000 + SUB_BATCH_SIZE = 100 + + def up + queue_batched_background_migration( + MIGRATION, + :packages_nuget_metadata, + :package_id, + :project_id, + :packages_packages, + :project_id, + :package_id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration( + MIGRATION, + :packages_nuget_metadata, + :package_id, + [ + :project_id, + :packages_packages, + :project_id, + :package_id + ] + ) + end +end diff --git a/db/schema_migrations/20240930121132 b/db/schema_migrations/20240930121132 new file mode 100644 index 0000000000000000000000000000000000000000..5bd4f69344ed8f673665ce99eeec7c1fc70b6ed6 --- /dev/null +++ b/db/schema_migrations/20240930121132 @@ -0,0 +1 @@ +1f8ef02007a199d46b542de398345c66d5106d18c72dd5cc18f162a4a17fa315 \ No newline at end of file diff --git a/db/schema_migrations/20240930121133 b/db/schema_migrations/20240930121133 new file mode 100644 index 0000000000000000000000000000000000000000..7e054400244fe5d057c69c343c5c0a6b764775d7 --- /dev/null +++ b/db/schema_migrations/20240930121133 @@ -0,0 +1 @@ +0062f4cb57d36d99b9e647419d874b02f4ae135944f73f6f7bc96d38498a7374 \ No newline at end of file diff --git a/db/schema_migrations/20240930121134 b/db/schema_migrations/20240930121134 new file mode 100644 index 0000000000000000000000000000000000000000..51f582836cfd02f723aa64fa82e67b9a5e8c8fc5 --- /dev/null +++ b/db/schema_migrations/20240930121134 @@ -0,0 +1 @@ +704079bebfa5cc60e7f5eb3e6e50d823af9f68acc9b518da0df1b10b33398462 \ No newline at end of file diff --git a/db/schema_migrations/20240930121135 b/db/schema_migrations/20240930121135 new file mode 100644 index 0000000000000000000000000000000000000000..9e7646f635c96945ba1a8a8b1d0f872649e406a6 --- /dev/null +++ b/db/schema_migrations/20240930121135 @@ -0,0 +1 @@ +63f9db5b418f528a9356396c958ca6d56453b72b6e01efd10b85a30cee58487c \ No newline at end of file diff --git a/db/schema_migrations/20240930121136 b/db/schema_migrations/20240930121136 new file mode 100644 index 0000000000000000000000000000000000000000..409f03218cdbfe6ec0d2cdf1f9a055836c52916f --- /dev/null +++ b/db/schema_migrations/20240930121136 @@ -0,0 +1 @@ +39d969b1f1b2edf41be73346a1b89a9c313272eebbd483cffc0b0328e7849f76 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index a6add27da228539d7c6d055aeda8e4152ceee1cf..24c101e6e6e11c0a120baefcf0d0c94e8d526193 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -905,6 +905,22 @@ RETURN NEW; END $$; +CREATE FUNCTION trigger_14a39509be0a() RETURNS trigger + LANGUAGE plpgsql + AS $$ +BEGIN +IF NEW."project_id" IS NULL THEN + SELECT "project_id" + INTO NEW."project_id" + FROM "packages_packages" + WHERE "packages_packages"."id" = NEW."package_id"; +END IF; + +RETURN NEW; + +END +$$; + CREATE FUNCTION trigger_158ac875f254() RETURNS trigger LANGUAGE plpgsql AS $$ @@ -15276,6 +15292,7 @@ CREATE TABLE packages_nuget_metadata ( authors text, description text, normalized_version text, + project_id bigint, CONSTRAINT check_9973c0cc33 CHECK ((char_length(normalized_version) <= 255)), CONSTRAINT check_d39a5fe9ee CHECK ((char_length(description) <= 4000)), CONSTRAINT check_e2fc129ebd CHECK ((char_length(authors) <= 255)), @@ -29828,6 +29845,8 @@ CREATE INDEX index_packages_npm_metadata_caches_on_project_id_status ON packages CREATE INDEX index_packages_nuget_dl_metadata_on_dependency_link_id ON packages_nuget_dependency_link_metadata USING btree (dependency_link_id); +CREATE INDEX index_packages_nuget_metadata_on_project_id ON packages_nuget_metadata USING btree (project_id); + CREATE UNIQUE INDEX index_packages_nuget_symbols_on_object_storage_key ON packages_nuget_symbols USING btree (object_storage_key); CREATE INDEX index_packages_nuget_symbols_on_package_id ON packages_nuget_symbols USING btree (package_id); @@ -33258,6 +33277,8 @@ CREATE TRIGGER trigger_0f38e5af9adf BEFORE INSERT OR UPDATE ON ml_candidate_para CREATE TRIGGER trigger_13d4aa8fe3dd BEFORE INSERT OR UPDATE ON draft_notes FOR EACH ROW EXECUTE FUNCTION trigger_13d4aa8fe3dd(); +CREATE TRIGGER trigger_14a39509be0a BEFORE INSERT OR UPDATE ON packages_nuget_metadata FOR EACH ROW EXECUTE FUNCTION trigger_14a39509be0a(); + CREATE TRIGGER trigger_158ac875f254 BEFORE INSERT OR UPDATE ON approval_group_rules_users FOR EACH ROW EXECUTE FUNCTION trigger_158ac875f254(); CREATE TRIGGER trigger_174b23fa3dfb BEFORE INSERT OR UPDATE ON approval_project_rules_users FOR EACH ROW EXECUTE FUNCTION trigger_174b23fa3dfb(); @@ -33710,6 +33731,9 @@ ALTER TABLE ONLY coverage_fuzzing_corpuses ALTER TABLE ONLY namespace_settings ADD CONSTRAINT fk_20cf0eb2f9 FOREIGN KEY (default_compliance_framework_id) REFERENCES compliance_management_frameworks(id) ON DELETE SET NULL; +ALTER TABLE ONLY packages_nuget_metadata + ADD CONSTRAINT fk_21569c0856 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; + ALTER TABLE p_ci_build_trace_metadata ADD CONSTRAINT fk_21d25cac1a_p FOREIGN KEY (partition_id, trace_artifact_id) REFERENCES p_ci_job_artifacts(partition_id, id) ON UPDATE CASCADE ON DELETE CASCADE; diff --git a/lib/gitlab/background_migration/backfill_packages_nuget_metadata_project_id.rb b/lib/gitlab/background_migration/backfill_packages_nuget_metadata_project_id.rb new file mode 100644 index 0000000000000000000000000000000000000000..3f42b6498d66a76b4007da4160ea1d56b87d2b74 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_packages_nuget_metadata_project_id.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class BackfillPackagesNugetMetadataProjectId < BackfillDesiredShardingKeyJob + operation_name :backfill_packages_nuget_metadata_project_id + feature_category :package_registry + end + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_packages_nuget_metadata_project_id_spec.rb b/spec/lib/gitlab/background_migration/backfill_packages_nuget_metadata_project_id_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..69374f9cfc763bb42783aaba391b67fcd1d32ce5 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_packages_nuget_metadata_project_id_spec.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillPackagesNugetMetadataProjectId, + feature_category: :package_registry, + schema: 20240930121132 do + include_examples 'desired sharding key backfill job' do + let(:batch_table) { :packages_nuget_metadata } + let(:batch_column) { :package_id } + let(:backfill_column) { :project_id } + let(:backfill_via_table) { :packages_packages } + let(:backfill_via_column) { :project_id } + let(:backfill_via_foreign_key) { :package_id } + end +end diff --git a/spec/migrations/20240930121136_queue_backfill_packages_nuget_metadata_project_id_spec.rb b/spec/migrations/20240930121136_queue_backfill_packages_nuget_metadata_project_id_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..a2db34efd43b4bdbd8e6bbbabb89124df2414ea8 --- /dev/null +++ b/spec/migrations/20240930121136_queue_backfill_packages_nuget_metadata_project_id_spec.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueBackfillPackagesNugetMetadataProjectId, feature_category: :package_registry do + let!(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + table_name: :packages_nuget_metadata, + column_name: :package_id, + interval: described_class::DELAY_INTERVAL, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE, + gitlab_schema: :gitlab_main_cell, + job_arguments: [ + :project_id, + :packages_packages, + :project_id, + :package_id + ] + ) + } + end + end +end