Merge branch 'backfill-desired-sharding-key-small-table-packages_rubygems_metadata' into 'master'

Add and backfill project_id for packages_rubygems_metadata

See merge request https://gitlab.com/gitlab-org/gitlab/-/merge_requests/167611



Merged-by: Adam Hegyi <ahegyi@gitlab.com>
Approved-by: Adie (she/her) <avpfestin@gitlab.com>
Approved-by: John Mason <9717668-johnmason@users.noreply.gitlab.com>
Approved-by: Adam Hegyi <ahegyi@gitlab.com>
Approved-by: Dzmitry (Dima) Meshcharakou <12459192-dmeshcharakou@users.noreply.gitlab.com>
Co-authored-by: Shubham Kumar <shukumar@gitlab.com>

db/docs/batched_background_migrations/backfill_packages_rubygems_metadata_project_id.yml

+---
+migration_job_name: BackfillPackagesRubygemsMetadataProjectId
+description: Backfills sharding key `packages_rubygems_metadata.project_id` from `packages_packages`.
+feature_category: package_registry
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/167611
+milestone: '17.5'
+queued_migration_version: 20240930122643
+finalize_after: '2024-10-22'
+finalized_by: # version of the migration that finalized this BBM

db/docs/packages_rubygems_metadata.yml

@@ -19,3 +19,4 @@ desired_sharding_key:
         table: packages_packages
         sharding_key: project_id
         belongs_to: package
+desired_sharding_key_migration_job_name: BackfillPackagesRubygemsMetadataProjectId

db/migrate/20240930122639_add_project_id_to_packages_rubygems_metadata.rb

+# frozen_string_literal: true
+
+class AddProjectIdToPackagesRubygemsMetadata < Gitlab::Database::Migration[2.2]
+  milestone '17.5'
+
+  def change
+    add_column :packages_rubygems_metadata, :project_id, :bigint
+  end
+end

db/post_migrate/20240930122640_index_packages_rubygems_metadata_on_project_id.rb

+# frozen_string_literal: true
+
+class IndexPackagesRubygemsMetadataOnProjectId < Gitlab::Database::Migration[2.2]
+  milestone '17.5'
+  disable_ddl_transaction!
+
+  INDEX_NAME = 'index_packages_rubygems_metadata_on_project_id'
+
+  def up
+    add_concurrent_index :packages_rubygems_metadata, :project_id, name: INDEX_NAME
+  end
+
+  def down
+    remove_concurrent_index_by_name :packages_rubygems_metadata, INDEX_NAME
+  end
+end

db/post_migrate/20240930122641_add_packages_rubygems_metadata_project_id_fk.rb

+# frozen_string_literal: true
+
+class AddPackagesRubygemsMetadataProjectIdFk < Gitlab::Database::Migration[2.2]
+  milestone '17.5'
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_foreign_key :packages_rubygems_metadata, :projects, column: :project_id, on_delete: :cascade
+  end
+
+  def down
+    with_lock_retries do
+      remove_foreign_key :packages_rubygems_metadata, column: :project_id
+    end
+  end
+end

db/post_migrate/20240930122642_add_packages_rubygems_metadata_project_id_trigger.rb

+# frozen_string_literal: true
+
+class AddPackagesRubygemsMetadataProjectIdTrigger < Gitlab::Database::Migration[2.2]
+  milestone '17.5'
+
+  def up
+    install_sharding_key_assignment_trigger(
+      table: :packages_rubygems_metadata,
+      sharding_key: :project_id,
+      parent_table: :packages_packages,
+      parent_sharding_key: :project_id,
+      foreign_key: :package_id
+    )
+  end
+
+  def down
+    remove_sharding_key_assignment_trigger(
+      table: :packages_rubygems_metadata,
+      sharding_key: :project_id,
+      parent_table: :packages_packages,
+      parent_sharding_key: :project_id,
+      foreign_key: :package_id
+    )
+  end
+end

db/post_migrate/20240930122643_queue_backfill_packages_rubygems_metadata_project_id.rb

+# frozen_string_literal: true
+
+class QueueBackfillPackagesRubygemsMetadataProjectId < Gitlab::Database::Migration[2.2]
+  milestone '17.5'
+  restrict_gitlab_migration gitlab_schema: :gitlab_main_cell
+
+  MIGRATION = "BackfillPackagesRubygemsMetadataProjectId"
+  DELAY_INTERVAL = 2.minutes
+  BATCH_SIZE = 1000
+  SUB_BATCH_SIZE = 100
+
+  def up
+    queue_batched_background_migration(
+      MIGRATION,
+      :packages_rubygems_metadata,
+      :package_id,
+      :project_id,
+      :packages_packages,
+      :project_id,
+      :package_id,
+      job_interval: DELAY_INTERVAL,
+      batch_size: BATCH_SIZE,
+      sub_batch_size: SUB_BATCH_SIZE
+    )
+  end
+
+  def down
+    delete_batched_background_migration(
+      MIGRATION,
+      :packages_rubygems_metadata,
+      :package_id,
+      [
+        :project_id,
+        :packages_packages,
+        :project_id,
+        :package_id
+      ]
+    )
+  end
+end

db/schema_migrations/20240930122639

+fa54d0f7fd89d3b9fa64d76fd48228070bdd55dc4840c37e60fd7b42c12ea327
\ No newline at end of file

db/schema_migrations/20240930122640

+a432dae36492a800316b32c86343fb1f6a0164c08596c62935e68879c8332159
\ No newline at end of file

db/schema_migrations/20240930122641

+7ec41a97fa56b1d6ea95eb95603f9547bbae01f832e0d4f4a0ec6e1b9d552b07
\ No newline at end of file

db/schema_migrations/20240930122642

+9ea93c14618c93e0d3b7e825c95b5b1d37159df9eff1580eb10f8e12c0bc75ad
\ No newline at end of file

db/schema_migrations/20240930122643

+b7280f94dcd97acbc5b48ad26d2c1907bd5a82a40ff33d44c476f3ae1c92c39f
\ No newline at end of file

db/structure.sql

@@ -1609,6 +1609,22 @@ RETURN NEW;
 END
 $$;
+CREATE FUNCTION trigger_700f29b1312e() RETURNS trigger
+    LANGUAGE plpgsql
+    AS $$
+BEGIN
+IF NEW."project_id" IS NULL THEN
+  SELECT "project_id"
+  INTO NEW."project_id"
+  FROM "packages_packages"
+  WHERE "packages_packages"."id" = NEW."package_id";
+END IF;
+
+RETURN NEW;
+
+END
+$$;
+
 CREATE FUNCTION trigger_70d3f0bba1de() RETURNS trigger
     LANGUAGE plpgsql
     AS $$
@@ -15644,6 +15660,7 @@ CREATE TABLE packages_rubygems_metadata (
     requirements text,
     rubygems_version text,
     signing_key text,
+    project_id bigint,
     CONSTRAINT check_0154a18c82 CHECK ((char_length(description) <= 1024)),
     CONSTRAINT check_22814c771b CHECK ((char_length(email) <= 255)),
     CONSTRAINT check_242293030e CHECK ((char_length(extensions) <= 255)),
@@ -30051,6 +30068,8 @@ CREATE INDEX index_packages_rpm_metadata_on_project_id ON packages_rpm_metadata
 CREATE INDEX index_packages_rpm_repository_files_on_project_id_and_file_name ON packages_rpm_repository_files USING btree (project_id, file_name);
+CREATE INDEX index_packages_rubygems_metadata_on_project_id ON packages_rubygems_metadata USING btree (project_id);
+
 CREATE INDEX index_packages_tags_on_package_id_and_updated_at ON packages_tags USING btree (package_id, updated_at DESC);
 CREATE INDEX index_packages_tags_on_project_id ON packages_tags USING btree (project_id);
@@ -33523,6 +33542,8 @@ CREATE TRIGGER trigger_6cdea9559242 BEFORE INSERT OR UPDATE ON issue_links FOR E
 CREATE TRIGGER trigger_6d6c79ce74e1 BEFORE INSERT OR UPDATE ON protected_environment_deploy_access_levels FOR EACH ROW EXECUTE FUNCTION trigger_6d6c79ce74e1();
+CREATE TRIGGER trigger_700f29b1312e BEFORE INSERT OR UPDATE ON packages_rubygems_metadata FOR EACH ROW EXECUTE FUNCTION trigger_700f29b1312e();
+
 CREATE TRIGGER trigger_70d3f0bba1de BEFORE INSERT OR UPDATE ON compliance_framework_security_policies FOR EACH ROW EXECUTE FUNCTION trigger_70d3f0bba1de();
 CREATE TRIGGER trigger_740afa9807b8 BEFORE INSERT OR UPDATE ON subscription_user_add_on_assignments FOR EACH ROW EXECUTE FUNCTION trigger_740afa9807b8();
@@ -34760,6 +34781,9 @@ ALTER TABLE ONLY bulk_import_entities
 ALTER TABLE ONLY security_policy_requirements
     ADD CONSTRAINT fk_b6e48e3428 FOREIGN KEY (compliance_framework_security_policy_id) REFERENCES compliance_framework_security_policies(id) ON DELETE CASCADE;
+ALTER TABLE ONLY packages_rubygems_metadata
+    ADD CONSTRAINT fk_b73c052149 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY compliance_management_frameworks
     ADD CONSTRAINT fk_b74c45b71f FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE;

lib/gitlab/background_migration/backfill_packages_rubygems_metadata_project_id.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    class BackfillPackagesRubygemsMetadataProjectId < BackfillDesiredShardingKeyJob
+      operation_name :backfill_packages_rubygems_metadata_project_id
+      feature_category :package_registry
+    end
+  end
+end

spec/lib/gitlab/background_migration/backfill_packages_rubygems_metadata_project_id_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::BackfillPackagesRubygemsMetadataProjectId,
+  feature_category: :package_registry,
+  schema: 20240930122639 do
+  include_examples 'desired sharding key backfill job' do
+    let(:batch_table) { :packages_rubygems_metadata }
+    let(:batch_column) { :package_id }
+    let(:backfill_column) { :project_id }
+    let(:backfill_via_table) { :packages_packages }
+    let(:backfill_via_column) { :project_id }
+    let(:backfill_via_foreign_key) { :package_id }
+  end
+end

spec/migrations/20240930122643_queue_backfill_packages_rubygems_metadata_project_id_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe QueueBackfillPackagesRubygemsMetadataProjectId, feature_category: :package_registry do
+  let!(:batched_migration) { described_class::MIGRATION }
+
+  it 'schedules a new batched migration' do
+    reversible_migration do |migration|
+      migration.before -> {
+        expect(batched_migration).not_to have_scheduled_batched_migration
+      }
+
+      migration.after -> {
+        expect(batched_migration).to have_scheduled_batched_migration(
+          table_name: :packages_rubygems_metadata,
+          column_name: :package_id,
+          interval: described_class::DELAY_INTERVAL,
+          batch_size: described_class::BATCH_SIZE,
+          sub_batch_size: described_class::SUB_BATCH_SIZE,
+          gitlab_schema: :gitlab_main_cell,
+          job_arguments: [
+            :project_id,
+            :packages_packages,
+            :project_id,
+            :package_id
+          ]
+        )
+      }
+    end
+  end
+end