Merge branch 'backfill-desired-sharding-key-small-table-packages_conan_metadata' into 'master'

Add and backfill project_id for packages_conan_metadata

See merge request https://gitlab.com/gitlab-org/gitlab/-/merge_requests/156254



Merged-by: Tiger Watson <twatson@gitlab.com>
Approved-by: Tiger Watson <twatson@gitlab.com>
Co-authored-by: Omar Qunsul <oqunsul@gitlab.com>

db/docs/batched_background_migrations/backfill_packages_conan_metadata_project_id.yml

+---
+migration_job_name: BackfillPackagesConanMetadataProjectId
+description: Backfills sharding key `packages_conan_metadata.project_id` from `packages_packages`.
+feature_category: package_registry
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/156254
+milestone: '17.2'
+queued_migration_version: 20240613071715
+finalize_after: '2024-07-22'
+finalized_by: # version of the migration that finalized this BBM

db/docs/packages_conan_metadata.yml

@@ -19,3 +19,4 @@ desired_sharding_key:
         table: packages_packages
         sharding_key: project_id
         belongs_to: package
+desired_sharding_key_migration_job_name: BackfillPackagesConanMetadataProjectId

db/migrate/20240613071711_add_project_id_to_packages_conan_metadata.rb

+# frozen_string_literal: true
+
+class AddProjectIdToPackagesConanMetadata < Gitlab::Database::Migration[2.2]
+  milestone '17.2'
+
+  def change
+    add_column :packages_conan_metadata, :project_id, :bigint
+  end
+end

db/post_migrate/20240613071712_index_packages_conan_metadata_on_project_id.rb

+# frozen_string_literal: true
+
+class IndexPackagesConanMetadataOnProjectId < Gitlab::Database::Migration[2.2]
+  milestone '17.2'
+  disable_ddl_transaction!
+
+  INDEX_NAME = 'index_packages_conan_metadata_on_project_id'
+
+  def up
+    add_concurrent_index :packages_conan_metadata, :project_id, name: INDEX_NAME
+  end
+
+  def down
+    remove_concurrent_index_by_name :packages_conan_metadata, INDEX_NAME
+  end
+end

db/post_migrate/20240613071713_add_packages_conan_metadata_project_id_fk.rb

+# frozen_string_literal: true
+
+class AddPackagesConanMetadataProjectIdFk < Gitlab::Database::Migration[2.2]
+  milestone '17.2'
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_foreign_key :packages_conan_metadata, :projects, column: :project_id, on_delete: :cascade
+  end
+
+  def down
+    with_lock_retries do
+      remove_foreign_key :packages_conan_metadata, column: :project_id
+    end
+  end
+end

db/post_migrate/20240613071714_add_packages_conan_metadata_project_id_trigger.rb

+# frozen_string_literal: true
+
+class AddPackagesConanMetadataProjectIdTrigger < Gitlab::Database::Migration[2.2]
+  milestone '17.2'
+
+  def up
+    install_sharding_key_assignment_trigger(
+      table: :packages_conan_metadata,
+      sharding_key: :project_id,
+      parent_table: :packages_packages,
+      parent_sharding_key: :project_id,
+      foreign_key: :package_id
+    )
+  end
+
+  def down
+    remove_sharding_key_assignment_trigger(
+      table: :packages_conan_metadata,
+      sharding_key: :project_id,
+      parent_table: :packages_packages,
+      parent_sharding_key: :project_id,
+      foreign_key: :package_id
+    )
+  end
+end

db/post_migrate/20240613071715_queue_backfill_packages_conan_metadata_project_id.rb

+# frozen_string_literal: true
+
+class QueueBackfillPackagesConanMetadataProjectId < Gitlab::Database::Migration[2.2]
+  milestone '17.2'
+  restrict_gitlab_migration gitlab_schema: :gitlab_main_cell
+
+  MIGRATION = "BackfillPackagesConanMetadataProjectId"
+  DELAY_INTERVAL = 2.minutes
+  BATCH_SIZE = 1000
+  SUB_BATCH_SIZE = 100
+
+  def up
+    queue_batched_background_migration(
+      MIGRATION,
+      :packages_conan_metadata,
+      :id,
+      :project_id,
+      :packages_packages,
+      :project_id,
+      :package_id,
+      job_interval: DELAY_INTERVAL,
+      batch_size: BATCH_SIZE,
+      sub_batch_size: SUB_BATCH_SIZE
+    )
+  end
+
+  def down
+    delete_batched_background_migration(
+      MIGRATION,
+      :packages_conan_metadata,
+      :id,
+      [
+        :project_id,
+        :packages_packages,
+        :project_id,
+        :package_id
+      ]
+    )
+  end
+end

db/schema_migrations/20240613071711

+fa0b49f89efeed78982d1c8f96b068fa54024eed6a7ab2d847b7c37b9be147e2
\ No newline at end of file

db/schema_migrations/20240613071712

+41d1a6bef277adaf4bfffe31757bcacdb3f12eec6795d80901db329b3bb41e71
\ No newline at end of file

db/schema_migrations/20240613071713

+42d0b244f1163155ffd2839796a9f5077cd37fc96d10778d5bbdd17ff8394b42
\ No newline at end of file

db/schema_migrations/20240613071714

+51abebcfd2ac74b9ad980f2f6374ee49504c88289b6d79a28928bcc914082ef2
\ No newline at end of file

db/schema_migrations/20240613071715

+ad61d6d55b22a37b7052d6de321ebdd3f0a1543cacebd5fb128e052c2b38ba1c
\ No newline at end of file

db/structure.sql

@@ -829,6 +829,22 @@ RETURN NEW;
 END
 $$;
+CREATE FUNCTION trigger_18bc439a6741() RETURNS trigger
+    LANGUAGE plpgsql
+    AS $$
+BEGIN
+IF NEW."project_id" IS NULL THEN
+  SELECT "project_id"
+  INTO NEW."project_id"
+  FROM "packages_packages"
+  WHERE "packages_packages"."id" = NEW."package_id";
+END IF;
+
+RETURN NEW;
+
+END
+$$;
+
 CREATE FUNCTION trigger_1ed40f4d5f4e() RETURNS trigger
     LANGUAGE plpgsql
     AS $$
@@ -13739,7 +13755,8 @@ CREATE TABLE packages_conan_metadata (
     created_at timestamp with time zone NOT NULL,
     updated_at timestamp with time zone NOT NULL,
     package_username character varying(255) NOT NULL,
-    package_channel character varying(255) NOT NULL
+    package_channel character varying(255) NOT NULL,
+    project_id bigint
 );
 CREATE SEQUENCE packages_conan_metadata_id_seq
@@ -27948,6 +27965,8 @@ CREATE UNIQUE INDEX index_packages_conan_file_metadata_on_package_file_id ON pac
 CREATE UNIQUE INDEX index_packages_conan_metadata_on_package_id_username_channel ON packages_conan_metadata USING btree (package_id, package_username, package_channel);
+CREATE INDEX index_packages_conan_metadata_on_project_id ON packages_conan_metadata USING btree (project_id);
+
 CREATE INDEX index_packages_debian_group_component_files_on_component_id ON packages_debian_group_component_files USING btree (component_id);
 CREATE INDEX index_packages_debian_group_distribution_keys_on_group_id ON packages_debian_group_distribution_keys USING btree (group_id);
@@ -31292,6 +31311,8 @@ CREATE TRIGGER trigger_13d4aa8fe3dd BEFORE INSERT OR UPDATE ON draft_notes FOR E
 CREATE TRIGGER trigger_174b23fa3dfb BEFORE INSERT OR UPDATE ON approval_project_rules_users FOR EACH ROW EXECUTE FUNCTION trigger_174b23fa3dfb();
+CREATE TRIGGER trigger_18bc439a6741 BEFORE INSERT OR UPDATE ON packages_conan_metadata FOR EACH ROW EXECUTE FUNCTION trigger_18bc439a6741();
+
 CREATE TRIGGER trigger_1ed40f4d5f4e BEFORE INSERT OR UPDATE ON packages_maven_metadata FOR EACH ROW EXECUTE FUNCTION trigger_1ed40f4d5f4e();
 CREATE TRIGGER trigger_207005e8e995 BEFORE INSERT OR UPDATE ON operations_strategies FOR EACH ROW EXECUTE FUNCTION trigger_207005e8e995();
@@ -32062,6 +32083,9 @@ ALTER TABLE ONLY subscription_user_add_on_assignments
 ALTER TABLE ONLY vulnerabilities
     ADD CONSTRAINT fk_725465b774 FOREIGN KEY (dismissed_by_id) REFERENCES users(id) ON DELETE SET NULL;
+ALTER TABLE ONLY packages_conan_metadata
+    ADD CONSTRAINT fk_7302a29cd9 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY approval_merge_request_rules
     ADD CONSTRAINT fk_73fec3d7e5 FOREIGN KEY (approval_policy_rule_id) REFERENCES approval_policy_rules(id) ON DELETE CASCADE;

lib/gitlab/background_migration/backfill_packages_conan_metadata_project_id.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    class BackfillPackagesConanMetadataProjectId < BackfillDesiredShardingKeyJob
+      operation_name :backfill_packages_conan_metadata_project_id
+      feature_category :package_registry
+    end
+  end
+end

spec/lib/gitlab/background_migration/backfill_packages_conan_metadata_project_id_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::BackfillPackagesConanMetadataProjectId,
+  feature_category: :package_registry,
+  schema: 20240613071711 do
+  include_examples 'desired sharding key backfill job' do
+    let(:batch_table) { :packages_conan_metadata }
+    let(:backfill_column) { :project_id }
+    let(:backfill_via_table) { :packages_packages }
+    let(:backfill_via_column) { :project_id }
+    let(:backfill_via_foreign_key) { :package_id }
+  end
+end

spec/migrations/20240613071715_queue_backfill_packages_conan_metadata_project_id_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe QueueBackfillPackagesConanMetadataProjectId, feature_category: :package_registry do
+  let!(:batched_migration) { described_class::MIGRATION }
+
+  it 'schedules a new batched migration' do
+    reversible_migration do |migration|
+      migration.before -> {
+        expect(batched_migration).not_to have_scheduled_batched_migration
+      }
+
+      migration.after -> {
+        expect(batched_migration).to have_scheduled_batched_migration(
+          table_name: :packages_conan_metadata,
+          column_name: :id,
+          interval: described_class::DELAY_INTERVAL,
+          batch_size: described_class::BATCH_SIZE,
+          sub_batch_size: described_class::SUB_BATCH_SIZE,
+          gitlab_schema: :gitlab_main_cell,
+          job_arguments: [
+            :project_id,
+            :packages_packages,
+            :project_id,
+            :package_id
+          ]
+        )
+      }
+    end
+  end
+end