diff --git a/db/docs/batched_background_migrations/backfill_packages_pypi_metadata_project_id.yml b/db/docs/batched_background_migrations/backfill_packages_pypi_metadata_project_id.yml
new file mode 100644
index 0000000000000000000000000000000000000000..c6d1e82e25443acf113b248ab7f96003f13e7cd0
--- /dev/null
+++ b/db/docs/batched_background_migrations/backfill_packages_pypi_metadata_project_id.yml
@@ -0,0 +1,9 @@
+---
+migration_job_name: BackfillPackagesPypiMetadataProjectId
+description: Backfills sharding key `packages_pypi_metadata.project_id` from `packages_packages`.
+feature_category: package_registry
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/167608
+milestone: '17.5'
+queued_migration_version: 20240930121830
+finalize_after: '2024-10-22'
+finalized_by: # version of the migration that finalized this BBM
diff --git a/db/docs/packages_pypi_metadata.yml b/db/docs/packages_pypi_metadata.yml
index 95b1f0f635fba3f4ce98212656af02f9aba2f50c..1304012e0b3fca7c37a74e4a0a8ad3e6c9bb35b6 100644
--- a/db/docs/packages_pypi_metadata.yml
+++ b/db/docs/packages_pypi_metadata.yml
@@ -19,3 +19,4 @@ desired_sharding_key:
table: packages_packages
sharding_key: project_id
belongs_to: package
+desired_sharding_key_migration_job_name: BackfillPackagesPypiMetadataProjectId
diff --git a/db/migrate/20240930121826_add_project_id_to_packages_pypi_metadata.rb b/db/migrate/20240930121826_add_project_id_to_packages_pypi_metadata.rb
new file mode 100644
index 0000000000000000000000000000000000000000..2c908f3599e9360c924501b45fb4c6f3690a6413
--- /dev/null
+++ b/db/migrate/20240930121826_add_project_id_to_packages_pypi_metadata.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+class AddProjectIdToPackagesPypiMetadata < Gitlab::Database::Migration[2.2]
+ milestone '17.5'
+
+ def change
+ add_column :packages_pypi_metadata, :project_id, :bigint
+ end
+end
diff --git a/db/post_migrate/20240930121827_index_packages_pypi_metadata_on_project_id.rb b/db/post_migrate/20240930121827_index_packages_pypi_metadata_on_project_id.rb
new file mode 100644
index 0000000000000000000000000000000000000000..d4427fa3d06f8586d3c211f06767103195d8c4c3
--- /dev/null
+++ b/db/post_migrate/20240930121827_index_packages_pypi_metadata_on_project_id.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class IndexPackagesPypiMetadataOnProjectId < Gitlab::Database::Migration[2.2]
+ milestone '17.5'
+ disable_ddl_transaction!
+
+ INDEX_NAME = 'index_packages_pypi_metadata_on_project_id'
+
+ def up
+ add_concurrent_index :packages_pypi_metadata, :project_id, name: INDEX_NAME
+ end
+
+ def down
+ remove_concurrent_index_by_name :packages_pypi_metadata, INDEX_NAME
+ end
+end
diff --git a/db/post_migrate/20240930121828_add_packages_pypi_metadata_project_id_fk.rb b/db/post_migrate/20240930121828_add_packages_pypi_metadata_project_id_fk.rb
new file mode 100644
index 0000000000000000000000000000000000000000..f4114ab8d4d0d3e139f7d11e5e1730d396a94950
--- /dev/null
+++ b/db/post_migrate/20240930121828_add_packages_pypi_metadata_project_id_fk.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+class AddPackagesPypiMetadataProjectIdFk < Gitlab::Database::Migration[2.2]
+ milestone '17.5'
+ disable_ddl_transaction!
+
+ def up
+ add_concurrent_foreign_key :packages_pypi_metadata, :projects, column: :project_id, on_delete: :cascade
+ end
+
+ def down
+ with_lock_retries do
+ remove_foreign_key :packages_pypi_metadata, column: :project_id
+ end
+ end
+end
diff --git a/db/post_migrate/20240930121829_add_packages_pypi_metadata_project_id_trigger.rb b/db/post_migrate/20240930121829_add_packages_pypi_metadata_project_id_trigger.rb
new file mode 100644
index 0000000000000000000000000000000000000000..c9eeca1f951b081b6b160c18607f80bae3aa0530
--- /dev/null
+++ b/db/post_migrate/20240930121829_add_packages_pypi_metadata_project_id_trigger.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class AddPackagesPypiMetadataProjectIdTrigger < Gitlab::Database::Migration[2.2]
+ milestone '17.5'
+
+ def up
+ install_sharding_key_assignment_trigger(
+ table: :packages_pypi_metadata,
+ sharding_key: :project_id,
+ parent_table: :packages_packages,
+ parent_sharding_key: :project_id,
+ foreign_key: :package_id
+ )
+ end
+
+ def down
+ remove_sharding_key_assignment_trigger(
+ table: :packages_pypi_metadata,
+ sharding_key: :project_id,
+ parent_table: :packages_packages,
+ parent_sharding_key: :project_id,
+ foreign_key: :package_id
+ )
+ end
+end
diff --git a/db/post_migrate/20240930121830_queue_backfill_packages_pypi_metadata_project_id.rb b/db/post_migrate/20240930121830_queue_backfill_packages_pypi_metadata_project_id.rb
new file mode 100644
index 0000000000000000000000000000000000000000..2a4a613166bba58af6d6d25b459a6ca4536b064c
--- /dev/null
+++ b/db/post_migrate/20240930121830_queue_backfill_packages_pypi_metadata_project_id.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+class QueueBackfillPackagesPypiMetadataProjectId < Gitlab::Database::Migration[2.2]
+ milestone '17.5'
+ restrict_gitlab_migration gitlab_schema: :gitlab_main_cell
+
+ MIGRATION = "BackfillPackagesPypiMetadataProjectId"
+ DELAY_INTERVAL = 2.minutes
+ BATCH_SIZE = 1000
+ SUB_BATCH_SIZE = 100
+
+ def up
+ queue_batched_background_migration(
+ MIGRATION,
+ :packages_pypi_metadata,
+ :package_id,
+ :project_id,
+ :packages_packages,
+ :project_id,
+ :package_id,
+ job_interval: DELAY_INTERVAL,
+ batch_size: BATCH_SIZE,
+ sub_batch_size: SUB_BATCH_SIZE
+ )
+ end
+
+ def down
+ delete_batched_background_migration(
+ MIGRATION,
+ :packages_pypi_metadata,
+ :package_id,
+ [
+ :project_id,
+ :packages_packages,
+ :project_id,
+ :package_id
+ ]
+ )
+ end
+end
diff --git a/db/schema_migrations/20240930121826 b/db/schema_migrations/20240930121826
new file mode 100644
index 0000000000000000000000000000000000000000..cee1343df809d392d2e04051ab76e84b8c2da8c3
--- /dev/null
+++ b/db/schema_migrations/20240930121826
@@ -0,0 +1 @@
+8bcd22d80c42565f63b10f6e770d1fc6e298155e23b9d600322f0a60162d8f6f
\ No newline at end of file
diff --git a/db/schema_migrations/20240930121827 b/db/schema_migrations/20240930121827
new file mode 100644
index 0000000000000000000000000000000000000000..ba8062bd856d9e5a0fdb85cb1f9fb2ce6870c521
--- /dev/null
+++ b/db/schema_migrations/20240930121827
@@ -0,0 +1 @@
+a218cdf1cd33cf8aa2be34db650ce3acee4ee8908924e02fe3eeb10834d0f902
\ No newline at end of file
diff --git a/db/schema_migrations/20240930121828 b/db/schema_migrations/20240930121828
new file mode 100644
index 0000000000000000000000000000000000000000..cc66a479c0b23b2a26f03e134187a3b8253a9a9b
--- /dev/null
+++ b/db/schema_migrations/20240930121828
@@ -0,0 +1 @@
+9e855dcd7310302142e9c434f27a57df6ec1fc6abe7a615833e4427e13975871
\ No newline at end of file
diff --git a/db/schema_migrations/20240930121829 b/db/schema_migrations/20240930121829
new file mode 100644
index 0000000000000000000000000000000000000000..a1897ba541ce93c6682228fd28980e3fbe3e879c
--- /dev/null
+++ b/db/schema_migrations/20240930121829
@@ -0,0 +1 @@
+fb139a0a83d84d8a4c296974ac6fc5698810425b5d0c112878fc47ffd2bf3237
\ No newline at end of file
diff --git a/db/schema_migrations/20240930121830 b/db/schema_migrations/20240930121830
new file mode 100644
index 0000000000000000000000000000000000000000..458584909bb21e44c097a55f2753fefa923241f4
--- /dev/null
+++ b/db/schema_migrations/20240930121830
@@ -0,0 +1 @@
+c17f2fa09d28e36f0d8f5ca0fba72e303090a27fb505416199fb531b8cdcdbb0
\ No newline at end of file
diff --git a/db/structure.sql b/db/structure.sql
index 65102b234b3f014dea899e8d44307c20c78f236e..28f58af15e0cee5145004dfb9fdfd4642c8bc930 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -1337,6 +1337,22 @@ RETURN NEW;
END
$$;
+CREATE FUNCTION trigger_44ff19ad0ab2() RETURNS trigger
+ LANGUAGE plpgsql
+ AS $$
+BEGIN
+IF NEW."project_id" IS NULL THEN
+ SELECT "project_id"
+ INTO NEW."project_id"
+ FROM "packages_packages"
+ WHERE "packages_packages"."id" = NEW."package_id";
+END IF;
+
+RETURN NEW;
+
+END
+$$;
+
CREATE FUNCTION trigger_46ebe375f632() RETURNS trigger
LANGUAGE plpgsql
AS $$
@@ -15636,6 +15652,7 @@ CREATE TABLE packages_pypi_metadata (
author_email text,
description text,
description_content_type text,
+ project_id bigint,
CONSTRAINT check_0d9aed55b2 CHECK ((required_python IS NOT NULL)),
CONSTRAINT check_222e4f5b58 CHECK ((char_length(keywords) <= 1024)),
CONSTRAINT check_2d3ed32225 CHECK ((char_length(metadata_version) <= 16)),
@@ -30162,6 +30179,8 @@ CREATE INDEX index_packages_packages_on_project_id_and_version ON packages_packa
CREATE INDEX index_packages_project_id_name_partial_for_nuget ON packages_packages USING btree (project_id, name) WHERE (((name)::text <> 'NuGet.Temporary.Package'::text) AND (version IS NOT NULL) AND (package_type = 4));
+CREATE INDEX index_packages_pypi_metadata_on_project_id ON packages_pypi_metadata USING btree (project_id);
+
CREATE INDEX index_packages_rpm_metadata_on_package_id ON packages_rpm_metadata USING btree (package_id);
CREATE INDEX index_packages_rpm_metadata_on_project_id ON packages_rpm_metadata USING btree (project_id);
@@ -33612,6 +33631,8 @@ CREATE TRIGGER trigger_43484cb41aca BEFORE INSERT OR UPDATE ON wiki_repository_s
CREATE TRIGGER trigger_44558add1625 BEFORE INSERT OR UPDATE ON merge_request_assignees FOR EACH ROW EXECUTE FUNCTION trigger_44558add1625();
+CREATE TRIGGER trigger_44ff19ad0ab2 BEFORE INSERT OR UPDATE ON packages_pypi_metadata FOR EACH ROW EXECUTE FUNCTION trigger_44ff19ad0ab2();
+
CREATE TRIGGER trigger_46ebe375f632 BEFORE INSERT OR UPDATE ON epic_issues FOR EACH ROW EXECUTE FUNCTION trigger_46ebe375f632();
CREATE TRIGGER trigger_49862b4b3035 BEFORE INSERT OR UPDATE ON approval_group_rules_protected_branches FOR EACH ROW EXECUTE FUNCTION trigger_49862b4b3035();
@@ -34611,6 +34632,9 @@ ALTER TABLE ONLY packages_package_files
ALTER TABLE p_ci_builds
ADD CONSTRAINT fk_87f4cefcda_p FOREIGN KEY (upstream_pipeline_partition_id, upstream_pipeline_id) REFERENCES p_ci_pipelines(partition_id, id) ON UPDATE CASCADE ON DELETE CASCADE;
+ALTER TABLE ONLY packages_pypi_metadata
+ ADD CONSTRAINT fk_884056a10f FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE;
+
ALTER TABLE ONLY approval_group_rules_users
ADD CONSTRAINT fk_888a0df3b7 FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE;
diff --git a/lib/gitlab/background_migration/backfill_packages_pypi_metadata_project_id.rb b/lib/gitlab/background_migration/backfill_packages_pypi_metadata_project_id.rb
new file mode 100644
index 0000000000000000000000000000000000000000..501c88d305a6b175861995ab7fd8eb19ce26045f
--- /dev/null
+++ b/lib/gitlab/background_migration/backfill_packages_pypi_metadata_project_id.rb
@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+module Gitlab
+ module BackgroundMigration
+ class BackfillPackagesPypiMetadataProjectId < BackfillDesiredShardingKeyJob
+ operation_name :backfill_packages_pypi_metadata_project_id
+ feature_category :package_registry
+ end
+ end
+end
diff --git a/spec/lib/gitlab/background_migration/backfill_packages_pypi_metadata_project_id_spec.rb b/spec/lib/gitlab/background_migration/backfill_packages_pypi_metadata_project_id_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..8de8dfd476e87096e459776d5485bcb41e087c44
--- /dev/null
+++ b/spec/lib/gitlab/background_migration/backfill_packages_pypi_metadata_project_id_spec.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::BackfillPackagesPypiMetadataProjectId,
+ feature_category: :package_registry,
+ schema: 20240930121826 do
+ include_examples 'desired sharding key backfill job' do
+ let(:batch_table) { :packages_pypi_metadata }
+ let(:batch_column) { :package_id }
+ let(:backfill_column) { :project_id }
+ let(:backfill_via_table) { :packages_packages }
+ let(:backfill_via_column) { :project_id }
+ let(:backfill_via_foreign_key) { :package_id }
+ end
+end
diff --git a/spec/migrations/20240930121830_queue_backfill_packages_pypi_metadata_project_id_spec.rb b/spec/migrations/20240930121830_queue_backfill_packages_pypi_metadata_project_id_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..dbe38483f8c035bad4781df83ad51132c627ad7b
--- /dev/null
+++ b/spec/migrations/20240930121830_queue_backfill_packages_pypi_metadata_project_id_spec.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe QueueBackfillPackagesPypiMetadataProjectId, feature_category: :package_registry do
+ let!(:batched_migration) { described_class::MIGRATION }
+
+ it 'schedules a new batched migration' do
+ reversible_migration do |migration|
+ migration.before -> {
+ expect(batched_migration).not_to have_scheduled_batched_migration
+ }
+
+ migration.after -> {
+ expect(batched_migration).to have_scheduled_batched_migration(
+ table_name: :packages_pypi_metadata,
+ column_name: :package_id,
+ interval: described_class::DELAY_INTERVAL,
+ batch_size: described_class::BATCH_SIZE,
+ sub_batch_size: described_class::SUB_BATCH_SIZE,
+ gitlab_schema: :gitlab_main_cell,
+ job_arguments: [
+ :project_id,
+ :packages_packages,
+ :project_id,
+ :package_id
+ ]
+ )
+ }
+ end
+ end
+end