diff --git a/app/models/ci/build_trace_metadata.rb b/app/models/ci/build_trace_metadata.rb index 292b520b7708467ae214874363b2e9629c58ed2b..b47493e7047e33a5c16280dbe79c5d599e186389 100644 --- a/app/models/ci/build_trace_metadata.rb +++ b/app/models/ci/build_trace_metadata.rb @@ -8,6 +8,8 @@ class BuildTraceMetadata < Ci::ApplicationRecord self.table_name = :p_ci_build_trace_metadata self.primary_key = :build_id + before_validation :set_project_id, on: :create + belongs_to :build, ->(trace_metadata) { in_partition(trace_metadata) }, class_name: 'Ci::Build', @@ -71,5 +73,9 @@ def remote_checksum_valid? def backoff ::Gitlab::Ci::Trace::Backoff.new(archival_attempts).value_with_jitter end + + def set_project_id + self.project_id ||= build&.project_id + end end end diff --git a/db/docs/batched_background_migrations/backfill_ci_build_trace_metadata_project_id.yml b/db/docs/batched_background_migrations/backfill_ci_build_trace_metadata_project_id.yml new file mode 100644 index 0000000000000000000000000000000000000000..cd3c2832916acf7dd88beb728cadb497206a027f --- /dev/null +++ b/db/docs/batched_background_migrations/backfill_ci_build_trace_metadata_project_id.yml @@ -0,0 +1,9 @@ +--- +migration_job_name: BackfillCiBuildTraceMetadataProjectId +description: Backfills column to be used for as sharding key +feature_category: continuous_integration +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/174780 +milestone: '17.7' +queued_migration_version: 20241204223441 +finalize_after: '2025-01-07' +finalized_by: # version of the migration that finalized this BBM diff --git a/db/docs/p_ci_build_trace_metadata.yml b/db/docs/p_ci_build_trace_metadata.yml index 17ded45a9ae9706d6eb7003597f29d05fa64ef2f..efb7c6231f1f95c1fa97a2c3419222bc0d33e343 100644 --- a/db/docs/p_ci_build_trace_metadata.yml +++ b/db/docs/p_ci_build_trace_metadata.yml @@ -18,4 +18,5 @@ desired_sharding_key: sharding_key: project_id belongs_to: build foreign_key_name: fk_rails_aebc78111f_p -table_size: small \ No newline at end of file +desired_sharding_key_migration_job_name: BackfillCiBuildTraceMetadataProjectId +table_size: small diff --git a/db/migrate/20241204221832_add_project_id_to_ci_build_trace_metadata.rb b/db/migrate/20241204221832_add_project_id_to_ci_build_trace_metadata.rb new file mode 100644 index 0000000000000000000000000000000000000000..2cd36d71996a650195e565c6975afbd3c37ac294 --- /dev/null +++ b/db/migrate/20241204221832_add_project_id_to_ci_build_trace_metadata.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class AddProjectIdToCiBuildTraceMetadata < Gitlab::Database::Migration[2.2] + milestone '17.7' + + def change + add_column :p_ci_build_trace_metadata, :project_id, :bigint + end +end diff --git a/db/post_migrate/20241204223047_index_ci_build_trace_metadata_on_project_id.rb b/db/post_migrate/20241204223047_index_ci_build_trace_metadata_on_project_id.rb new file mode 100644 index 0000000000000000000000000000000000000000..b34793e01727a6634a861db564fffdf306191564 --- /dev/null +++ b/db/post_migrate/20241204223047_index_ci_build_trace_metadata_on_project_id.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class IndexCiBuildTraceMetadataOnProjectId < Gitlab::Database::Migration[2.2] + include Gitlab::Database::PartitioningMigrationHelpers + + disable_ddl_transaction! + milestone '17.7' + + TABLE_NAME = :p_ci_build_trace_metadata + INDEX_NAME = :index_p_ci_build_trace_metadata_on_project_id + + def up + add_concurrent_partitioned_index(TABLE_NAME, :project_id, name: INDEX_NAME) + end + + def down + remove_concurrent_partitioned_index_by_name(TABLE_NAME, INDEX_NAME) + end +end diff --git a/db/post_migrate/20241204223441_queue_backfill_ci_build_trace_metadata_project_id.rb b/db/post_migrate/20241204223441_queue_backfill_ci_build_trace_metadata_project_id.rb new file mode 100644 index 0000000000000000000000000000000000000000..addbf30c94fc1f196dae1f0b5485ae76e492799d --- /dev/null +++ b/db/post_migrate/20241204223441_queue_backfill_ci_build_trace_metadata_project_id.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +class QueueBackfillCiBuildTraceMetadataProjectId < Gitlab::Database::Migration[2.2] + milestone '17.7' + + restrict_gitlab_migration gitlab_schema: :gitlab_ci + + MIGRATION = "BackfillCiBuildTraceMetadataProjectId" + DELAY_INTERVAL = 2.minutes + BATCH_SIZE = 25_000 + SUB_BATCH_SIZE = 150 + + def up + queue_batched_background_migration( + MIGRATION, + :p_ci_build_trace_metadata, + :build_id, + :project_id, + :p_ci_builds, + :project_id, + :build_id, + :partition_id, + job_interval: DELAY_INTERVAL, + batch_size: BATCH_SIZE, + sub_batch_size: SUB_BATCH_SIZE + ) + end + + def down + delete_batched_background_migration( + MIGRATION, + :p_ci_build_trace_metadata, + :build_id, + [ + :project_id, + :p_ci_builds, + :project_id, + :build_id, + :partition_id + ] + ) + end +end diff --git a/db/schema_migrations/20241204221832 b/db/schema_migrations/20241204221832 new file mode 100644 index 0000000000000000000000000000000000000000..3f2fafacf8b52ff1e2e0247e5806a1e9230c669b --- /dev/null +++ b/db/schema_migrations/20241204221832 @@ -0,0 +1 @@ +ceccd90d872fb1dd06033627c44facfcd850e0c12104fd3f0f9b17b4ef9e229a \ No newline at end of file diff --git a/db/schema_migrations/20241204223047 b/db/schema_migrations/20241204223047 new file mode 100644 index 0000000000000000000000000000000000000000..08e698510d3fab42e3be1350933c9fbd96aa5e2d --- /dev/null +++ b/db/schema_migrations/20241204223047 @@ -0,0 +1 @@ +0cc1fb84a9708e15122466e9c9ca798a582e09c33295f392fb0bb263bd072eed \ No newline at end of file diff --git a/db/schema_migrations/20241204223441 b/db/schema_migrations/20241204223441 new file mode 100644 index 0000000000000000000000000000000000000000..4d9c750ba16069e1e860c20fa1533ad73fb3a568 --- /dev/null +++ b/db/schema_migrations/20241204223441 @@ -0,0 +1 @@ +83abb51dce093eb183c205accfbddff9cd9d057cf49918ce7fd5fc7da976b12a \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 028c0fbe69dec8b3da43562c3510b0d7cccbc9b1..4cd1ef0559a46bdc6a33449e8434220257096d30 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -3477,7 +3477,8 @@ CREATE TABLE p_ci_build_trace_metadata ( archived_at timestamp with time zone, archival_attempts smallint DEFAULT 0 NOT NULL, checksum bytea, - remote_checksum bytea + remote_checksum bytea, + project_id bigint ) PARTITION BY LIST (partition_id); @@ -31839,6 +31840,8 @@ CREATE INDEX index_p_ci_build_tags_on_project_id ON ONLY p_ci_build_tags USING b CREATE UNIQUE INDEX index_p_ci_build_tags_on_tag_id_and_build_id_and_partition_id ON ONLY p_ci_build_tags USING btree (tag_id, build_id, partition_id); +CREATE INDEX index_p_ci_build_trace_metadata_on_project_id ON ONLY p_ci_build_trace_metadata USING btree (project_id); + CREATE INDEX index_p_ci_build_trace_metadata_on_trace_artifact_id ON ONLY p_ci_build_trace_metadata USING btree (trace_artifact_id); CREATE INDEX index_p_ci_builds_execution_configs_on_pipeline_id ON ONLY p_ci_builds_execution_configs USING btree (pipeline_id); diff --git a/lib/gitlab/background_migration/backfill_ci_build_trace_metadata_project_id.rb b/lib/gitlab/background_migration/backfill_ci_build_trace_metadata_project_id.rb new file mode 100644 index 0000000000000000000000000000000000000000..3aef988e02c6eec1eab8eec631533c20a37664e3 --- /dev/null +++ b/lib/gitlab/background_migration/backfill_ci_build_trace_metadata_project_id.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + class BackfillCiBuildTraceMetadataProjectId < BackfillDesiredShardingKeyPartitionJob + operation_name :backfill_ci_build_trace_metadata_project_id + feature_category :continuous_integration + end + end +end diff --git a/spec/db/schema_spec.rb b/spec/db/schema_spec.rb index 6863a8da66150f22ad00b310ea3732cc0969c143..c9ec9201431bf464102bbe1bd494b50a123eb8c9 100644 --- a/spec/db/schema_spec.rb +++ b/spec/db/schema_spec.rb @@ -188,6 +188,7 @@ p_ci_builds: %w[erased_by_id trigger_request_id partition_id auto_canceled_by_partition_id execution_config_id upstream_pipeline_partition_id], p_ci_builds_metadata: %w[project_id build_id partition_id], + p_ci_build_trace_metadata: %w[project_id], p_batched_git_ref_updates_deletions: %w[project_id partition_id], p_catalog_resource_sync_events: %w[catalog_resource_id project_id partition_id], p_catalog_resource_component_usages: %w[used_by_project_id], # No FK constraint because we want to preserve historical usage data diff --git a/spec/lib/gitlab/background_migration/backfill_ci_build_trace_metadata_project_id_spec.rb b/spec/lib/gitlab/background_migration/backfill_ci_build_trace_metadata_project_id_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..59143f2ee82e26890436bee35bec941fbc3074ee --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_ci_build_trace_metadata_project_id_spec.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillCiBuildTraceMetadataProjectId, + feature_category: :continuous_integration, migration: :gitlab_ci do + include_examples 'desired sharding key backfill job' do + let(:batch_table) { :p_ci_build_trace_metadata } + let(:batch_column) { :build_id } + let(:backfill_column) { :project_id } + let(:backfill_via_table) { :p_ci_builds } + let(:backfill_via_column) { :project_id } + let(:backfill_via_foreign_key) { :build_id } + let(:partition_column) { :partition_id } + end +end diff --git a/spec/migrations/20241204223441_queue_backfill_ci_build_trace_metadata_project_id_spec.rb b/spec/migrations/20241204223441_queue_backfill_ci_build_trace_metadata_project_id_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..15a121e17048a6956558d3ce48b03f930358e448 --- /dev/null +++ b/spec/migrations/20241204223441_queue_backfill_ci_build_trace_metadata_project_id_spec.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe QueueBackfillCiBuildTraceMetadataProjectId, migration: :gitlab_ci, feature_category: :continuous_integration do + let!(:batched_migration) { described_class::MIGRATION } + + it 'schedules a new batched migration' do + reversible_migration do |migration| + migration.before -> { + expect(batched_migration).not_to have_scheduled_batched_migration + } + + migration.after -> { + expect(batched_migration).to have_scheduled_batched_migration( + gitlab_schema: :gitlab_ci, + table_name: :p_ci_build_trace_metadata, + column_name: :build_id, + interval: described_class::DELAY_INTERVAL, + batch_size: described_class::BATCH_SIZE, + sub_batch_size: described_class::SUB_BATCH_SIZE, + job_arguments: [ + :project_id, + :p_ci_builds, + :project_id, + :build_id, + :partition_id + ] + ) + } + end + end +end diff --git a/spec/models/ci/build_trace_metadata_spec.rb b/spec/models/ci/build_trace_metadata_spec.rb index 4b5eb8da064a440d7c7cf4388d9c5e790781e8bb..811506d5b973b082bc03b0ab7382bb9360b72bba 100644 --- a/spec/models/ci/build_trace_metadata_spec.rb +++ b/spec/models/ci/build_trace_metadata_spec.rb @@ -176,4 +176,23 @@ expect(metadata.partition_id).to eq(ci_testing_partition_id) end end + + describe '#set_project_id' do + context 'when project_id is not set' do + let(:metadata) { create(:ci_build_trace_metadata) } + + it 'sets the project_id from the build' do + expect(metadata.project_id).to eq(metadata.build.project_id) + end + end + + context 'when project_id is set' do + let(:existing_project) { build_stubbed(:project) } + let(:metadata) { create(:ci_build_trace_metadata, project_id: existing_project.id) } + + it 'does not override the project_id' do + expect(metadata.project_id).to eq(existing_project.id) + end + end + end end