Merge branch 'backfill-desired-sharding-key-small-table-ci_builds_runner_session' into 'master'

Add and backfill project_id for ci_builds_runner_session

See merge request https://gitlab.com/gitlab-org/gitlab/-/merge_requests/167634



Merged-by: Michał Zając <mzajac@gitlab.com>
Approved-by: Adam Hegyi <ahegyi@gitlab.com>
Approved-by: João Pereira <jpereira@gitlab.com>
Approved-by: Michał Zając <mzajac@gitlab.com>
Co-authored-by: Shubham Kumar <shukumar@gitlab.com>

db/docs/batched_background_migrations/backfill_ci_builds_runner_session_project_id.yml

+---
+migration_job_name: BackfillCiBuildsRunnerSessionProjectId
+description: Backfills sharding key `ci_builds_runner_session.project_id` from `p_ci_builds`.
+feature_category: runner
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/167634
+milestone: '17.5'
+queued_migration_version: 20240930144643
+finalize_after: '2024-10-22'
+finalized_by: # version of the migration that finalized this BBM

db/docs/ci_builds_runner_session.yml

@@ -4,7 +4,8 @@ classes:
 - Ci::BuildRunnerSession
 feature_categories:
 - runner
-description: Store build-related runner session. Data is removed after the respective job transitions from running to any state.
+description: Store build-related runner session. Data is removed after the respective
+  job transitions from running to any state.
 introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/6208
 milestone: '11.1'
 gitlab_schema: gitlab_ci
@@ -18,3 +19,4 @@ desired_sharding_key:
         sharding_key: project_id
         belongs_to: build
         foreign_key_name: fk_rails_70707857d3_p
+desired_sharding_key_migration_job_name: BackfillCiBuildsRunnerSessionProjectId

db/migrate/20240930144640_add_project_id_to_ci_builds_runner_session.rb

+# frozen_string_literal: true
+
+class AddProjectIdToCiBuildsRunnerSession < Gitlab::Database::Migration[2.2]
+  milestone '17.5'
+
+  def change
+    add_column :ci_builds_runner_session, :project_id, :bigint
+  end
+end

db/post_migrate/20240930144641_index_ci_builds_runner_session_on_project_id.rb

+# frozen_string_literal: true
+
+class IndexCiBuildsRunnerSessionOnProjectId < Gitlab::Database::Migration[2.2]
+  milestone '17.5'
+  disable_ddl_transaction!
+
+  INDEX_NAME = 'index_ci_builds_runner_session_on_project_id'
+
+  def up
+    add_concurrent_index :ci_builds_runner_session, :project_id, name: INDEX_NAME
+  end
+
+  def down
+    remove_concurrent_index_by_name :ci_builds_runner_session, INDEX_NAME
+  end
+end

db/post_migrate/20240930144642_add_ci_builds_runner_session_project_id_trigger.rb

+# frozen_string_literal: true
+
+class AddCiBuildsRunnerSessionProjectIdTrigger < Gitlab::Database::Migration[2.2]
+  milestone '17.5'
+
+  def up
+    install_sharding_key_assignment_trigger(
+      table: :ci_builds_runner_session,
+      sharding_key: :project_id,
+      parent_table: :p_ci_builds,
+      parent_sharding_key: :project_id,
+      foreign_key: :build_id
+    )
+  end
+
+  def down
+    remove_sharding_key_assignment_trigger(
+      table: :ci_builds_runner_session,
+      sharding_key: :project_id,
+      parent_table: :p_ci_builds,
+      parent_sharding_key: :project_id,
+      foreign_key: :build_id
+    )
+  end
+end

db/post_migrate/20240930144643_queue_backfill_ci_builds_runner_session_project_id.rb

+# frozen_string_literal: true
+
+class QueueBackfillCiBuildsRunnerSessionProjectId < Gitlab::Database::Migration[2.2]
+  milestone '17.5'
+  restrict_gitlab_migration gitlab_schema: :gitlab_ci
+
+  MIGRATION = "BackfillCiBuildsRunnerSessionProjectId"
+  DELAY_INTERVAL = 2.minutes
+  BATCH_SIZE = 1000
+  SUB_BATCH_SIZE = 100
+
+  def up
+    queue_batched_background_migration(
+      MIGRATION,
+      :ci_builds_runner_session,
+      :id,
+      :project_id,
+      :p_ci_builds,
+      :project_id,
+      :build_id,
+      job_interval: DELAY_INTERVAL,
+      batch_size: BATCH_SIZE,
+      sub_batch_size: SUB_BATCH_SIZE
+    )
+  end
+
+  def down
+    delete_batched_background_migration(
+      MIGRATION,
+      :ci_builds_runner_session,
+      :id,
+      [
+        :project_id,
+        :p_ci_builds,
+        :project_id,
+        :build_id
+      ]
+    )
+  end
+end

db/schema_migrations/20240930144640

+15875b22bda9b9bfaedf0307133ae7fe2457afd97354c7802fbff085d4b7986a
\ No newline at end of file

db/schema_migrations/20240930144641

+b838586b920d690a9281775417a18c2ab6326258bf5dd5298c82f6b5c655204b
\ No newline at end of file

db/schema_migrations/20240930144642

+36e73986fdb23fa5dcbce209a40e633d3f8b05d4f6c16eacec4be2dd68fba91b
\ No newline at end of file

db/schema_migrations/20240930144643

+1938a7329e5f216c004d265cb8b84eac9e1b9c3d8b4c87caa8db759bd71a85d7
\ No newline at end of file

db/structure.sql

@@ -1177,6 +1177,22 @@ RETURN NEW;
 END
 $$;
+CREATE FUNCTION trigger_388de55cd36c() RETURNS trigger
+    LANGUAGE plpgsql
+    AS $$
+BEGIN
+IF NEW."project_id" IS NULL THEN
+  SELECT "project_id"
+  INTO NEW."project_id"
+  FROM "p_ci_builds"
+  WHERE "p_ci_builds"."id" = NEW."build_id";
+END IF;
+
+RETURN NEW;
+
+END
+$$;
+
 CREATE FUNCTION trigger_3d1a58344b29() RETURNS trigger
     LANGUAGE plpgsql
     AS $$
@@ -8169,7 +8185,8 @@ CREATE TABLE ci_builds_runner_session (
     certificate character varying,
     "authorization" character varying,
     build_id bigint NOT NULL,
-    partition_id bigint NOT NULL
+    partition_id bigint NOT NULL,
+    project_id bigint
 );
 CREATE SEQUENCE ci_builds_runner_session_id_seq
@@ -27892,6 +27909,8 @@ CREATE UNIQUE INDEX index_ci_builds_runner_session_on_build_id ON ci_builds_runn
 CREATE UNIQUE INDEX index_ci_builds_runner_session_on_partition_id_build_id ON ci_builds_runner_session USING btree (partition_id, build_id);
+CREATE INDEX index_ci_builds_runner_session_on_project_id ON ci_builds_runner_session USING btree (project_id);
+
 CREATE INDEX index_ci_daily_build_group_report_results_on_group_id ON ci_daily_build_group_report_results USING btree (group_id);
 CREATE INDEX index_ci_daily_build_group_report_results_on_last_pipeline_id ON ci_daily_build_group_report_results USING btree (last_pipeline_id);
@@ -33420,6 +33439,8 @@ CREATE TRIGGER trigger_30209d0fba3e BEFORE INSERT OR UPDATE ON alert_management_
 CREATE TRIGGER trigger_3691f9f6a69f BEFORE INSERT OR UPDATE ON remote_development_agent_configs FOR EACH ROW EXECUTE FUNCTION trigger_3691f9f6a69f();
+CREATE TRIGGER trigger_388de55cd36c BEFORE INSERT OR UPDATE ON ci_builds_runner_session FOR EACH ROW EXECUTE FUNCTION trigger_388de55cd36c();
+
 CREATE TRIGGER trigger_3d1a58344b29 BEFORE INSERT OR UPDATE ON alert_management_alert_assignees FOR EACH ROW EXECUTE FUNCTION trigger_3d1a58344b29();
 CREATE TRIGGER trigger_3e067fa9bfe3 BEFORE INSERT OR UPDATE ON incident_management_timeline_event_tag_links FOR EACH ROW EXECUTE FUNCTION trigger_3e067fa9bfe3();

lib/gitlab/background_migration/backfill_ci_builds_runner_session_project_id.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    class BackfillCiBuildsRunnerSessionProjectId < BackfillDesiredShardingKeyJob
+      operation_name :backfill_ci_builds_runner_session_project_id
+      feature_category :runner
+    end
+  end
+end

spec/db/schema_spec.rb

@@ -77,6 +77,7 @@
     ci_builds: %w[project_id runner_id user_id erased_by_id trigger_request_id partition_id auto_canceled_by_partition_id execution_config_id upstream_pipeline_partition_id],
     ci_builds_metadata: %w[partition_id project_id build_id],
     ci_build_needs: %w[project_id],
+    ci_builds_runner_session: %w[project_id],
     ci_daily_build_group_report_results: %w[partition_id],
     ci_deleted_objects: %w[project_id],
     ci_job_artifacts: %w[partition_id project_id job_id],

spec/lib/gitlab/background_migration/backfill_ci_builds_runner_session_project_id_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::BackfillCiBuildsRunnerSessionProjectId,
+  feature_category: :runner,
+  schema: 20240930144640,
+  migration: :gitlab_ci do
+  include_examples 'desired sharding key backfill job' do
+    let(:batch_table) { :ci_builds_runner_session }
+    let(:backfill_column) { :project_id }
+    let(:backfill_via_table) { :p_ci_builds }
+    let(:backfill_via_column) { :project_id }
+    let(:backfill_via_foreign_key) { :build_id }
+  end
+end

spec/migrations/20240930144644_queue_backfill_ci_builds_runner_session_project_id_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe QueueBackfillCiBuildsRunnerSessionProjectId, migration: :gitlab_ci, feature_category: :runner do
+  let!(:batched_migration) { described_class::MIGRATION }
+
+  it 'schedules a new batched migration' do
+    reversible_migration do |migration|
+      migration.before -> {
+        expect(batched_migration).not_to have_scheduled_batched_migration
+      }
+
+      migration.after -> {
+        expect(batched_migration).to have_scheduled_batched_migration(
+          table_name: :ci_builds_runner_session,
+          column_name: :id,
+          interval: described_class::DELAY_INTERVAL,
+          batch_size: described_class::BATCH_SIZE,
+          sub_batch_size: described_class::SUB_BATCH_SIZE,
+          gitlab_schema: :gitlab_ci,
+          job_arguments: [
+            :project_id,
+            :p_ci_builds,
+            :project_id,
+            :build_id
+          ]
+        )
+      }
+    end
+  end
+end