diff --git a/db/post_migrate/20240409070036_sync_index_for_pipelines_unique_id_bigint.rb b/db/post_migrate/20240409070036_sync_index_for_pipelines_unique_id_bigint.rb new file mode 100644 index 0000000000000000000000000000000000000000..5f614ef10221609ae2545ef3564a213a403942d8 --- /dev/null +++ b/db/post_migrate/20240409070036_sync_index_for_pipelines_unique_id_bigint.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class SyncIndexForPipelinesUniqueIdBigint < Gitlab::Database::Migration[2.2] + milestone '17.0' + disable_ddl_transaction! + + TABLE = :ci_pipelines + INDEX = { + name: :index_ci_pipelines_on_id_convert_to_bigint, + columns: [:id_convert_to_bigint], + options: { unique: true } + } + + def up + add_concurrent_index TABLE, INDEX[:columns], name: INDEX[:name], **INDEX[:options] + end + + def down + remove_concurrent_index_by_name TABLE, INDEX[:name] + end +end diff --git a/db/post_migrate/20240410070036_prepare_async_indexes_for_pipelines_id_bigint.rb b/db/post_migrate/20240410070036_prepare_async_indexes_for_pipelines_id_bigint.rb new file mode 100644 index 0000000000000000000000000000000000000000..a7cf2ee2cfda43c7c892faa2e05cbcfbe3b96fb4 --- /dev/null +++ b/db/post_migrate/20240410070036_prepare_async_indexes_for_pipelines_id_bigint.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +class PrepareAsyncIndexesForPipelinesIdBigint < Gitlab::Database::Migration[2.2] + milestone '17.0' + disable_ddl_transaction! + + TABLE = :ci_pipelines + INDEXES = [ + { + name: :idx_ci_pipelines_artifacts_locked_bigint, + columns: [:ci_ref_id, :id_convert_to_bigint], + options: { where: 'locked = 1' } + }, + { + name: :index_ci_pipelines_for_ondemand_dast_scans_bigint, + columns: [:id_convert_to_bigint], + options: { where: 'source = 13' } + }, + { + name: :index_ci_pipelines_on_ci_ref_id_and_more_bigint, + columns: [:ci_ref_id, :id_convert_to_bigint, :source, :status], + options: { order: { id_convert_to_bigint: :desc }, where: 'ci_ref_id IS NOT NULL' } + }, + { + name: :index_ci_pipelines_on_pipeline_schedule_id_and_id_bigint, + columns: [:pipeline_schedule_id, :id_convert_to_bigint] + }, + { + name: :index_ci_pipelines_on_project_id_and_id_desc_bigint, + columns: [:project_id, :id_convert_to_bigint], + options: { order: { id_convert_to_bigint: :desc } } + }, + { + name: :idx_ci_pipelines_on_project_id_and_ref_and_status_and_id_bigint, + columns: [:project_id, :ref, :status, :id_convert_to_bigint] + }, + { + name: :index_ci_pipelines_on_project_id_and_ref_and_id_desc_bigint, + columns: [:project_id, :ref, :id_convert_to_bigint], + options: { order: { id_convert_to_bigint: :desc } } + }, + { + name: :index_ci_pipelines_on_status_and_id_bigint, + columns: [:status, :id_convert_to_bigint] + }, + { + name: :idx_ci_pipelines_on_user_id_and_id_and_cancelable_status_bigint, + columns: [:user_id, :id_convert_to_bigint], + options: { where: "((status)::text = ANY (ARRAY[('running'::character varying)::text, ('waiting_for_resource'::character varying)::text, ('preparing'::character varying)::text, ('pending'::character varying)::text, ('created'::character varying)::text, ('scheduled'::character varying)::text]))" } # rubocop:disable Layout/LineLength -- Just too long + }, + { + name: :idx_ci_pipelines_on_user_id_and_user_not_verified_bigint, + columns: [:user_id, :id_convert_to_bigint], + options: { order: { id_convert_to_bigint: :desc }, where: "failure_reason = 3" } + } + ] + + def up + INDEXES.each do |index| + prepare_async_index(TABLE, index[:columns], name: index[:name], **(index[:options] || {})) + end + end + + def down + INDEXES.each do |index| + unprepare_async_index(TABLE, index[:columns], name: index[:name]) + end + end +end diff --git a/db/post_migrate/20240411070036_async_fk_id_bigint4_ci_pipelines_p_ci_builds_ci_pipelines_config_p_ci_stages.rb b/db/post_migrate/20240411070036_async_fk_id_bigint4_ci_pipelines_p_ci_builds_ci_pipelines_config_p_ci_stages.rb new file mode 100644 index 0000000000000000000000000000000000000000..df6c1403a457a59ecbfbad76af27c7859dcaac56 --- /dev/null +++ b/db/post_migrate/20240411070036_async_fk_id_bigint4_ci_pipelines_p_ci_builds_ci_pipelines_config_p_ci_stages.rb @@ -0,0 +1,175 @@ +# frozen_string_literal: true + +class AsyncFkIdBigint4CiPipelinesPCiBuildsCiPipelinesConfigPCiStages < Gitlab::Database::Migration[2.2] + include ::Gitlab::Database::PartitioningMigrationHelpers + + milestone '17.0' + disable_ddl_transaction! + + TABLE = :ci_pipelines + FOREIGN_KEYS = [ + { + source_table: :ci_pipelines, + options: { + name: :fk_262d4c2d19_tmp, + column: [:auto_canceled_by_id], + on_delete: :nullify + } + }, + { + source_table: :ci_pipeline_chat_data, + options: { + name: :fk_64ebfab6b3_tmp, + column: [:pipeline_id], + on_delete: :cascade + } + }, + { + source_table: :ci_sources_pipelines, + options: { + name: :fk_d4e29af7d7_tmp, + column: [:source_pipeline_id], + on_delete: :cascade + } + }, + { + source_table: :ci_sources_pipelines, + options: { + name: :fk_e1bad85861_tmp, + column: [:pipeline_id], + on_delete: :cascade + } + }, + { + source_table: :ci_sources_projects, + options: { + name: :fk_rails_10a1eb379a_tmp, + column: [:pipeline_id], + on_delete: :cascade + } + }, + { + source_table: :ci_pipeline_metadata, + options: { + name: :fk_rails_50c1e9ea10_tmp, + column: [:pipeline_id], + on_delete: :cascade + } + }, + { + source_table: :ci_pipeline_messages, + options: { + name: :fk_rails_8d3b04e3e1_tmp, + column: [:pipeline_id], + on_delete: :cascade + } + }, + { + source_table: :ci_pipelines_config, + options: { + name: :fk_rails_906c9a2533_tmp, + column: [:pipeline_id], + on_delete: :cascade + } + }, + { + source_table: :ci_pipeline_artifacts, + options: { + name: :fk_rails_a9e811a466_tmp, + column: [:pipeline_id], + on_delete: :cascade + } + }, + { + source_table: :ci_daily_build_group_report_results, + options: { + name: :fk_rails_ee072d13b3_tmp, + column: [:last_pipeline_id], + on_delete: :cascade + } + } + ] + + P_FOREIGN_KEYS = [ + { + source_table: :p_ci_builds, + options: { + name: :fk_87f4cefcda_tmp, + column: [:upstream_pipeline_id], + on_delete: :cascade + } + }, + { + source_table: :p_ci_builds, + options: { + name: :fk_d3130c9a7f_tmp, + column: [:commit_id], + on_delete: :cascade + } + }, + { + source_table: :p_ci_builds, + options: { + name: :fk_a2141b1522_tmp, + column: [:auto_canceled_by_id], + on_delete: :nullify + } + }, + { + source_table: :p_ci_pipeline_variables, + options: { + name: :fk_f29c5f4380_tmp, + column: [:pipeline_id], + on_delete: :cascade + } + }, + { + source_table: :p_ci_stages, + options: { + name: :fk_fb57e6cc56_tmp, + column: [:pipeline_id], + on_delete: :cascade + } + } + ] + + def up + FOREIGN_KEYS.each do |fk| + add_concurrent_foreign_key(fk[:source_table], TABLE, **default_options.merge(fk[:options])) + prepare_async_foreign_key_validation(fk[:source_table], name: fk[:options][:name]) + end + + P_FOREIGN_KEYS.each do |fk| + add_concurrent_partitioned_foreign_key(fk[:source_table], TABLE, **default_options.merge(fk[:options])) + prepare_partitioned_async_foreign_key_validation(fk[:source_table], name: fk[:options][:name]) + end + end + + def down + FOREIGN_KEYS.each do |fk| + with_lock_retries do + remove_foreign_key_if_exists( + fk[:source_table], TABLE, + name: fk[:options][:name], reverse_lock_order: true + ) + end + end + + P_FOREIGN_KEYS.each do |fk| + Gitlab::Database::PostgresPartitionedTable.each_partition(fk[:source_table]) do |partition| + with_lock_retries do + remove_foreign_key_if_exists( + partition.identifier, TABLE, + name: fk[:options][:name], reverse_lock_order: true + ) + end + end + end + end + + private + + def default_options + { target_column: [:id_convert_to_bigint], reverse_lock_order: true, validate: false } + end +end diff --git a/db/schema_migrations/20240409070036 b/db/schema_migrations/20240409070036 new file mode 100644 index 0000000000000000000000000000000000000000..43bae4452a7899f44a0b6f4ee0d3e6498e6c0b67 --- /dev/null +++ b/db/schema_migrations/20240409070036 @@ -0,0 +1 @@ +feba50a7cbaec374e92252d4d2ef5c0a2849dd43d402f47ea0913a5d1b6c71b1 \ No newline at end of file diff --git a/db/schema_migrations/20240410070036 b/db/schema_migrations/20240410070036 new file mode 100644 index 0000000000000000000000000000000000000000..1bd1330479fa43b5cd25e5fe61839294d4d159e6 --- /dev/null +++ b/db/schema_migrations/20240410070036 @@ -0,0 +1 @@ +be5d0c0415bf06c0e07b894ae97d8e10c237b469c9b83411ca55c7fe5e25688a \ No newline at end of file diff --git a/db/schema_migrations/20240411070036 b/db/schema_migrations/20240411070036 new file mode 100644 index 0000000000000000000000000000000000000000..8e0f53a9cb6dbc660d95bf8577e7c9085ba2e664 --- /dev/null +++ b/db/schema_migrations/20240411070036 @@ -0,0 +1 @@ +0ee6f8cf184da3306d03bb555db4fd9a2f96d960594653761e8a672ed3fcbb09 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index f0eab50afdd66b451f80db4c69df9cdc9d5e4e60..0269e419e31c2abd2712e9e46b8845f19a31fa6d 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -24760,6 +24760,8 @@ CREATE INDEX index_ci_pipelines_on_ci_ref_id_and_more ON ci_pipelines USING btre CREATE INDEX index_ci_pipelines_on_external_pull_request_id ON ci_pipelines USING btree (external_pull_request_id) WHERE (external_pull_request_id IS NOT NULL); +CREATE UNIQUE INDEX index_ci_pipelines_on_id_convert_to_bigint ON ci_pipelines USING btree (id_convert_to_bigint); + CREATE INDEX index_ci_pipelines_on_merge_request_id ON ci_pipelines USING btree (merge_request_id) WHERE (merge_request_id IS NOT NULL); CREATE INDEX index_ci_pipelines_on_pipeline_schedule_id_and_id ON ci_pipelines USING btree (pipeline_schedule_id, id); @@ -29864,6 +29866,9 @@ ALTER TABLE ONLY zoekt_repositories ALTER TABLE ONLY ci_pipelines ADD CONSTRAINT fk_262d4c2d19 FOREIGN KEY (auto_canceled_by_id) REFERENCES ci_pipelines(id) ON DELETE SET NULL; +ALTER TABLE ONLY ci_pipelines + ADD CONSTRAINT fk_262d4c2d19_tmp FOREIGN KEY (auto_canceled_by_id) REFERENCES ci_pipelines(id_convert_to_bigint) ON DELETE SET NULL NOT VALID; + ALTER TABLE ONLY geo_event_log ADD CONSTRAINT fk_27548c6db3 FOREIGN KEY (hashed_storage_migrated_event_id) REFERENCES geo_hashed_storage_migrated_events(id) ON DELETE CASCADE; @@ -30134,6 +30139,9 @@ ALTER TABLE ONLY approval_group_rules ALTER TABLE ONLY ci_pipeline_chat_data ADD CONSTRAINT fk_64ebfab6b3 FOREIGN KEY (pipeline_id) REFERENCES ci_pipelines(id) ON DELETE CASCADE; +ALTER TABLE ONLY ci_pipeline_chat_data + ADD CONSTRAINT fk_64ebfab6b3_tmp FOREIGN KEY (pipeline_id) REFERENCES ci_pipelines(id_convert_to_bigint) ON DELETE CASCADE NOT VALID; + ALTER TABLE ONLY cluster_agent_tokens ADD CONSTRAINT fk_64f741f626 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; @@ -30278,6 +30286,9 @@ ALTER TABLE ONLY packages_package_files ALTER TABLE p_ci_builds ADD CONSTRAINT fk_87f4cefcda FOREIGN KEY (upstream_pipeline_id) REFERENCES ci_pipelines(id) ON DELETE CASCADE; +ALTER TABLE ONLY ci_builds + ADD CONSTRAINT fk_87f4cefcda_tmp FOREIGN KEY (upstream_pipeline_id) REFERENCES ci_pipelines(id_convert_to_bigint) ON DELETE CASCADE NOT VALID; + ALTER TABLE ONLY approval_group_rules_users ADD CONSTRAINT fk_888a0df3b7 FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE; @@ -30401,6 +30412,9 @@ ALTER TABLE ONLY subscription_add_on_purchases ALTER TABLE p_ci_builds ADD CONSTRAINT fk_a2141b1522 FOREIGN KEY (auto_canceled_by_id) REFERENCES ci_pipelines(id) ON DELETE SET NULL; +ALTER TABLE ONLY ci_builds + ADD CONSTRAINT fk_a2141b1522_tmp FOREIGN KEY (auto_canceled_by_id) REFERENCES ci_pipelines(id_convert_to_bigint) ON DELETE SET NULL NOT VALID; + ALTER TABLE ONLY bulk_import_entities ADD CONSTRAINT fk_a44ff95be5 FOREIGN KEY (parent_id) REFERENCES bulk_import_entities(id) ON DELETE CASCADE; @@ -30656,9 +30670,15 @@ ALTER TABLE ONLY environments ALTER TABLE p_ci_builds ADD CONSTRAINT fk_d3130c9a7f FOREIGN KEY (commit_id) REFERENCES ci_pipelines(id) ON DELETE CASCADE; +ALTER TABLE ONLY ci_builds + ADD CONSTRAINT fk_d3130c9a7f_tmp FOREIGN KEY (commit_id) REFERENCES ci_pipelines(id_convert_to_bigint) ON DELETE CASCADE NOT VALID; + ALTER TABLE ONLY ci_sources_pipelines ADD CONSTRAINT fk_d4e29af7d7 FOREIGN KEY (source_pipeline_id) REFERENCES ci_pipelines(id) ON DELETE CASCADE; +ALTER TABLE ONLY ci_sources_pipelines + ADD CONSTRAINT fk_d4e29af7d7_tmp FOREIGN KEY (source_pipeline_id) REFERENCES ci_pipelines(id_convert_to_bigint) ON DELETE CASCADE NOT VALID; + ALTER TABLE ONLY incident_management_timeline_events ADD CONSTRAINT fk_d606a2a890 FOREIGN KEY (promoted_from_note_id) REFERENCES notes(id) ON DELETE SET NULL; @@ -30731,6 +30751,9 @@ ALTER TABLE ONLY ci_resources ALTER TABLE ONLY ci_sources_pipelines ADD CONSTRAINT fk_e1bad85861 FOREIGN KEY (pipeline_id) REFERENCES ci_pipelines(id) ON DELETE CASCADE; +ALTER TABLE ONLY ci_sources_pipelines + ADD CONSTRAINT fk_e1bad85861_tmp FOREIGN KEY (pipeline_id) REFERENCES ci_pipelines(id_convert_to_bigint) ON DELETE CASCADE NOT VALID; + ALTER TABLE p_ci_builds_metadata ADD CONSTRAINT fk_e20479742e_p FOREIGN KEY (partition_id, build_id) REFERENCES p_ci_builds(partition_id, id) ON UPDATE CASCADE ON DELETE CASCADE; @@ -30827,6 +30850,9 @@ ALTER TABLE ONLY boards ALTER TABLE p_ci_pipeline_variables ADD CONSTRAINT fk_f29c5f4380 FOREIGN KEY (pipeline_id) REFERENCES ci_pipelines(id) ON DELETE CASCADE; +ALTER TABLE ONLY ci_pipeline_variables + ADD CONSTRAINT fk_f29c5f4380_tmp FOREIGN KEY (pipeline_id) REFERENCES ci_pipelines(id_convert_to_bigint) ON DELETE CASCADE NOT VALID; + ALTER TABLE ONLY zoekt_indices ADD CONSTRAINT fk_f34800a202 FOREIGN KEY (zoekt_node_id) REFERENCES zoekt_nodes(id) ON DELETE CASCADE; @@ -30863,6 +30889,9 @@ ALTER TABLE ONLY application_settings ALTER TABLE p_ci_stages ADD CONSTRAINT fk_fb57e6cc56 FOREIGN KEY (pipeline_id) REFERENCES ci_pipelines(id) ON DELETE CASCADE; +ALTER TABLE ONLY ci_stages + ADD CONSTRAINT fk_fb57e6cc56_tmp FOREIGN KEY (pipeline_id) REFERENCES ci_pipelines(id_convert_to_bigint) ON DELETE CASCADE NOT VALID; + ALTER TABLE ONLY agent_group_authorizations ADD CONSTRAINT fk_fb70782616 FOREIGN KEY (agent_id) REFERENCES cluster_agents(id) ON DELETE CASCADE; @@ -31022,6 +31051,9 @@ ALTER TABLE ONLY audit_events_streaming_headers ALTER TABLE ONLY ci_sources_projects ADD CONSTRAINT fk_rails_10a1eb379a FOREIGN KEY (pipeline_id) REFERENCES ci_pipelines(id) ON DELETE CASCADE; +ALTER TABLE ONLY ci_sources_projects + ADD CONSTRAINT fk_rails_10a1eb379a_tmp FOREIGN KEY (pipeline_id) REFERENCES ci_pipelines(id_convert_to_bigint) ON DELETE CASCADE NOT VALID; + ALTER TABLE ONLY zoom_meetings ADD CONSTRAINT fk_rails_1190f0e0fa FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; @@ -31478,6 +31510,9 @@ ALTER TABLE ONLY status_page_settings ALTER TABLE ONLY ci_pipeline_metadata ADD CONSTRAINT fk_rails_50c1e9ea10 FOREIGN KEY (pipeline_id) REFERENCES ci_pipelines(id) ON DELETE CASCADE; +ALTER TABLE ONLY ci_pipeline_metadata + ADD CONSTRAINT fk_rails_50c1e9ea10_tmp FOREIGN KEY (pipeline_id) REFERENCES ci_pipelines(id_convert_to_bigint) ON DELETE CASCADE NOT VALID; + ALTER TABLE ONLY project_repository_storage_moves ADD CONSTRAINT fk_rails_5106dbd44a FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; @@ -31922,6 +31957,9 @@ ALTER TABLE ONLY vulnerability_feedback ALTER TABLE ONLY ci_pipeline_messages ADD CONSTRAINT fk_rails_8d3b04e3e1 FOREIGN KEY (pipeline_id) REFERENCES ci_pipelines(id) ON DELETE CASCADE; +ALTER TABLE ONLY ci_pipeline_messages + ADD CONSTRAINT fk_rails_8d3b04e3e1_tmp FOREIGN KEY (pipeline_id) REFERENCES ci_pipelines(id_convert_to_bigint) ON DELETE CASCADE NOT VALID; + ALTER TABLE incident_management_pending_alert_escalations ADD CONSTRAINT fk_rails_8d8de95da9 FOREIGN KEY (alert_id) REFERENCES alert_management_alerts(id) ON DELETE CASCADE; @@ -31949,6 +31987,9 @@ ALTER TABLE ONLY organization_details ALTER TABLE ONLY ci_pipelines_config ADD CONSTRAINT fk_rails_906c9a2533 FOREIGN KEY (pipeline_id) REFERENCES ci_pipelines(id) ON DELETE CASCADE; +ALTER TABLE ONLY ci_pipelines_config + ADD CONSTRAINT fk_rails_906c9a2533_tmp FOREIGN KEY (pipeline_id) REFERENCES ci_pipelines(id_convert_to_bigint) ON DELETE CASCADE NOT VALID; + ALTER TABLE ONLY approval_project_rules_groups ADD CONSTRAINT fk_rails_9071e863d1 FOREIGN KEY (approval_project_rule_id) REFERENCES approval_project_rules(id) ON DELETE CASCADE; @@ -32135,6 +32176,9 @@ ALTER TABLE ONLY saved_replies ALTER TABLE ONLY ci_pipeline_artifacts ADD CONSTRAINT fk_rails_a9e811a466 FOREIGN KEY (pipeline_id) REFERENCES ci_pipelines(id) ON DELETE CASCADE; +ALTER TABLE ONLY ci_pipeline_artifacts + ADD CONSTRAINT fk_rails_a9e811a466_tmp FOREIGN KEY (pipeline_id) REFERENCES ci_pipelines(id_convert_to_bigint) ON DELETE CASCADE NOT VALID; + ALTER TABLE ONLY merge_request_user_mentions ADD CONSTRAINT fk_rails_aa1b2961b1 FOREIGN KEY (merge_request_id) REFERENCES merge_requests(id) ON DELETE CASCADE; @@ -32636,6 +32680,9 @@ ALTER TABLE ONLY packages_debian_group_distributions ALTER TABLE ONLY ci_daily_build_group_report_results ADD CONSTRAINT fk_rails_ee072d13b3 FOREIGN KEY (last_pipeline_id) REFERENCES ci_pipelines(id) ON DELETE CASCADE; +ALTER TABLE ONLY ci_daily_build_group_report_results + ADD CONSTRAINT fk_rails_ee072d13b3_tmp FOREIGN KEY (last_pipeline_id) REFERENCES ci_pipelines(id_convert_to_bigint) ON DELETE CASCADE NOT VALID; + ALTER TABLE ONLY audit_events_group_streaming_event_type_filters ADD CONSTRAINT fk_rails_ee6950967f FOREIGN KEY (external_streaming_destination_id) REFERENCES audit_events_group_external_streaming_destinations(id) ON DELETE CASCADE;