diff --git a/app/models/ci/build_need.rb b/app/models/ci/build_need.rb index aaec8a540116de76088b9ba5e8c73eb27e6148e2..317f2523f69f70881f22c95d3a80013ef89ccf20 100644 --- a/app/models/ci/build_need.rb +++ b/app/models/ci/build_need.rb @@ -3,9 +3,11 @@ module Ci class BuildNeed < Ci::ApplicationRecord include Ci::Partitionable - include BulkInsertSafe include IgnorableColumns + include SafelyChangeColumnDefault + include BulkInsertSafe + columns_changing_default :partition_id ignore_column :id_convert_to_bigint, remove_with: '16.4', remove_after: '2023-09-22' belongs_to :build, class_name: "Ci::Processable", foreign_key: :build_id, inverse_of: :needs diff --git a/app/models/ci/build_pending_state.rb b/app/models/ci/build_pending_state.rb index 966884ae15854249d956ecdce42cd8b4be34abf9..0b88f745d78faf2e8916f39a006ebf045e30b5b5 100644 --- a/app/models/ci/build_pending_state.rb +++ b/app/models/ci/build_pending_state.rb @@ -2,6 +2,9 @@ class Ci::BuildPendingState < Ci::ApplicationRecord include Ci::Partitionable + include SafelyChangeColumnDefault + + columns_changing_default :partition_id belongs_to :build, class_name: 'Ci::Build', foreign_key: :build_id, inverse_of: :pending_state diff --git a/app/models/ci/build_report_result.rb b/app/models/ci/build_report_result.rb index b2d99fab295e113d379e9c79722d56ed67654455..90b621b8da1f4e6497cb3e4b6fdebdccb95b7d75 100644 --- a/app/models/ci/build_report_result.rb +++ b/app/models/ci/build_report_result.rb @@ -3,6 +3,9 @@ module Ci class BuildReportResult < Ci::ApplicationRecord include Ci::Partitionable + include SafelyChangeColumnDefault + + columns_changing_default :partition_id self.primary_key = :build_id diff --git a/app/models/ci/build_runner_session.rb b/app/models/ci/build_runner_session.rb index 5773b6132bea54739f8df5d14abfdab11c4ebade..eaa2e1c428e6ba844663f91923262020ae7e2e60 100644 --- a/app/models/ci/build_runner_session.rb +++ b/app/models/ci/build_runner_session.rb @@ -5,6 +5,9 @@ module Ci # Data will be removed after transitioning from running to any state. class BuildRunnerSession < Ci::ApplicationRecord include Ci::Partitionable + include SafelyChangeColumnDefault + + columns_changing_default :partition_id TERMINAL_SUBPROTOCOL = 'terminal.gitlab.com' DEFAULT_SERVICE_NAME = 'build' diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index 7ab89533d7637eb3cbf6dfc3d90ba11ea6296afa..0a0f401c9d5903da8c23e59d5ade002dff20d5b3 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -8,6 +8,9 @@ class BuildTraceChunk < Ci::ApplicationRecord include ::Checksummable include ::Gitlab::ExclusiveLeaseHelpers include ::Gitlab::OptimisticLocking + include SafelyChangeColumnDefault + + columns_changing_default :partition_id belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id, inverse_of: :trace_chunks diff --git a/app/models/ci/build_trace_metadata.rb b/app/models/ci/build_trace_metadata.rb index 4c76089617f044432c6c6e464f3a8226b487c973..c5ad3d1942583ca98ab24a86860f624b1d93b844 100644 --- a/app/models/ci/build_trace_metadata.rb +++ b/app/models/ci/build_trace_metadata.rb @@ -3,6 +3,9 @@ module Ci class BuildTraceMetadata < Ci::ApplicationRecord include Ci::Partitionable + include SafelyChangeColumnDefault + + columns_changing_default :partition_id MAX_ATTEMPTS = 5 self.table_name = 'ci_build_trace_metadata' diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index 5cd7988837e12b9eeb2b06af9aa49d9428097bbf..11d70e088e9e2d84703bd4568afc06fdb23e73d3 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -13,6 +13,9 @@ class JobArtifact < Ci::ApplicationRecord include FileStoreMounter include EachBatch include Gitlab::Utils::StrongMemoize + include SafelyChangeColumnDefault + + columns_changing_default :partition_id enum accessibility: { public: 0, private: 1 }, _suffix: true diff --git a/app/models/ci/job_variable.rb b/app/models/ci/job_variable.rb index 573999995bc99ebde8e48371ee844bc2168a7401..21c9842399ee1ede85e9ca618dbe84f3c54478bf 100644 --- a/app/models/ci/job_variable.rb +++ b/app/models/ci/job_variable.rb @@ -5,8 +5,11 @@ class JobVariable < Ci::ApplicationRecord include Ci::Partitionable include Ci::NewHasVariable include Ci::RawVariable + include SafelyChangeColumnDefault include BulkInsertSafe + columns_changing_default :partition_id + belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id, inverse_of: :job_variables partitionable scope: :job diff --git a/app/models/ci/pending_build.rb b/app/models/ci/pending_build.rb index 14050a1e78e45280aeec5fbb3e46fd85d45d0f48..dc9a8b7a1bf7046326e0c7d931a2b7a85d96b20e 100644 --- a/app/models/ci/pending_build.rb +++ b/app/models/ci/pending_build.rb @@ -4,6 +4,9 @@ module Ci class PendingBuild < Ci::ApplicationRecord include EachBatch include Ci::Partitionable + include SafelyChangeColumnDefault + + columns_changing_default :partition_id belongs_to :project belongs_to :build, class_name: 'Ci::Build' diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 69e48cadf975f966f181ed87c7dd7735c5d2e900..bd327cfbe7b61736b7003a7396012bbeca5e3751 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -17,6 +17,9 @@ class Pipeline < Ci::ApplicationRecord include UpdatedAtFilterable include EachBatch include FastDestroyAll::Helpers + include SafelyChangeColumnDefault + + columns_changing_default :partition_id include IgnorableColumns ignore_column :id_convert_to_bigint, remove_with: '16.3', remove_after: '2023-08-22' diff --git a/app/models/ci/pipeline_variable.rb b/app/models/ci/pipeline_variable.rb index 37d94272fb1a3cb9f5b9b10202f8a339a23b4d75..9747f9ef5271d4576717887b8c94cf5733c599c7 100644 --- a/app/models/ci/pipeline_variable.rb +++ b/app/models/ci/pipeline_variable.rb @@ -5,8 +5,10 @@ class PipelineVariable < Ci::ApplicationRecord include Ci::Partitionable include Ci::HasVariable include Ci::RawVariable - include IgnorableColumns + include SafelyChangeColumnDefault + + columns_changing_default :partition_id ignore_column :id_convert_to_bigint, remove_with: '16.3', remove_after: '2023-08-22' ignore_column :pipeline_id_convert_to_bigint, remove_with: '16.5', remove_after: '2023-10-22' diff --git a/app/models/ci/running_build.rb b/app/models/ci/running_build.rb index e6f80658f5d7021fee5363fff651fda581866ec7..cfdc47de531fc124fb12855a9e30587d7014fa76 100644 --- a/app/models/ci/running_build.rb +++ b/app/models/ci/running_build.rb @@ -10,6 +10,9 @@ module Ci # of the running builds there is worth the additional pressure. class RunningBuild < Ci::ApplicationRecord include Ci::Partitionable + include SafelyChangeColumnDefault + + columns_changing_default :partition_id partitionable scope: :build diff --git a/app/models/ci/sources/pipeline.rb b/app/models/ci/sources/pipeline.rb index 719d19f41692d5cee5475c44c745c13f2a1c7dd7..4853c57d41f5e20bebb811ef9b9504e671cee80e 100644 --- a/app/models/ci/sources/pipeline.rb +++ b/app/models/ci/sources/pipeline.rb @@ -5,6 +5,9 @@ module Sources class Pipeline < Ci::ApplicationRecord include Ci::Partitionable include Ci::NamespacedModelName + include SafelyChangeColumnDefault + + columns_changing_default :partition_id self.table_name = "ci_sources_pipelines" diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index af0eee974817bf714706cdb3572b90aaa5cc3d8b..4f9a2e4456272c3039ea63ddfef494083ab1219f 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -7,6 +7,9 @@ class Stage < Ci::ApplicationRecord include Ci::HasStatus include Gitlab::OptimisticLocking include Presentable + include SafelyChangeColumnDefault + + columns_changing_default :partition_id partitionable scope: :pipeline diff --git a/app/models/ci/unit_test_failure.rb b/app/models/ci/unit_test_failure.rb index cfef124916402f4ab929f2d58afd7841efd59855..37893f6cdaeceba3960de2a33942bd877b8bba20 100644 --- a/app/models/ci/unit_test_failure.rb +++ b/app/models/ci/unit_test_failure.rb @@ -3,6 +3,9 @@ module Ci class UnitTestFailure < Ci::ApplicationRecord include Ci::Partitionable + include SafelyChangeColumnDefault + + columns_changing_default :partition_id REPORT_WINDOW = 14.days diff --git a/db/post_migrate/20230711093010_drop_default_partition_id_value_for_ci_tables.rb b/db/post_migrate/20230711093010_drop_default_partition_id_value_for_ci_tables.rb new file mode 100644 index 0000000000000000000000000000000000000000..733cab057f6d1fe4e4b0d4c325dc1408c6831298 --- /dev/null +++ b/db/post_migrate/20230711093010_drop_default_partition_id_value_for_ci_tables.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +class DropDefaultPartitionIdValueForCiTables < Gitlab::Database::Migration[2.1] + disable_ddl_transaction! + + TABLES = { + ci_build_needs: [:partition_id], + ci_build_pending_states: [:partition_id], + ci_build_report_results: [:partition_id], + ci_build_trace_chunks: [:partition_id], + ci_builds_runner_session: [:partition_id], + ci_job_variables: [:partition_id], + ci_pending_builds: [:partition_id], + ci_pipelines: [:partition_id], + ci_running_builds: [:partition_id], + ci_sources_pipelines: [:partition_id, :source_partition_id], + ci_unit_test_failures: [:partition_id] + } + + def up + TABLES.each do |table_name, columns| + with_lock_retries do + columns.each do |column_name| # rubocop:disable Migration/WithLockRetriesDisallowedMethod + change_column_default(table_name, column_name, from: 100, to: nil) + end + end + end + end + + def down + TABLES.each do |table_name, columns| + with_lock_retries do + columns.each do |column_name| # rubocop:disable Migration/WithLockRetriesDisallowedMethod + change_column_default(table_name, column_name, from: nil, to: 100) + end + end + end + end +end diff --git a/db/post_migrate/20230712145323_drop_ci_job_artifacts_partition_id_default.rb b/db/post_migrate/20230712145323_drop_ci_job_artifacts_partition_id_default.rb new file mode 100644 index 0000000000000000000000000000000000000000..af475ba62d0fdc48ac16e113131dd0f865b8308c --- /dev/null +++ b/db/post_migrate/20230712145323_drop_ci_job_artifacts_partition_id_default.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +class DropCiJobArtifactsPartitionIdDefault < Gitlab::Database::Migration[2.1] + include Gitlab::Database::MigrationHelpers::WraparoundAutovacuum + + enable_lock_retries! + + TABLE_NAME = :ci_job_artifacts + COLUMN_NAME = :partition_id + + def up + change_column_default(TABLE_NAME, COLUMN_NAME, from: 100, to: nil) if should_run? + end + + def down + change_column_default(TABLE_NAME, COLUMN_NAME, from: nil, to: 100) if should_run? + end + + private + + def should_run? + can_execute_on?(TABLE_NAME) + end +end diff --git a/db/post_migrate/20230712145821_drop_ci_stages_partition_id_default.rb b/db/post_migrate/20230712145821_drop_ci_stages_partition_id_default.rb new file mode 100644 index 0000000000000000000000000000000000000000..a7c23ac35d2b882f2bb80887fa1fd388386ba68b --- /dev/null +++ b/db/post_migrate/20230712145821_drop_ci_stages_partition_id_default.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +class DropCiStagesPartitionIdDefault < Gitlab::Database::Migration[2.1] + include Gitlab::Database::MigrationHelpers::WraparoundAutovacuum + + enable_lock_retries! + + TABLE_NAME = :ci_stages + COLUMN_NAME = :partition_id + + def up + change_column_default(TABLE_NAME, COLUMN_NAME, from: 100, to: nil) if should_run? + end + + def down + change_column_default(TABLE_NAME, COLUMN_NAME, from: nil, to: 100) if should_run? + end + + private + + def should_run? + can_execute_on?(TABLE_NAME) + end +end diff --git a/db/post_migrate/20230712145926_drop_ci_build_trace_metadata_partition_id_default.rb b/db/post_migrate/20230712145926_drop_ci_build_trace_metadata_partition_id_default.rb new file mode 100644 index 0000000000000000000000000000000000000000..dad647246a57469f092e848aa80feb7a7bff15f5 --- /dev/null +++ b/db/post_migrate/20230712145926_drop_ci_build_trace_metadata_partition_id_default.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +class DropCiBuildTraceMetadataPartitionIdDefault < Gitlab::Database::Migration[2.1] + include Gitlab::Database::MigrationHelpers::WraparoundAutovacuum + + enable_lock_retries! + + TABLE_NAME = :ci_build_trace_metadata + COLUMN_NAME = :partition_id + + def up + change_column_default(TABLE_NAME, COLUMN_NAME, from: 100, to: nil) if should_run? + end + + def down + change_column_default(TABLE_NAME, COLUMN_NAME, from: nil, to: 100) if should_run? + end + + private + + def should_run? + can_execute_on?(TABLE_NAME) + end +end diff --git a/db/post_migrate/20230713100100_drop_ci_pipeline_variable_partition_id_default.rb b/db/post_migrate/20230713100100_drop_ci_pipeline_variable_partition_id_default.rb new file mode 100644 index 0000000000000000000000000000000000000000..5ef3a2caa823badd7443b3b2188c240b31117439 --- /dev/null +++ b/db/post_migrate/20230713100100_drop_ci_pipeline_variable_partition_id_default.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +class DropCiPipelineVariablePartitionIdDefault < Gitlab::Database::Migration[2.1] + include Gitlab::Database::MigrationHelpers::WraparoundAutovacuum + + enable_lock_retries! + + TABLE_NAME = :ci_pipeline_variables + COLUMN_NAME = :partition_id + + def up + change_column_default(TABLE_NAME, COLUMN_NAME, from: 100, to: nil) if should_run? + end + + def down + change_column_default(TABLE_NAME, COLUMN_NAME, from: nil, to: 100) if should_run? + end + + private + + def should_run? + can_execute_on?(TABLE_NAME) + end +end diff --git a/db/schema_migrations/20230711093010 b/db/schema_migrations/20230711093010 new file mode 100644 index 0000000000000000000000000000000000000000..7fcdd69e7dfd875d8209e655c9e1569f95ae05b9 --- /dev/null +++ b/db/schema_migrations/20230711093010 @@ -0,0 +1 @@ +1634c4bbe3fd813098a4865db2d84e83fd9eac967723c4500da4c78e9255af46 \ No newline at end of file diff --git a/db/schema_migrations/20230712145323 b/db/schema_migrations/20230712145323 new file mode 100644 index 0000000000000000000000000000000000000000..00d938eb59934cadd0e092bc86341e801f22e4fc --- /dev/null +++ b/db/schema_migrations/20230712145323 @@ -0,0 +1 @@ +3102bb4fb27322ca9a0b2f60757a0bb90d7fce74a8607226f2e2bcf336563ceb \ No newline at end of file diff --git a/db/schema_migrations/20230712145821 b/db/schema_migrations/20230712145821 new file mode 100644 index 0000000000000000000000000000000000000000..8282462f0de8a7dcdd08b75ea753ac974e1730b7 --- /dev/null +++ b/db/schema_migrations/20230712145821 @@ -0,0 +1 @@ +cf697183d364abeab6c9701f51db570f153d88949b51a8e157a502c0a38f007e \ No newline at end of file diff --git a/db/schema_migrations/20230712145926 b/db/schema_migrations/20230712145926 new file mode 100644 index 0000000000000000000000000000000000000000..b0666ce0cf6f41922acca94a0e20a341e1bef8d4 --- /dev/null +++ b/db/schema_migrations/20230712145926 @@ -0,0 +1 @@ +992137f8611e57d44b4b3a861fce07353f0eccfafa894dd3cede02554bcde424 \ No newline at end of file diff --git a/db/schema_migrations/20230713100100 b/db/schema_migrations/20230713100100 new file mode 100644 index 0000000000000000000000000000000000000000..ee77da01273d8c45ce7fcd212fae0132b351295f --- /dev/null +++ b/db/schema_migrations/20230713100100 @@ -0,0 +1 @@ +9787c717ecf74dd05277bde331b554fb824d6d39df83b5d46bac45ba1962e96a \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 9685a18a6a93015fbafda299dbd81d28da698db0..239970ac95991dba34dc588555cda3cd259614ef 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -12993,7 +12993,7 @@ CREATE TABLE ci_build_needs ( artifacts boolean DEFAULT true NOT NULL, optional boolean DEFAULT false NOT NULL, build_id bigint NOT NULL, - partition_id bigint DEFAULT 100 NOT NULL, + partition_id bigint NOT NULL, id bigint NOT NULL ); @@ -13015,7 +13015,7 @@ CREATE TABLE ci_build_pending_states ( failure_reason smallint, trace_checksum bytea, trace_bytesize bigint, - partition_id bigint DEFAULT 100 NOT NULL + partition_id bigint NOT NULL ); CREATE SEQUENCE ci_build_pending_states_id_seq @@ -13031,7 +13031,7 @@ CREATE TABLE ci_build_report_results ( build_id bigint NOT NULL, project_id bigint NOT NULL, data jsonb DEFAULT '{}'::jsonb NOT NULL, - partition_id bigint DEFAULT 100 NOT NULL + partition_id bigint NOT NULL ); CREATE TABLE ci_build_trace_chunks ( @@ -13042,7 +13042,7 @@ CREATE TABLE ci_build_trace_chunks ( checksum bytea, lock_version integer DEFAULT 0 NOT NULL, build_id bigint NOT NULL, - partition_id bigint DEFAULT 100 NOT NULL + partition_id bigint NOT NULL ); CREATE SEQUENCE ci_build_trace_chunks_id_seq @@ -13062,7 +13062,7 @@ CREATE TABLE ci_build_trace_metadata ( remote_checksum bytea, last_archival_attempt_at timestamp with time zone, archived_at timestamp with time zone, - partition_id bigint DEFAULT 100 NOT NULL + partition_id bigint NOT NULL ); CREATE TABLE p_ci_builds ( @@ -13227,7 +13227,7 @@ CREATE TABLE ci_builds_runner_session ( certificate character varying, "authorization" character varying, build_id bigint NOT NULL, - partition_id bigint DEFAULT 100 NOT NULL + partition_id bigint NOT NULL ); CREATE SEQUENCE ci_builds_runner_session_id_seq @@ -13407,7 +13407,7 @@ CREATE TABLE ci_job_artifacts ( id bigint NOT NULL, job_id bigint NOT NULL, locked smallint DEFAULT 2, - partition_id bigint DEFAULT 100 NOT NULL, + partition_id bigint NOT NULL, accessibility smallint DEFAULT 0 NOT NULL, file_final_path text, CONSTRAINT check_27f0f6dbab CHECK ((file_store IS NOT NULL)), @@ -13450,7 +13450,7 @@ CREATE TABLE ci_job_variables ( variable_type smallint DEFAULT 1 NOT NULL, source smallint DEFAULT 0 NOT NULL, raw boolean DEFAULT false NOT NULL, - partition_id bigint DEFAULT 100 NOT NULL + partition_id bigint NOT NULL ); CREATE SEQUENCE ci_job_variables_id_seq @@ -13543,7 +13543,7 @@ CREATE TABLE ci_pending_builds ( minutes_exceeded boolean DEFAULT false NOT NULL, tag_ids integer[] DEFAULT '{}'::integer[], namespace_traversal_ids integer[] DEFAULT '{}'::integer[], - partition_id bigint DEFAULT 100 NOT NULL + partition_id bigint NOT NULL ); CREATE SEQUENCE ci_pending_builds_id_seq @@ -13684,7 +13684,7 @@ CREATE TABLE ci_pipeline_variables ( encrypted_value_iv character varying, pipeline_id integer NOT NULL, variable_type smallint DEFAULT 1 NOT NULL, - partition_id bigint DEFAULT 100 NOT NULL, + partition_id bigint NOT NULL, raw boolean DEFAULT false NOT NULL, id bigint NOT NULL, pipeline_id_convert_to_bigint bigint DEFAULT 0 NOT NULL @@ -13729,7 +13729,7 @@ CREATE TABLE ci_pipelines ( external_pull_request_id bigint, ci_ref_id bigint, locked smallint DEFAULT 1 NOT NULL, - partition_id bigint DEFAULT 100 NOT NULL, + partition_id bigint NOT NULL, id_convert_to_bigint bigint DEFAULT 0 NOT NULL, CONSTRAINT check_d7e99a025e CHECK ((lock_version IS NOT NULL)) ); @@ -13971,7 +13971,7 @@ CREATE TABLE ci_running_builds ( runner_id bigint NOT NULL, created_at timestamp with time zone DEFAULT now() NOT NULL, runner_type smallint NOT NULL, - partition_id bigint DEFAULT 100 NOT NULL + partition_id bigint NOT NULL ); CREATE SEQUENCE ci_running_builds_id_seq @@ -14037,8 +14037,8 @@ CREATE TABLE ci_sources_pipelines ( source_project_id integer, source_pipeline_id integer, source_job_id bigint, - partition_id bigint DEFAULT 100 NOT NULL, - source_partition_id bigint DEFAULT 100 NOT NULL + partition_id bigint NOT NULL, + source_partition_id bigint NOT NULL ); CREATE SEQUENCE ci_sources_pipelines_id_seq @@ -14075,7 +14075,7 @@ CREATE TABLE ci_stages ( lock_version integer DEFAULT 0, "position" integer, id bigint NOT NULL, - partition_id bigint DEFAULT 100 NOT NULL, + partition_id bigint NOT NULL, CONSTRAINT check_81b431e49b CHECK ((lock_version IS NOT NULL)) ); @@ -14148,7 +14148,7 @@ CREATE TABLE ci_unit_test_failures ( failed_at timestamp with time zone NOT NULL, unit_test_id bigint NOT NULL, build_id bigint NOT NULL, - partition_id bigint DEFAULT 100 NOT NULL + partition_id bigint NOT NULL ); CREATE SEQUENCE ci_unit_test_failures_id_seq