From b9aa55410a47894e4e8ccd9968fab43676c07b0c Mon Sep 17 00:00:00 2001 From: Max Orefice <morefice@gitlab.com> Date: Thu, 13 Jul 2023 12:02:28 +0000 Subject: [PATCH] Drop devault value CI partition tables Changelog: other --- app/models/ci/build_need.rb | 4 +- app/models/ci/build_pending_state.rb | 3 ++ app/models/ci/build_report_result.rb | 3 ++ app/models/ci/build_runner_session.rb | 3 ++ app/models/ci/build_trace_chunk.rb | 3 ++ app/models/ci/build_trace_metadata.rb | 3 ++ app/models/ci/job_artifact.rb | 3 ++ app/models/ci/job_variable.rb | 3 ++ app/models/ci/pending_build.rb | 3 ++ app/models/ci/pipeline.rb | 3 ++ app/models/ci/pipeline_variable.rb | 4 +- app/models/ci/running_build.rb | 3 ++ app/models/ci/sources/pipeline.rb | 3 ++ app/models/ci/stage.rb | 3 ++ app/models/ci/unit_test_failure.rb | 3 ++ ...efault_partition_id_value_for_ci_tables.rb | 39 +++++++++++++++++++ ...p_ci_job_artifacts_partition_id_default.rb | 24 ++++++++++++ ...821_drop_ci_stages_partition_id_default.rb | 24 ++++++++++++ ...ild_trace_metadata_partition_id_default.rb | 24 ++++++++++++ ..._pipeline_variable_partition_id_default.rb | 24 ++++++++++++ db/schema_migrations/20230711093010 | 1 + db/schema_migrations/20230712145323 | 1 + db/schema_migrations/20230712145821 | 1 + db/schema_migrations/20230712145926 | 1 + db/schema_migrations/20230713100100 | 1 + db/structure.sql | 32 +++++++-------- 26 files changed, 201 insertions(+), 18 deletions(-) create mode 100644 db/post_migrate/20230711093010_drop_default_partition_id_value_for_ci_tables.rb create mode 100644 db/post_migrate/20230712145323_drop_ci_job_artifacts_partition_id_default.rb create mode 100644 db/post_migrate/20230712145821_drop_ci_stages_partition_id_default.rb create mode 100644 db/post_migrate/20230712145926_drop_ci_build_trace_metadata_partition_id_default.rb create mode 100644 db/post_migrate/20230713100100_drop_ci_pipeline_variable_partition_id_default.rb create mode 100644 db/schema_migrations/20230711093010 create mode 100644 db/schema_migrations/20230712145323 create mode 100644 db/schema_migrations/20230712145821 create mode 100644 db/schema_migrations/20230712145926 create mode 100644 db/schema_migrations/20230713100100 diff --git a/app/models/ci/build_need.rb b/app/models/ci/build_need.rb index aaec8a540116d..317f2523f69f7 100644 --- a/app/models/ci/build_need.rb +++ b/app/models/ci/build_need.rb @@ -3,9 +3,11 @@ module Ci class BuildNeed < Ci::ApplicationRecord include Ci::Partitionable - include BulkInsertSafe include IgnorableColumns + include SafelyChangeColumnDefault + include BulkInsertSafe + columns_changing_default :partition_id ignore_column :id_convert_to_bigint, remove_with: '16.4', remove_after: '2023-09-22' belongs_to :build, class_name: "Ci::Processable", foreign_key: :build_id, inverse_of: :needs diff --git a/app/models/ci/build_pending_state.rb b/app/models/ci/build_pending_state.rb index 966884ae15854..0b88f745d78fa 100644 --- a/app/models/ci/build_pending_state.rb +++ b/app/models/ci/build_pending_state.rb @@ -2,6 +2,9 @@ class Ci::BuildPendingState < Ci::ApplicationRecord include Ci::Partitionable + include SafelyChangeColumnDefault + + columns_changing_default :partition_id belongs_to :build, class_name: 'Ci::Build', foreign_key: :build_id, inverse_of: :pending_state diff --git a/app/models/ci/build_report_result.rb b/app/models/ci/build_report_result.rb index b2d99fab295e1..90b621b8da1f4 100644 --- a/app/models/ci/build_report_result.rb +++ b/app/models/ci/build_report_result.rb @@ -3,6 +3,9 @@ module Ci class BuildReportResult < Ci::ApplicationRecord include Ci::Partitionable + include SafelyChangeColumnDefault + + columns_changing_default :partition_id self.primary_key = :build_id diff --git a/app/models/ci/build_runner_session.rb b/app/models/ci/build_runner_session.rb index 5773b6132bea5..eaa2e1c428e6b 100644 --- a/app/models/ci/build_runner_session.rb +++ b/app/models/ci/build_runner_session.rb @@ -5,6 +5,9 @@ module Ci # Data will be removed after transitioning from running to any state. class BuildRunnerSession < Ci::ApplicationRecord include Ci::Partitionable + include SafelyChangeColumnDefault + + columns_changing_default :partition_id TERMINAL_SUBPROTOCOL = 'terminal.gitlab.com' DEFAULT_SERVICE_NAME = 'build' diff --git a/app/models/ci/build_trace_chunk.rb b/app/models/ci/build_trace_chunk.rb index 7ab89533d7637..0a0f401c9d590 100644 --- a/app/models/ci/build_trace_chunk.rb +++ b/app/models/ci/build_trace_chunk.rb @@ -8,6 +8,9 @@ class BuildTraceChunk < Ci::ApplicationRecord include ::Checksummable include ::Gitlab::ExclusiveLeaseHelpers include ::Gitlab::OptimisticLocking + include SafelyChangeColumnDefault + + columns_changing_default :partition_id belongs_to :build, class_name: "Ci::Build", foreign_key: :build_id, inverse_of: :trace_chunks diff --git a/app/models/ci/build_trace_metadata.rb b/app/models/ci/build_trace_metadata.rb index 4c76089617f04..c5ad3d1942583 100644 --- a/app/models/ci/build_trace_metadata.rb +++ b/app/models/ci/build_trace_metadata.rb @@ -3,6 +3,9 @@ module Ci class BuildTraceMetadata < Ci::ApplicationRecord include Ci::Partitionable + include SafelyChangeColumnDefault + + columns_changing_default :partition_id MAX_ATTEMPTS = 5 self.table_name = 'ci_build_trace_metadata' diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index 5cd7988837e12..11d70e088e9e2 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -13,6 +13,9 @@ class JobArtifact < Ci::ApplicationRecord include FileStoreMounter include EachBatch include Gitlab::Utils::StrongMemoize + include SafelyChangeColumnDefault + + columns_changing_default :partition_id enum accessibility: { public: 0, private: 1 }, _suffix: true diff --git a/app/models/ci/job_variable.rb b/app/models/ci/job_variable.rb index 573999995bc99..21c9842399ee1 100644 --- a/app/models/ci/job_variable.rb +++ b/app/models/ci/job_variable.rb @@ -5,8 +5,11 @@ class JobVariable < Ci::ApplicationRecord include Ci::Partitionable include Ci::NewHasVariable include Ci::RawVariable + include SafelyChangeColumnDefault include BulkInsertSafe + columns_changing_default :partition_id + belongs_to :job, class_name: "Ci::Build", foreign_key: :job_id, inverse_of: :job_variables partitionable scope: :job diff --git a/app/models/ci/pending_build.rb b/app/models/ci/pending_build.rb index 14050a1e78e45..dc9a8b7a1bf70 100644 --- a/app/models/ci/pending_build.rb +++ b/app/models/ci/pending_build.rb @@ -4,6 +4,9 @@ module Ci class PendingBuild < Ci::ApplicationRecord include EachBatch include Ci::Partitionable + include SafelyChangeColumnDefault + + columns_changing_default :partition_id belongs_to :project belongs_to :build, class_name: 'Ci::Build' diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 69e48cadf975f..bd327cfbe7b61 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -17,6 +17,9 @@ class Pipeline < Ci::ApplicationRecord include UpdatedAtFilterable include EachBatch include FastDestroyAll::Helpers + include SafelyChangeColumnDefault + + columns_changing_default :partition_id include IgnorableColumns ignore_column :id_convert_to_bigint, remove_with: '16.3', remove_after: '2023-08-22' diff --git a/app/models/ci/pipeline_variable.rb b/app/models/ci/pipeline_variable.rb index 37d94272fb1a3..9747f9ef5271d 100644 --- a/app/models/ci/pipeline_variable.rb +++ b/app/models/ci/pipeline_variable.rb @@ -5,8 +5,10 @@ class PipelineVariable < Ci::ApplicationRecord include Ci::Partitionable include Ci::HasVariable include Ci::RawVariable - include IgnorableColumns + include SafelyChangeColumnDefault + + columns_changing_default :partition_id ignore_column :id_convert_to_bigint, remove_with: '16.3', remove_after: '2023-08-22' ignore_column :pipeline_id_convert_to_bigint, remove_with: '16.5', remove_after: '2023-10-22' diff --git a/app/models/ci/running_build.rb b/app/models/ci/running_build.rb index e6f80658f5d70..cfdc47de531fc 100644 --- a/app/models/ci/running_build.rb +++ b/app/models/ci/running_build.rb @@ -10,6 +10,9 @@ module Ci # of the running builds there is worth the additional pressure. class RunningBuild < Ci::ApplicationRecord include Ci::Partitionable + include SafelyChangeColumnDefault + + columns_changing_default :partition_id partitionable scope: :build diff --git a/app/models/ci/sources/pipeline.rb b/app/models/ci/sources/pipeline.rb index 719d19f41692d..4853c57d41f5e 100644 --- a/app/models/ci/sources/pipeline.rb +++ b/app/models/ci/sources/pipeline.rb @@ -5,6 +5,9 @@ module Sources class Pipeline < Ci::ApplicationRecord include Ci::Partitionable include Ci::NamespacedModelName + include SafelyChangeColumnDefault + + columns_changing_default :partition_id self.table_name = "ci_sources_pipelines" diff --git a/app/models/ci/stage.rb b/app/models/ci/stage.rb index af0eee974817b..4f9a2e4456272 100644 --- a/app/models/ci/stage.rb +++ b/app/models/ci/stage.rb @@ -7,6 +7,9 @@ class Stage < Ci::ApplicationRecord include Ci::HasStatus include Gitlab::OptimisticLocking include Presentable + include SafelyChangeColumnDefault + + columns_changing_default :partition_id partitionable scope: :pipeline diff --git a/app/models/ci/unit_test_failure.rb b/app/models/ci/unit_test_failure.rb index cfef124916402..37893f6cdaece 100644 --- a/app/models/ci/unit_test_failure.rb +++ b/app/models/ci/unit_test_failure.rb @@ -3,6 +3,9 @@ module Ci class UnitTestFailure < Ci::ApplicationRecord include Ci::Partitionable + include SafelyChangeColumnDefault + + columns_changing_default :partition_id REPORT_WINDOW = 14.days diff --git a/db/post_migrate/20230711093010_drop_default_partition_id_value_for_ci_tables.rb b/db/post_migrate/20230711093010_drop_default_partition_id_value_for_ci_tables.rb new file mode 100644 index 0000000000000..733cab057f6d1 --- /dev/null +++ b/db/post_migrate/20230711093010_drop_default_partition_id_value_for_ci_tables.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +class DropDefaultPartitionIdValueForCiTables < Gitlab::Database::Migration[2.1] + disable_ddl_transaction! + + TABLES = { + ci_build_needs: [:partition_id], + ci_build_pending_states: [:partition_id], + ci_build_report_results: [:partition_id], + ci_build_trace_chunks: [:partition_id], + ci_builds_runner_session: [:partition_id], + ci_job_variables: [:partition_id], + ci_pending_builds: [:partition_id], + ci_pipelines: [:partition_id], + ci_running_builds: [:partition_id], + ci_sources_pipelines: [:partition_id, :source_partition_id], + ci_unit_test_failures: [:partition_id] + } + + def up + TABLES.each do |table_name, columns| + with_lock_retries do + columns.each do |column_name| # rubocop:disable Migration/WithLockRetriesDisallowedMethod + change_column_default(table_name, column_name, from: 100, to: nil) + end + end + end + end + + def down + TABLES.each do |table_name, columns| + with_lock_retries do + columns.each do |column_name| # rubocop:disable Migration/WithLockRetriesDisallowedMethod + change_column_default(table_name, column_name, from: nil, to: 100) + end + end + end + end +end diff --git a/db/post_migrate/20230712145323_drop_ci_job_artifacts_partition_id_default.rb b/db/post_migrate/20230712145323_drop_ci_job_artifacts_partition_id_default.rb new file mode 100644 index 0000000000000..af475ba62d0fd --- /dev/null +++ b/db/post_migrate/20230712145323_drop_ci_job_artifacts_partition_id_default.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +class DropCiJobArtifactsPartitionIdDefault < Gitlab::Database::Migration[2.1] + include Gitlab::Database::MigrationHelpers::WraparoundAutovacuum + + enable_lock_retries! + + TABLE_NAME = :ci_job_artifacts + COLUMN_NAME = :partition_id + + def up + change_column_default(TABLE_NAME, COLUMN_NAME, from: 100, to: nil) if should_run? + end + + def down + change_column_default(TABLE_NAME, COLUMN_NAME, from: nil, to: 100) if should_run? + end + + private + + def should_run? + can_execute_on?(TABLE_NAME) + end +end diff --git a/db/post_migrate/20230712145821_drop_ci_stages_partition_id_default.rb b/db/post_migrate/20230712145821_drop_ci_stages_partition_id_default.rb new file mode 100644 index 0000000000000..a7c23ac35d2b8 --- /dev/null +++ b/db/post_migrate/20230712145821_drop_ci_stages_partition_id_default.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +class DropCiStagesPartitionIdDefault < Gitlab::Database::Migration[2.1] + include Gitlab::Database::MigrationHelpers::WraparoundAutovacuum + + enable_lock_retries! + + TABLE_NAME = :ci_stages + COLUMN_NAME = :partition_id + + def up + change_column_default(TABLE_NAME, COLUMN_NAME, from: 100, to: nil) if should_run? + end + + def down + change_column_default(TABLE_NAME, COLUMN_NAME, from: nil, to: 100) if should_run? + end + + private + + def should_run? + can_execute_on?(TABLE_NAME) + end +end diff --git a/db/post_migrate/20230712145926_drop_ci_build_trace_metadata_partition_id_default.rb b/db/post_migrate/20230712145926_drop_ci_build_trace_metadata_partition_id_default.rb new file mode 100644 index 0000000000000..dad647246a574 --- /dev/null +++ b/db/post_migrate/20230712145926_drop_ci_build_trace_metadata_partition_id_default.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +class DropCiBuildTraceMetadataPartitionIdDefault < Gitlab::Database::Migration[2.1] + include Gitlab::Database::MigrationHelpers::WraparoundAutovacuum + + enable_lock_retries! + + TABLE_NAME = :ci_build_trace_metadata + COLUMN_NAME = :partition_id + + def up + change_column_default(TABLE_NAME, COLUMN_NAME, from: 100, to: nil) if should_run? + end + + def down + change_column_default(TABLE_NAME, COLUMN_NAME, from: nil, to: 100) if should_run? + end + + private + + def should_run? + can_execute_on?(TABLE_NAME) + end +end diff --git a/db/post_migrate/20230713100100_drop_ci_pipeline_variable_partition_id_default.rb b/db/post_migrate/20230713100100_drop_ci_pipeline_variable_partition_id_default.rb new file mode 100644 index 0000000000000..5ef3a2caa823b --- /dev/null +++ b/db/post_migrate/20230713100100_drop_ci_pipeline_variable_partition_id_default.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +class DropCiPipelineVariablePartitionIdDefault < Gitlab::Database::Migration[2.1] + include Gitlab::Database::MigrationHelpers::WraparoundAutovacuum + + enable_lock_retries! + + TABLE_NAME = :ci_pipeline_variables + COLUMN_NAME = :partition_id + + def up + change_column_default(TABLE_NAME, COLUMN_NAME, from: 100, to: nil) if should_run? + end + + def down + change_column_default(TABLE_NAME, COLUMN_NAME, from: nil, to: 100) if should_run? + end + + private + + def should_run? + can_execute_on?(TABLE_NAME) + end +end diff --git a/db/schema_migrations/20230711093010 b/db/schema_migrations/20230711093010 new file mode 100644 index 0000000000000..7fcdd69e7dfd8 --- /dev/null +++ b/db/schema_migrations/20230711093010 @@ -0,0 +1 @@ +1634c4bbe3fd813098a4865db2d84e83fd9eac967723c4500da4c78e9255af46 \ No newline at end of file diff --git a/db/schema_migrations/20230712145323 b/db/schema_migrations/20230712145323 new file mode 100644 index 0000000000000..00d938eb59934 --- /dev/null +++ b/db/schema_migrations/20230712145323 @@ -0,0 +1 @@ +3102bb4fb27322ca9a0b2f60757a0bb90d7fce74a8607226f2e2bcf336563ceb \ No newline at end of file diff --git a/db/schema_migrations/20230712145821 b/db/schema_migrations/20230712145821 new file mode 100644 index 0000000000000..8282462f0de8a --- /dev/null +++ b/db/schema_migrations/20230712145821 @@ -0,0 +1 @@ +cf697183d364abeab6c9701f51db570f153d88949b51a8e157a502c0a38f007e \ No newline at end of file diff --git a/db/schema_migrations/20230712145926 b/db/schema_migrations/20230712145926 new file mode 100644 index 0000000000000..b0666ce0cf6f4 --- /dev/null +++ b/db/schema_migrations/20230712145926 @@ -0,0 +1 @@ +992137f8611e57d44b4b3a861fce07353f0eccfafa894dd3cede02554bcde424 \ No newline at end of file diff --git a/db/schema_migrations/20230713100100 b/db/schema_migrations/20230713100100 new file mode 100644 index 0000000000000..ee77da01273d8 --- /dev/null +++ b/db/schema_migrations/20230713100100 @@ -0,0 +1 @@ +9787c717ecf74dd05277bde331b554fb824d6d39df83b5d46bac45ba1962e96a \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 9685a18a6a930..239970ac95991 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -12993,7 +12993,7 @@ CREATE TABLE ci_build_needs ( artifacts boolean DEFAULT true NOT NULL, optional boolean DEFAULT false NOT NULL, build_id bigint NOT NULL, - partition_id bigint DEFAULT 100 NOT NULL, + partition_id bigint NOT NULL, id bigint NOT NULL ); @@ -13015,7 +13015,7 @@ CREATE TABLE ci_build_pending_states ( failure_reason smallint, trace_checksum bytea, trace_bytesize bigint, - partition_id bigint DEFAULT 100 NOT NULL + partition_id bigint NOT NULL ); CREATE SEQUENCE ci_build_pending_states_id_seq @@ -13031,7 +13031,7 @@ CREATE TABLE ci_build_report_results ( build_id bigint NOT NULL, project_id bigint NOT NULL, data jsonb DEFAULT '{}'::jsonb NOT NULL, - partition_id bigint DEFAULT 100 NOT NULL + partition_id bigint NOT NULL ); CREATE TABLE ci_build_trace_chunks ( @@ -13042,7 +13042,7 @@ CREATE TABLE ci_build_trace_chunks ( checksum bytea, lock_version integer DEFAULT 0 NOT NULL, build_id bigint NOT NULL, - partition_id bigint DEFAULT 100 NOT NULL + partition_id bigint NOT NULL ); CREATE SEQUENCE ci_build_trace_chunks_id_seq @@ -13062,7 +13062,7 @@ CREATE TABLE ci_build_trace_metadata ( remote_checksum bytea, last_archival_attempt_at timestamp with time zone, archived_at timestamp with time zone, - partition_id bigint DEFAULT 100 NOT NULL + partition_id bigint NOT NULL ); CREATE TABLE p_ci_builds ( @@ -13227,7 +13227,7 @@ CREATE TABLE ci_builds_runner_session ( certificate character varying, "authorization" character varying, build_id bigint NOT NULL, - partition_id bigint DEFAULT 100 NOT NULL + partition_id bigint NOT NULL ); CREATE SEQUENCE ci_builds_runner_session_id_seq @@ -13407,7 +13407,7 @@ CREATE TABLE ci_job_artifacts ( id bigint NOT NULL, job_id bigint NOT NULL, locked smallint DEFAULT 2, - partition_id bigint DEFAULT 100 NOT NULL, + partition_id bigint NOT NULL, accessibility smallint DEFAULT 0 NOT NULL, file_final_path text, CONSTRAINT check_27f0f6dbab CHECK ((file_store IS NOT NULL)), @@ -13450,7 +13450,7 @@ CREATE TABLE ci_job_variables ( variable_type smallint DEFAULT 1 NOT NULL, source smallint DEFAULT 0 NOT NULL, raw boolean DEFAULT false NOT NULL, - partition_id bigint DEFAULT 100 NOT NULL + partition_id bigint NOT NULL ); CREATE SEQUENCE ci_job_variables_id_seq @@ -13543,7 +13543,7 @@ CREATE TABLE ci_pending_builds ( minutes_exceeded boolean DEFAULT false NOT NULL, tag_ids integer[] DEFAULT '{}'::integer[], namespace_traversal_ids integer[] DEFAULT '{}'::integer[], - partition_id bigint DEFAULT 100 NOT NULL + partition_id bigint NOT NULL ); CREATE SEQUENCE ci_pending_builds_id_seq @@ -13684,7 +13684,7 @@ CREATE TABLE ci_pipeline_variables ( encrypted_value_iv character varying, pipeline_id integer NOT NULL, variable_type smallint DEFAULT 1 NOT NULL, - partition_id bigint DEFAULT 100 NOT NULL, + partition_id bigint NOT NULL, raw boolean DEFAULT false NOT NULL, id bigint NOT NULL, pipeline_id_convert_to_bigint bigint DEFAULT 0 NOT NULL @@ -13729,7 +13729,7 @@ CREATE TABLE ci_pipelines ( external_pull_request_id bigint, ci_ref_id bigint, locked smallint DEFAULT 1 NOT NULL, - partition_id bigint DEFAULT 100 NOT NULL, + partition_id bigint NOT NULL, id_convert_to_bigint bigint DEFAULT 0 NOT NULL, CONSTRAINT check_d7e99a025e CHECK ((lock_version IS NOT NULL)) ); @@ -13971,7 +13971,7 @@ CREATE TABLE ci_running_builds ( runner_id bigint NOT NULL, created_at timestamp with time zone DEFAULT now() NOT NULL, runner_type smallint NOT NULL, - partition_id bigint DEFAULT 100 NOT NULL + partition_id bigint NOT NULL ); CREATE SEQUENCE ci_running_builds_id_seq @@ -14037,8 +14037,8 @@ CREATE TABLE ci_sources_pipelines ( source_project_id integer, source_pipeline_id integer, source_job_id bigint, - partition_id bigint DEFAULT 100 NOT NULL, - source_partition_id bigint DEFAULT 100 NOT NULL + partition_id bigint NOT NULL, + source_partition_id bigint NOT NULL ); CREATE SEQUENCE ci_sources_pipelines_id_seq @@ -14075,7 +14075,7 @@ CREATE TABLE ci_stages ( lock_version integer DEFAULT 0, "position" integer, id bigint NOT NULL, - partition_id bigint DEFAULT 100 NOT NULL, + partition_id bigint NOT NULL, CONSTRAINT check_81b431e49b CHECK ((lock_version IS NOT NULL)) ); @@ -14148,7 +14148,7 @@ CREATE TABLE ci_unit_test_failures ( failed_at timestamp with time zone NOT NULL, unit_test_id bigint NOT NULL, build_id bigint NOT NULL, - partition_id bigint DEFAULT 100 NOT NULL + partition_id bigint NOT NULL ); CREATE SEQUENCE ci_unit_test_failures_id_seq -- GitLab