From fd0f7a053d2c4953a57bc48ad54290313ba2e60e Mon Sep 17 00:00:00 2001
From: Tianwen Chen <tchen@gitlab.com>
Date: Thu, 11 Jul 2024 10:02:21 +0000
Subject: [PATCH] Create table p_ci_build_tags
Changelog: other
---
app/models/ci/build_tag.rb | 18 +++++++
.../concerns/ci/partitionable/testing.rb | 1 +
config/gitlab_loose_foreign_keys.yml | 4 ++
config/initializers/postgres_partitioning.rb | 1 +
db/docs/p_ci_build_tags.yml | 12 +++++
...0703043908_create_table_p_ci_build_tags.rb | 31 ++++++++++++
...01_ensure_unique_id_for_p_ci_build_tags.rb | 20 ++++++++
...3_create_partitions_for_p_ci_build_tags.rb | 34 +++++++++++++
..._fk_from_p_ci_build_tags_to_p_ci_builds.rb | 39 +++++++++++++++
...831_add_fk_from_p_ci_build_tags_to_tags.rb | 37 ++++++++++++++
db/schema_migrations/20240703043908 | 1 +
db/schema_migrations/20240703054001 | 1 +
db/schema_migrations/20240703082453 | 1 +
db/schema_migrations/20240703094839 | 1 +
db/schema_migrations/20240703104831 | 1 +
db/structure.sql | 48 +++++++++++++++++++
spec/factories/ci/build_tag.rb | 9 ++++
spec/factories/tag.rb | 7 +++
spec/models/ci/build_tag_spec.rb | 31 ++++++++++++
spec/models/concerns/ci/partitionable_spec.rb | 1 +
20 files changed, 298 insertions(+)
create mode 100644 app/models/ci/build_tag.rb
create mode 100644 db/docs/p_ci_build_tags.yml
create mode 100644 db/migrate/20240703043908_create_table_p_ci_build_tags.rb
create mode 100644 db/migrate/20240703054001_ensure_unique_id_for_p_ci_build_tags.rb
create mode 100644 db/migrate/20240703082453_create_partitions_for_p_ci_build_tags.rb
create mode 100644 db/post_migrate/20240703094839_add_fk_from_p_ci_build_tags_to_p_ci_builds.rb
create mode 100644 db/post_migrate/20240703104831_add_fk_from_p_ci_build_tags_to_tags.rb
create mode 100644 db/schema_migrations/20240703043908
create mode 100644 db/schema_migrations/20240703054001
create mode 100644 db/schema_migrations/20240703082453
create mode 100644 db/schema_migrations/20240703094839
create mode 100644 db/schema_migrations/20240703104831
create mode 100644 spec/factories/ci/build_tag.rb
create mode 100644 spec/factories/tag.rb
create mode 100644 spec/models/ci/build_tag_spec.rb
diff --git a/app/models/ci/build_tag.rb b/app/models/ci/build_tag.rb
new file mode 100644
index 0000000000000..30436f2da73af
--- /dev/null
+++ b/app/models/ci/build_tag.rb
@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Ci
+ class BuildTag < Ci::ApplicationRecord
+ include Ci::Partitionable
+
+ self.table_name = :p_ci_build_tags
+
+ query_constraints :build_id, :partition_id
+ partitionable scope: :build, partitioned: true
+
+ belongs_to :build, ->(build_tag) { in_partition(build_tag) }, # rubocop:disable Rails/InverseOf -- Will be added once association on build is added
+ class_name: 'Ci::Build', partition_foreign_key: :partition_id, optional: false
+ belongs_to :tag, class_name: 'ActsAsTaggableOn::Tag', optional: false
+
+ validates :project_id, presence: true
+ end
+end
diff --git a/app/models/concerns/ci/partitionable/testing.rb b/app/models/concerns/ci/partitionable/testing.rb
index d397244cd9016..db34be42541fb 100644
--- a/app/models/concerns/ci/partitionable/testing.rb
+++ b/app/models/concerns/ci/partitionable/testing.rb
@@ -14,6 +14,7 @@ module Testing
Ci::BuildReportResult
Ci::BuildRunnerSession
Ci::BuildSource
+ Ci::BuildTag
Ci::BuildTraceChunk
Ci::BuildTraceMetadata
Ci::BuildPendingState
diff --git a/config/gitlab_loose_foreign_keys.yml b/config/gitlab_loose_foreign_keys.yml
index 79ebc52ddf674..8f51d7869c551 100644
--- a/config/gitlab_loose_foreign_keys.yml
+++ b/config/gitlab_loose_foreign_keys.yml
@@ -293,6 +293,10 @@ p_ci_build_sources:
- table: projects
column: project_id
on_delete: async_delete
+p_ci_build_tags:
+ - table: projects
+ column: project_id
+ on_delete: async_delete
p_ci_builds:
- table: users
column: user_id
diff --git a/config/initializers/postgres_partitioning.rb b/config/initializers/postgres_partitioning.rb
index 1fca5d738a1b9..f7c3560d3949c 100644
--- a/config/initializers/postgres_partitioning.rb
+++ b/config/initializers/postgres_partitioning.rb
@@ -17,6 +17,7 @@
Ci::BuildMetadata,
Ci::BuildExecutionConfig,
Ci::BuildName,
+ Ci::BuildTag,
Ci::BuildSource,
Ci::Catalog::Resources::Components::Usage,
Ci::Catalog::Resources::SyncEvent,
diff --git a/db/docs/p_ci_build_tags.yml b/db/docs/p_ci_build_tags.yml
new file mode 100644
index 0000000000000..bfd2146ba0acf
--- /dev/null
+++ b/db/docs/p_ci_build_tags.yml
@@ -0,0 +1,12 @@
+---
+table_name: p_ci_build_tags
+classes:
+- Ci::BuildTag
+feature_categories:
+- continuous_integration
+description: Routing table that holds information for build tags
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/158327
+milestone: '17.2'
+gitlab_schema: gitlab_ci
+sharding_key:
+ project_id: projects
diff --git a/db/migrate/20240703043908_create_table_p_ci_build_tags.rb b/db/migrate/20240703043908_create_table_p_ci_build_tags.rb
new file mode 100644
index 0000000000000..528dcb27d9546
--- /dev/null
+++ b/db/migrate/20240703043908_create_table_p_ci_build_tags.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+class CreateTablePCiBuildTags < Gitlab::Database::Migration[2.2]
+ milestone '17.3'
+
+ enable_lock_retries!
+
+ OPTIONS = {
+ if_not_exists: true,
+ options: 'PARTITION BY LIST (partition_id)',
+ primary_key: [:id, :partition_id]
+ }
+
+ def up
+ create_table(:p_ci_build_tags, **OPTIONS) do |t| # rubocop:disable Migration/EnsureFactoryForTable -- https://gitlab.com/gitlab-org/gitlab/-/issues/468630
+ t.bigserial :id, null: false
+ t.bigint :tag_id, null: false
+ t.bigint :build_id, null: false
+ t.bigint :partition_id, null: false
+ t.bigint :project_id, null: false
+
+ t.index [:tag_id, :build_id, :partition_id], unique: true
+ t.index [:build_id, :partition_id]
+ t.index [:project_id]
+ end
+ end
+
+ def down
+ drop_table :p_ci_build_tags
+ end
+end
diff --git a/db/migrate/20240703054001_ensure_unique_id_for_p_ci_build_tags.rb b/db/migrate/20240703054001_ensure_unique_id_for_p_ci_build_tags.rb
new file mode 100644
index 0000000000000..7b6c8735074c7
--- /dev/null
+++ b/db/migrate/20240703054001_ensure_unique_id_for_p_ci_build_tags.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+class EnsureUniqueIdForPCiBuildTags < Gitlab::Database::Migration[2.2]
+ include Gitlab::Database::PartitioningMigrationHelpers::UniquenessHelpers
+
+ milestone '17.3'
+
+ enable_lock_retries!
+
+ TABLE_NAME = :p_ci_build_tags
+ SEQ_NAME = :p_ci_build_tags_id_seq
+
+ def up
+ ensure_unique_id(TABLE_NAME, seq: SEQ_NAME)
+ end
+
+ def down
+ revert_ensure_unique_id(TABLE_NAME, seq: SEQ_NAME)
+ end
+end
diff --git a/db/migrate/20240703082453_create_partitions_for_p_ci_build_tags.rb b/db/migrate/20240703082453_create_partitions_for_p_ci_build_tags.rb
new file mode 100644
index 0000000000000..862bc88cd64c6
--- /dev/null
+++ b/db/migrate/20240703082453_create_partitions_for_p_ci_build_tags.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+class CreatePartitionsForPCiBuildTags < Gitlab::Database::Migration[2.2]
+ milestone '17.3'
+
+ enable_lock_retries!
+
+ def up
+ connection.execute(<<~SQL)
+ LOCK TABLE p_ci_builds IN SHARE ROW EXCLUSIVE MODE;
+ LOCK TABLE ONLY p_ci_build_tags IN ACCESS EXCLUSIVE MODE;
+
+ CREATE TABLE IF NOT EXISTS gitlab_partitions_dynamic.ci_build_tags_100
+ PARTITION OF p_ci_build_tags
+ FOR VALUES IN (100);
+
+ CREATE TABLE IF NOT EXISTS gitlab_partitions_dynamic.ci_build_tags_101
+ PARTITION OF p_ci_build_tags
+ FOR VALUES IN (101);
+
+ CREATE TABLE IF NOT EXISTS gitlab_partitions_dynamic.ci_build_tags_102
+ PARTITION OF p_ci_build_tags
+ FOR VALUES IN (102);
+ SQL
+ end
+
+ def down
+ connection.execute(<<~SQL)
+ DROP TABLE IF EXISTS gitlab_partitions_dynamic.ci_build_tags_100;
+ DROP TABLE IF EXISTS gitlab_partitions_dynamic.ci_build_tags_101;
+ DROP TABLE IF EXISTS gitlab_partitions_dynamic.ci_build_tags_102;
+ SQL
+ end
+end
diff --git a/db/post_migrate/20240703094839_add_fk_from_p_ci_build_tags_to_p_ci_builds.rb b/db/post_migrate/20240703094839_add_fk_from_p_ci_build_tags_to_p_ci_builds.rb
new file mode 100644
index 0000000000000..a1608ddc40a4e
--- /dev/null
+++ b/db/post_migrate/20240703094839_add_fk_from_p_ci_build_tags_to_p_ci_builds.rb
@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+class AddFkFromPCiBuildTagsToPCiBuilds < Gitlab::Database::Migration[2.2]
+ include Gitlab::Database::PartitioningMigrationHelpers
+
+ milestone '17.3'
+
+ disable_ddl_transaction!
+
+ SOURCE_TABLE_NAME = :p_ci_build_tags
+ TARGET_TABLE_NAME = :p_ci_builds
+ FK_NAME = :fk_rails_d7bd025909
+ COLUMN = [:partition_id, :build_id]
+ TARGET_COLUMN = [:partition_id, :id]
+
+ def up
+ add_concurrent_partitioned_foreign_key(
+ SOURCE_TABLE_NAME, TARGET_TABLE_NAME,
+ column: COLUMN,
+ target_column: TARGET_COLUMN,
+ validate: true,
+ on_update: :cascade,
+ on_delete: :cascade,
+ reverse_lock_order: true,
+ name: FK_NAME
+ )
+ end
+
+ def down
+ with_lock_retries do
+ remove_foreign_key_if_exists(
+ SOURCE_TABLE_NAME,
+ TARGET_TABLE_NAME,
+ name: FK_NAME,
+ reverse_lock_order: true
+ )
+ end
+ end
+end
diff --git a/db/post_migrate/20240703104831_add_fk_from_p_ci_build_tags_to_tags.rb b/db/post_migrate/20240703104831_add_fk_from_p_ci_build_tags_to_tags.rb
new file mode 100644
index 0000000000000..3fefb702add5a
--- /dev/null
+++ b/db/post_migrate/20240703104831_add_fk_from_p_ci_build_tags_to_tags.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+class AddFkFromPCiBuildTagsToTags < Gitlab::Database::Migration[2.2]
+ include Gitlab::Database::PartitioningMigrationHelpers
+
+ milestone '17.3'
+
+ disable_ddl_transaction!
+
+ SOURCE_TABLE_NAME = :p_ci_build_tags
+ TARGET_TABLE_NAME = :tags
+ FK_NAME = :fk_rails_8284d35c66
+ COLUMN = [:tag_id]
+ TARGET_COLUMN = [:id]
+
+ def up
+ add_concurrent_partitioned_foreign_key(
+ SOURCE_TABLE_NAME, TARGET_TABLE_NAME,
+ column: COLUMN,
+ target_column: TARGET_COLUMN,
+ validate: true,
+ on_delete: :cascade,
+ reverse_lock_order: true,
+ name: FK_NAME
+ )
+ end
+
+ def down
+ with_lock_retries do
+ remove_foreign_key_if_exists(
+ SOURCE_TABLE_NAME, TARGET_TABLE_NAME,
+ name: FK_NAME,
+ reverse_lock_order: true
+ )
+ end
+ end
+end
diff --git a/db/schema_migrations/20240703043908 b/db/schema_migrations/20240703043908
new file mode 100644
index 0000000000000..c5930f70f0543
--- /dev/null
+++ b/db/schema_migrations/20240703043908
@@ -0,0 +1 @@
+69b8dd14f4840ae66d09270323512c8d123fb1fdf5d9b9b120377c103d4c7af4
\ No newline at end of file
diff --git a/db/schema_migrations/20240703054001 b/db/schema_migrations/20240703054001
new file mode 100644
index 0000000000000..c4577af110a89
--- /dev/null
+++ b/db/schema_migrations/20240703054001
@@ -0,0 +1 @@
+ff5f6d4be9bfba44bd9b0040a3a99188f96cb6e12b8df79d1aa2ff60942427a8
\ No newline at end of file
diff --git a/db/schema_migrations/20240703082453 b/db/schema_migrations/20240703082453
new file mode 100644
index 0000000000000..87f750b1b10d2
--- /dev/null
+++ b/db/schema_migrations/20240703082453
@@ -0,0 +1 @@
+48a77d2758bfe22bb6dc6b607cf7d19c45e46a05858f1d671cbd578b4bae2f10
\ No newline at end of file
diff --git a/db/schema_migrations/20240703094839 b/db/schema_migrations/20240703094839
new file mode 100644
index 0000000000000..497388816919b
--- /dev/null
+++ b/db/schema_migrations/20240703094839
@@ -0,0 +1 @@
+4823f3b810910b92631acb55ce019c13184b199aa0cd0bf35380e3f39403d6d0
\ No newline at end of file
diff --git a/db/schema_migrations/20240703104831 b/db/schema_migrations/20240703104831
new file mode 100644
index 0000000000000..6ceb8d39a53da
--- /dev/null
+++ b/db/schema_migrations/20240703104831
@@ -0,0 +1 @@
+05182f84a8444fb8a9c683a33ae70b91516e6ae17e2587cf41ebf179b18ed772
\ No newline at end of file
diff --git a/db/structure.sql b/db/structure.sql
index 671e9130e13c5..91ebc3a786550 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -10,6 +10,19 @@ CREATE EXTENSION IF NOT EXISTS btree_gist;
CREATE EXTENSION IF NOT EXISTS pg_trgm;
+CREATE FUNCTION assign_p_ci_build_tags_id_value() RETURNS trigger
+ LANGUAGE plpgsql
+ AS $$
+BEGIN
+IF NEW."id" IS NOT NULL THEN
+ RAISE WARNING 'Manually assigning ids is not allowed, the value will be ignored';
+END IF;
+NEW."id" := nextval('p_ci_build_tags_id_seq'::regclass);
+RETURN NEW;
+
+END
+$$;
+
CREATE FUNCTION assign_p_ci_builds_execution_configs_id_value() RETURNS trigger
LANGUAGE plpgsql
AS $$
@@ -2130,6 +2143,15 @@ CREATE TABLE p_ci_build_sources (
)
PARTITION BY LIST (partition_id);
+CREATE TABLE p_ci_build_tags (
+ id bigint NOT NULL,
+ tag_id bigint NOT NULL,
+ build_id bigint NOT NULL,
+ partition_id bigint NOT NULL,
+ project_id bigint NOT NULL
+)
+PARTITION BY LIST (partition_id);
+
CREATE TABLE p_ci_builds (
status character varying,
finished_at timestamp without time zone,
@@ -13907,6 +13929,15 @@ CREATE SEQUENCE p_catalog_resource_sync_events_id_seq
ALTER SEQUENCE p_catalog_resource_sync_events_id_seq OWNED BY p_catalog_resource_sync_events.id;
+CREATE SEQUENCE p_ci_build_tags_id_seq
+ START WITH 1
+ INCREMENT BY 1
+ NO MINVALUE
+ NO MAXVALUE
+ CACHE 1;
+
+ALTER SEQUENCE p_ci_build_tags_id_seq OWNED BY p_ci_build_tags.id;
+
CREATE SEQUENCE p_ci_builds_execution_configs_id_seq
START WITH 1
INCREMENT BY 1
@@ -23520,6 +23551,9 @@ ALTER TABLE ONLY p_ci_build_names
ALTER TABLE ONLY p_ci_build_sources
ADD CONSTRAINT p_ci_build_sources_pkey PRIMARY KEY (build_id, partition_id);
+ALTER TABLE ONLY p_ci_build_tags
+ ADD CONSTRAINT p_ci_build_tags_pkey PRIMARY KEY (id, partition_id);
+
ALTER TABLE ONLY p_ci_builds_execution_configs
ADD CONSTRAINT p_ci_builds_execution_configs_pkey PRIMARY KEY (id, partition_id);
@@ -28267,6 +28301,12 @@ CREATE INDEX index_p_ci_build_names_on_search_vector ON ONLY p_ci_build_names US
CREATE INDEX index_p_ci_build_sources_on_project_id_and_build_id ON ONLY p_ci_build_sources USING btree (project_id, build_id);
+CREATE INDEX index_p_ci_build_tags_on_build_id_and_partition_id ON ONLY p_ci_build_tags USING btree (build_id, partition_id);
+
+CREATE INDEX index_p_ci_build_tags_on_project_id ON ONLY p_ci_build_tags USING btree (project_id);
+
+CREATE UNIQUE INDEX index_p_ci_build_tags_on_tag_id_and_build_id_and_partition_id ON ONLY p_ci_build_tags USING btree (tag_id, build_id, partition_id);
+
CREATE INDEX index_p_ci_builds_execution_configs_on_pipeline_id ON ONLY p_ci_builds_execution_configs USING btree (pipeline_id);
CREATE INDEX index_p_ci_builds_execution_configs_on_project_id ON ONLY p_ci_builds_execution_configs USING btree (project_id);
@@ -31593,6 +31633,8 @@ ALTER INDEX p_ci_job_artifacts_expire_at_job_id_idx1 ATTACH PARTITION tmp_index_
ALTER INDEX p_ci_builds_token_encrypted_partition_id_idx ATTACH PARTITION unique_ci_builds_token_encrypted_and_partition_id;
+CREATE TRIGGER assign_p_ci_build_tags_id_trigger BEFORE INSERT ON p_ci_build_tags FOR EACH ROW EXECUTE FUNCTION assign_p_ci_build_tags_id_value();
+
CREATE TRIGGER assign_p_ci_builds_execution_configs_id_trigger BEFORE INSERT ON p_ci_builds_execution_configs FOR EACH ROW EXECUTE FUNCTION assign_p_ci_builds_execution_configs_id_value();
CREATE TRIGGER assign_p_ci_builds_id_trigger BEFORE INSERT ON p_ci_builds FOR EACH ROW EXECUTE FUNCTION assign_p_ci_builds_id_value();
@@ -34293,6 +34335,9 @@ ALTER TABLE ONLY audit_events_instance_streaming_event_type_filters
ALTER TABLE ONLY required_code_owners_sections
ADD CONSTRAINT fk_rails_817708cf2d FOREIGN KEY (protected_branch_id) REFERENCES protected_branches(id) ON DELETE CASCADE;
+ALTER TABLE p_ci_build_tags
+ ADD CONSTRAINT fk_rails_8284d35c66 FOREIGN KEY (tag_id) REFERENCES tags(id) ON DELETE CASCADE;
+
ALTER TABLE ONLY namespace_ldap_settings
ADD CONSTRAINT fk_rails_82cd0ad4bb FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE;
@@ -34938,6 +34983,9 @@ ALTER TABLE ONLY packages_rpm_repository_files
ALTER TABLE ONLY packages_rpm_metadata
ADD CONSTRAINT fk_rails_d79f02264b FOREIGN KEY (package_id) REFERENCES packages_packages(id) ON DELETE CASCADE;
+ALTER TABLE p_ci_build_tags
+ ADD CONSTRAINT fk_rails_d7bd025909 FOREIGN KEY (partition_id, build_id) REFERENCES p_ci_builds(partition_id, id) ON UPDATE CASCADE ON DELETE CASCADE;
+
ALTER TABLE ONLY note_metadata
ADD CONSTRAINT fk_rails_d853224d37 FOREIGN KEY (note_id) REFERENCES notes(id) ON DELETE CASCADE;
diff --git a/spec/factories/ci/build_tag.rb b/spec/factories/ci/build_tag.rb
new file mode 100644
index 0000000000000..08bd426f30169
--- /dev/null
+++ b/spec/factories/ci/build_tag.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+ factory :ci_build_tag, class: 'Ci::BuildTag' do
+ build factory: :ci_build
+ tag factory: :tag
+ project_id { build.project_id }
+ end
+end
diff --git a/spec/factories/tag.rb b/spec/factories/tag.rb
new file mode 100644
index 0000000000000..f51c2b061b6de
--- /dev/null
+++ b/spec/factories/tag.rb
@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+ factory :tag, class: 'ActsAsTaggableOn::Tag' do
+ name { generate(:name) }
+ end
+end
diff --git a/spec/models/ci/build_tag_spec.rb b/spec/models/ci/build_tag_spec.rb
new file mode 100644
index 0000000000000..49d4de50d8a31
--- /dev/null
+++ b/spec/models/ci/build_tag_spec.rb
@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Ci::BuildTag, feature_category: :continuous_integration do
+ it { is_expected.to belong_to(:build).optional(false) }
+ it { is_expected.to belong_to(:tag).optional(false) }
+
+ describe 'validations' do
+ it { is_expected.to validate_presence_of(:project_id) }
+ end
+
+ describe 'partitioning' do
+ context 'with build' do
+ let_it_be(:build) { FactoryBot.build(:ci_build, partition_id: ci_testing_partition_id) }
+ let_it_be(:build_tag) { FactoryBot.build(:ci_build_tag, build: build) }
+
+ it 'sets partition_id to the current partition value' do
+ expect { build_tag.valid? }.to change { build_tag.partition_id }.to(ci_testing_partition_id)
+ end
+
+ context 'when it is already set' do
+ let_it_be(:build_tag) { FactoryBot.build(:ci_build_tag, partition_id: 125) }
+
+ it 'does not change the partition_id value' do
+ expect { build_tag.valid? }.not_to change { build_tag.partition_id }
+ end
+ end
+ end
+ end
+end
diff --git a/spec/models/concerns/ci/partitionable_spec.rb b/spec/models/concerns/ci/partitionable_spec.rb
index 70fe5677c2d12..11a0bd5be4100 100644
--- a/spec/models/concerns/ci/partitionable_spec.rb
+++ b/spec/models/concerns/ci/partitionable_spec.rb
@@ -194,6 +194,7 @@
Ci::BuildMetadata
Ci::BuildExecutionConfig
Ci::BuildName
+ Ci::BuildTag
Ci::BuildSource
Ci::JobAnnotation
Ci::JobArtifact
--
GitLab