From 9e1df0b61efb7f5b9c92d1e39ae045bf86fe7fbf Mon Sep 17 00:00:00 2001
From: Furkan Ayhan <furkanayhn@gmail.com>
Date: Wed, 1 Dec 2021 16:34:14 +0000
Subject: [PATCH] Create ci_namespace_mirrors and ci_project_mirrors tables
---
app/models/ci/namespace_mirror.rb | 9 ++++
app/models/ci/project_mirror.rb | 9 ++++
...11011140930_create_ci_namespace_mirrors.rb | 15 ++++++
...0211011140931_create_ci_project_mirrors.rb | 12 +++++
db/schema_migrations/20211011140930 | 1 +
db/schema_migrations/20211011140931 | 1 +
db/structure.sql | 48 +++++++++++++++++++
.../database/gitlab_loose_foreign_keys.yml | 11 +++++
lib/gitlab/database/gitlab_schemas.yml | 2 +
spec/models/namespace_spec.rb | 4 ++
spec/models/project_spec.rb | 6 +++
.../have_loose_foreign_key.rb | 10 ++--
12 files changed, 124 insertions(+), 4 deletions(-)
create mode 100644 app/models/ci/namespace_mirror.rb
create mode 100644 app/models/ci/project_mirror.rb
create mode 100644 db/migrate/20211011140930_create_ci_namespace_mirrors.rb
create mode 100644 db/migrate/20211011140931_create_ci_project_mirrors.rb
create mode 100644 db/schema_migrations/20211011140930
create mode 100644 db/schema_migrations/20211011140931
diff --git a/app/models/ci/namespace_mirror.rb b/app/models/ci/namespace_mirror.rb
new file mode 100644
index 0000000000000..a497d2cabe557
--- /dev/null
+++ b/app/models/ci/namespace_mirror.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Ci
+ # This model represents a record in a shadow table of the main database's namespaces table.
+ # It allows us to navigate the namespace hierarchy on the ci database without resorting to a JOIN.
+ class NamespaceMirror < ApplicationRecord
+ # Will be filled by https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75517
+ end
+end
diff --git a/app/models/ci/project_mirror.rb b/app/models/ci/project_mirror.rb
new file mode 100644
index 0000000000000..c6e3101fb3ab7
--- /dev/null
+++ b/app/models/ci/project_mirror.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Ci
+ # This model represents a shadow table of the main database's projects table.
+ # It allows us to navigate the project and namespace hierarchy on the ci database.
+ class ProjectMirror < ApplicationRecord
+ # Will be filled by https://gitlab.com/gitlab-org/gitlab/-/merge_requests/75517
+ end
+end
diff --git a/db/migrate/20211011140930_create_ci_namespace_mirrors.rb b/db/migrate/20211011140930_create_ci_namespace_mirrors.rb
new file mode 100644
index 0000000000000..b9a708c5d7ba3
--- /dev/null
+++ b/db/migrate/20211011140930_create_ci_namespace_mirrors.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class CreateCiNamespaceMirrors < Gitlab::Database::Migration[1.0]
+ TABLE_NAME = :ci_namespace_mirrors
+ INDEX_NAME = "index_gin_#{TABLE_NAME}_on_traversal_ids"
+
+ def change
+ create_table TABLE_NAME do |t|
+ t.integer :namespace_id, null: false, index: { unique: true }
+ t.integer :traversal_ids, array: true, default: [], null: false
+
+ t.index :traversal_ids, name: INDEX_NAME, using: :gin
+ end
+ end
+end
diff --git a/db/migrate/20211011140931_create_ci_project_mirrors.rb b/db/migrate/20211011140931_create_ci_project_mirrors.rb
new file mode 100644
index 0000000000000..2407b7e0b84e9
--- /dev/null
+++ b/db/migrate/20211011140931_create_ci_project_mirrors.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+class CreateCiProjectMirrors < Gitlab::Database::Migration[1.0]
+ TABLE_NAME = :ci_project_mirrors
+
+ def change
+ create_table TABLE_NAME do |t|
+ t.integer :project_id, null: false, index: { unique: true }
+ t.integer :namespace_id, null: false, index: true
+ end
+ end
+end
diff --git a/db/schema_migrations/20211011140930 b/db/schema_migrations/20211011140930
new file mode 100644
index 0000000000000..6347ee5d51d84
--- /dev/null
+++ b/db/schema_migrations/20211011140930
@@ -0,0 +1 @@
+cdae819e8de3b5ad721014376bfd9af97a45e953e2d345daf62784f986a5eb31
\ No newline at end of file
diff --git a/db/schema_migrations/20211011140931 b/db/schema_migrations/20211011140931
new file mode 100644
index 0000000000000..c959d97074e02
--- /dev/null
+++ b/db/schema_migrations/20211011140931
@@ -0,0 +1 @@
+7e51eb4443fd74da9bef4d9c1c3cc40376c311abbc05ca7871f725fada79b48a
\ No newline at end of file
diff --git a/db/structure.sql b/db/structure.sql
index a932ea30de515..3cb591e3b52ba 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -11767,6 +11767,21 @@ CREATE SEQUENCE ci_minutes_additional_packs_id_seq
ALTER SEQUENCE ci_minutes_additional_packs_id_seq OWNED BY ci_minutes_additional_packs.id;
+CREATE TABLE ci_namespace_mirrors (
+ id bigint NOT NULL,
+ namespace_id integer NOT NULL,
+ traversal_ids integer[] DEFAULT '{}'::integer[] NOT NULL
+);
+
+CREATE SEQUENCE ci_namespace_mirrors_id_seq
+ START WITH 1
+ INCREMENT BY 1
+ NO MINVALUE
+ NO MAXVALUE
+ CACHE 1;
+
+ALTER SEQUENCE ci_namespace_mirrors_id_seq OWNED BY ci_namespace_mirrors.id;
+
CREATE TABLE ci_namespace_monthly_usages (
id bigint NOT NULL,
namespace_id bigint NOT NULL,
@@ -12015,6 +12030,21 @@ CREATE SEQUENCE ci_platform_metrics_id_seq
ALTER SEQUENCE ci_platform_metrics_id_seq OWNED BY ci_platform_metrics.id;
+CREATE TABLE ci_project_mirrors (
+ id bigint NOT NULL,
+ project_id integer NOT NULL,
+ namespace_id integer NOT NULL
+);
+
+CREATE SEQUENCE ci_project_mirrors_id_seq
+ START WITH 1
+ INCREMENT BY 1
+ NO MINVALUE
+ NO MAXVALUE
+ CACHE 1;
+
+ALTER SEQUENCE ci_project_mirrors_id_seq OWNED BY ci_project_mirrors.id;
+
CREATE TABLE ci_project_monthly_usages (
id bigint NOT NULL,
project_id bigint NOT NULL,
@@ -21264,6 +21294,8 @@ ALTER TABLE ONLY ci_job_variables ALTER COLUMN id SET DEFAULT nextval('ci_job_va
ALTER TABLE ONLY ci_minutes_additional_packs ALTER COLUMN id SET DEFAULT nextval('ci_minutes_additional_packs_id_seq'::regclass);
+ALTER TABLE ONLY ci_namespace_mirrors ALTER COLUMN id SET DEFAULT nextval('ci_namespace_mirrors_id_seq'::regclass);
+
ALTER TABLE ONLY ci_namespace_monthly_usages ALTER COLUMN id SET DEFAULT nextval('ci_namespace_monthly_usages_id_seq'::regclass);
ALTER TABLE ONLY ci_pending_builds ALTER COLUMN id SET DEFAULT nextval('ci_pending_builds_id_seq'::regclass);
@@ -21286,6 +21318,8 @@ ALTER TABLE ONLY ci_pipelines_config ALTER COLUMN pipeline_id SET DEFAULT nextva
ALTER TABLE ONLY ci_platform_metrics ALTER COLUMN id SET DEFAULT nextval('ci_platform_metrics_id_seq'::regclass);
+ALTER TABLE ONLY ci_project_mirrors ALTER COLUMN id SET DEFAULT nextval('ci_project_mirrors_id_seq'::regclass);
+
ALTER TABLE ONLY ci_project_monthly_usages ALTER COLUMN id SET DEFAULT nextval('ci_project_monthly_usages_id_seq'::regclass);
ALTER TABLE ONLY ci_refs ALTER COLUMN id SET DEFAULT nextval('ci_refs_id_seq'::regclass);
@@ -22727,6 +22761,9 @@ ALTER TABLE ONLY ci_job_variables
ALTER TABLE ONLY ci_minutes_additional_packs
ADD CONSTRAINT ci_minutes_additional_packs_pkey PRIMARY KEY (id);
+ALTER TABLE ONLY ci_namespace_mirrors
+ ADD CONSTRAINT ci_namespace_mirrors_pkey PRIMARY KEY (id);
+
ALTER TABLE ONLY ci_namespace_monthly_usages
ADD CONSTRAINT ci_namespace_monthly_usages_pkey PRIMARY KEY (id);
@@ -22760,6 +22797,9 @@ ALTER TABLE ONLY ci_pipelines
ALTER TABLE ONLY ci_platform_metrics
ADD CONSTRAINT ci_platform_metrics_pkey PRIMARY KEY (id);
+ALTER TABLE ONLY ci_project_mirrors
+ ADD CONSTRAINT ci_project_mirrors_pkey PRIMARY KEY (id);
+
ALTER TABLE ONLY ci_project_monthly_usages
ADD CONSTRAINT ci_project_monthly_usages_pkey PRIMARY KEY (id);
@@ -25337,6 +25377,8 @@ CREATE UNIQUE INDEX index_ci_job_variables_on_key_and_job_id ON ci_job_variables
CREATE INDEX index_ci_minutes_additional_packs_on_namespace_id_purchase_xid ON ci_minutes_additional_packs USING btree (namespace_id, purchase_xid);
+CREATE UNIQUE INDEX index_ci_namespace_mirrors_on_namespace_id ON ci_namespace_mirrors USING btree (namespace_id);
+
CREATE UNIQUE INDEX index_ci_namespace_monthly_usages_on_namespace_id_and_date ON ci_namespace_monthly_usages USING btree (namespace_id, date);
CREATE INDEX index_ci_pending_builds_id_on_protected_partial ON ci_pending_builds USING btree (id) WHERE (protected = true);
@@ -25425,6 +25467,10 @@ CREATE INDEX index_ci_pipelines_on_user_id_and_created_at_and_source ON ci_pipel
CREATE INDEX index_ci_pipelines_on_user_id_and_id_and_cancelable_status ON ci_pipelines USING btree (user_id, id) WHERE ((status)::text = ANY (ARRAY[('running'::character varying)::text, ('waiting_for_resource'::character varying)::text, ('preparing'::character varying)::text, ('pending'::character varying)::text, ('created'::character varying)::text, ('scheduled'::character varying)::text]));
+CREATE INDEX index_ci_project_mirrors_on_namespace_id ON ci_project_mirrors USING btree (namespace_id);
+
+CREATE UNIQUE INDEX index_ci_project_mirrors_on_project_id ON ci_project_mirrors USING btree (project_id);
+
CREATE UNIQUE INDEX index_ci_project_monthly_usages_on_project_id_and_date ON ci_project_monthly_usages USING btree (project_id, date);
CREATE UNIQUE INDEX index_ci_refs_on_project_id_and_ref_path ON ci_refs USING btree (project_id, ref_path);
@@ -26001,6 +26047,8 @@ CREATE INDEX index_geo_repository_updated_events_on_source ON geo_repository_upd
CREATE INDEX index_geo_reset_checksum_events_on_project_id ON geo_reset_checksum_events USING btree (project_id);
+CREATE INDEX index_gin_ci_namespace_mirrors_on_traversal_ids ON ci_namespace_mirrors USING gin (traversal_ids);
+
CREATE INDEX index_gin_ci_pending_builds_on_namespace_traversal_ids ON ci_pending_builds USING gin (namespace_traversal_ids);
CREATE INDEX index_gitlab_subscription_histories_on_gitlab_subscription_id ON gitlab_subscription_histories USING btree (gitlab_subscription_id);
diff --git a/lib/gitlab/database/gitlab_loose_foreign_keys.yml b/lib/gitlab/database/gitlab_loose_foreign_keys.yml
index 430ac7854499c..907a10d0fa3c0 100644
--- a/lib/gitlab/database/gitlab_loose_foreign_keys.yml
+++ b/lib/gitlab/database/gitlab_loose_foreign_keys.yml
@@ -18,3 +18,14 @@ clusters_applications_runners:
- table: ci_runners
column: runner_id
on_delete: async_nullify
+ci_namespace_mirrors:
+ - table: namespaces
+ column: namespace_id
+ on_delete: async_delete
+ci_project_mirrors:
+ - table: projects
+ column: project_id
+ on_delete: async_delete
+ - table: namespaces
+ column: namespace_id
+ on_delete: async_delete
diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml
index 5695f2f1c14ec..6733318bfafdc 100644
--- a/lib/gitlab/database/gitlab_schemas.yml
+++ b/lib/gitlab/database/gitlab_schemas.yml
@@ -86,6 +86,7 @@ ci_job_token_project_scope_links: :gitlab_ci
ci_job_variables: :gitlab_ci
ci_minutes_additional_packs: :gitlab_ci
ci_namespace_monthly_usages: :gitlab_ci
+ci_namespace_mirrors: :gitlab_ci
ci_pending_builds: :gitlab_ci
ci_pipeline_artifacts: :gitlab_ci
ci_pipeline_chat_data: :gitlab_ci
@@ -97,6 +98,7 @@ ci_pipelines: :gitlab_ci
ci_pipeline_variables: :gitlab_ci
ci_platform_metrics: :gitlab_ci
ci_project_monthly_usages: :gitlab_ci
+ci_project_mirrors: :gitlab_ci
ci_refs: :gitlab_ci
ci_resource_groups: :gitlab_ci
ci_resources: :gitlab_ci
diff --git a/spec/models/namespace_spec.rb b/spec/models/namespace_spec.rb
index 0baf75f890f7f..84b72c54f01fe 100644
--- a/spec/models/namespace_spec.rb
+++ b/spec/models/namespace_spec.rb
@@ -2056,4 +2056,8 @@ def project_rugged(project)
it { is_expected.to be(true) }
end
end
+
+ it_behaves_like 'it has loose foreign keys' do
+ let(:factory_name) { :group }
+ end
end
diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb
index a612a30369d2d..20fec9c21a296 100644
--- a/spec/models/project_spec.rb
+++ b/spec/models/project_spec.rb
@@ -7453,6 +7453,12 @@ def has_external_wiki
end
end
+ it_behaves_like 'it has loose foreign keys' do
+ let(:factory_name) { :project }
+ end
+
+ private
+
def finish_job(export_job)
export_job.start
export_job.finish
diff --git a/spec/support/shared_examples/loose_foreign_keys/have_loose_foreign_key.rb b/spec/support/shared_examples/loose_foreign_keys/have_loose_foreign_key.rb
index 1f2b2937dc5f0..8c98254e134c8 100644
--- a/spec/support/shared_examples/loose_foreign_keys/have_loose_foreign_key.rb
+++ b/spec/support/shared_examples/loose_foreign_keys/have_loose_foreign_key.rb
@@ -4,6 +4,8 @@
let(:factory_name) { nil }
let(:table_name) { described_class.table_name }
let(:connection) { described_class.connection }
+ let(:fully_qualified_table_name) { "#{connection.current_schema}.#{table_name}" }
+ let(:deleted_records) { LooseForeignKeys::DeletedRecord.where(fully_qualified_table_name: fully_qualified_table_name) }
it 'has at least one loose foreign key definition' do
definitions = Gitlab::Database::LooseForeignKeys.definitions_by_table[table_name]
@@ -29,7 +31,7 @@
# using delete to avoid cross-database modification errors when associations with dependent option are present
model.delete
- deleted_record = LooseForeignKeys::DeletedRecord.find_by(fully_qualified_table_name: "#{connection.current_schema}.#{table_name}", primary_key_value: model.id)
+ deleted_record = deleted_records.find_by(primary_key_value: model.id)
expect(deleted_record).not_to be_nil
end
@@ -37,11 +39,11 @@
it 'cleans up record deletions' do
model = create(factory_name) # rubocop: disable Rails/SaveBang
- expect { model.delete }.to change { LooseForeignKeys::DeletedRecord.count }.by(1)
+ expect { model.delete }.to change { deleted_records.count }.by(1)
LooseForeignKeys::ProcessDeletedRecordsService.new(connection: connection).execute
- expect(LooseForeignKeys::DeletedRecord.status_pending.count).to be(0)
- expect(LooseForeignKeys::DeletedRecord.status_processed.count).to be(1)
+ expect(deleted_records.status_pending.count).to be(0)
+ expect(deleted_records.status_processed.count).to be(1)
end
end
--
GitLab