From beec3a0ec49cb8429fefca8dc31a34f9505c2bfe Mon Sep 17 00:00:00 2001
From: Andreas Brandl <abrandl@gitlab.com>
Date: Mon, 15 Jun 2020 12:42:22 +0200
Subject: [PATCH] Create time-space partitions in separate schema
See https://gitlab.com/gitlab-org/gitlab/-/issues/220321
---
.../unreleased/ab-partition-management.yml | 5 +++++
.../active_record_schema_ignore_tables.rb | 3 +++
...101135_create_dynamic_partitions_schema.rb | 19 +++++++++++++++++++
db/structure.sql | 5 +++++
.../table_management_helpers.rb | 11 ++++++-----
lib/gitlab/database/schema_helpers.rb | 6 ++++--
.../table_management_helpers_spec.rb | 2 +-
spec/support/helpers/partitioning_helpers.rb | 11 +++++++----
8 files changed, 50 insertions(+), 12 deletions(-)
create mode 100644 changelogs/unreleased/ab-partition-management.yml
create mode 100644 db/migrate/20200615101135_create_dynamic_partitions_schema.rb
diff --git a/changelogs/unreleased/ab-partition-management.yml b/changelogs/unreleased/ab-partition-management.yml
new file mode 100644
index 0000000000000..35d9ce9ed0ad2
--- /dev/null
+++ b/changelogs/unreleased/ab-partition-management.yml
@@ -0,0 +1,5 @@
+---
+title: Create time-space partitions in separate schema
+merge_request: 34504
+author:
+type: other
diff --git a/config/initializers/active_record_schema_ignore_tables.rb b/config/initializers/active_record_schema_ignore_tables.rb
index 661135f8ade36..559455cbf4672 100644
--- a/config/initializers/active_record_schema_ignore_tables.rb
+++ b/config/initializers/active_record_schema_ignore_tables.rb
@@ -1,2 +1,5 @@
# Ignore table used temporarily in background migration
ActiveRecord::SchemaDumper.ignore_tables = ["untracked_files_for_uploads"]
+
+# Ignore dynamically managed partitions in static application schema
+ActiveRecord::SchemaDumper.ignore_tables += ["partitions_dynamic.*"]
diff --git a/db/migrate/20200615101135_create_dynamic_partitions_schema.rb b/db/migrate/20200615101135_create_dynamic_partitions_schema.rb
new file mode 100644
index 0000000000000..4e6636a216460
--- /dev/null
+++ b/db/migrate/20200615101135_create_dynamic_partitions_schema.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+class CreateDynamicPartitionsSchema < ActiveRecord::Migration[6.0]
+ include Gitlab::Database::SchemaHelpers
+
+ DOWNTIME = false
+
+ def up
+ execute 'CREATE SCHEMA partitions_dynamic'
+
+ create_comment(:schema, :partitions_dynamic, <<~EOS.strip)
+ Schema to hold partitions managed dynamically from the application, e.g. for time space partitioning.
+ EOS
+ end
+
+ def down
+ execute 'DROP SCHEMA partitions_dynamic'
+ end
+end
diff --git a/db/structure.sql b/db/structure.sql
index 7cd30c15c332a..0f2ff89c70366 100644
--- a/db/structure.sql
+++ b/db/structure.sql
@@ -1,5 +1,9 @@
SET search_path=public;
+CREATE SCHEMA partitions_dynamic;
+
+COMMENT ON SCHEMA partitions_dynamic IS 'Schema to hold partitions managed dynamically from the application, e.g. for time space partitioning.';
+
CREATE EXTENSION IF NOT EXISTS pg_trgm WITH SCHEMA public;
CREATE TABLE public.abuse_reports (
@@ -13995,6 +13999,7 @@ COPY "schema_migrations" (version) FROM STDIN;
20200609142508
20200609212701
20200615083635
+20200615101135
20200615121217
20200615123055
20200615232735
diff --git a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb
index f77fbe98df167..d57ad03bd0507 100644
--- a/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb
+++ b/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb
@@ -8,6 +8,7 @@ module TableManagementHelpers
WHITELISTED_TABLES = %w[audit_events].freeze
ERROR_SCOPE = 'table partitioning'
+ DYNAMIC_PARTITIONS_SCHEMA = 'partitions_dynamic'
# Creates a partitioned copy of an existing table, using a RANGE partitioning strategy on a timestamp column.
# One partition is created per month between the given `min_date` and `max_date`.
@@ -125,7 +126,7 @@ def create_daterange_partitions(table_name, column_name, min_date, max_date)
min_date = min_date.beginning_of_month.to_date
max_date = max_date.next_month.beginning_of_month.to_date
- create_range_partition_safely("#{table_name}_000000", table_name, 'MINVALUE', to_sql_date_literal(min_date))
+ create_range_partition_safely("#{table_name}_000000", table_name, 'MINVALUE', to_sql_date_literal(min_date), schema: DYNAMIC_PARTITIONS_SCHEMA)
while min_date < max_date
partition_name = "#{table_name}_#{min_date.strftime('%Y%m')}"
@@ -133,7 +134,7 @@ def create_daterange_partitions(table_name, column_name, min_date, max_date)
lower_bound = to_sql_date_literal(min_date)
upper_bound = to_sql_date_literal(next_date)
- create_range_partition_safely(partition_name, table_name, lower_bound, upper_bound)
+ create_range_partition_safely(partition_name, table_name, lower_bound, upper_bound, schema: DYNAMIC_PARTITIONS_SCHEMA)
min_date = next_date
end
end
@@ -142,8 +143,8 @@ def to_sql_date_literal(date)
connection.quote(date.strftime('%Y-%m-%d'))
end
- def create_range_partition_safely(partition_name, table_name, lower_bound, upper_bound)
- if table_exists?(partition_name)
+ def create_range_partition_safely(partition_name, table_name, lower_bound, upper_bound, schema:)
+ if table_exists?("#{schema}.#{partition_name}")
# rubocop:disable Gitlab/RailsLogger
Rails.logger.warn "Partition not created because it already exists" \
" (this may be due to an aborted migration or similar): partition_name: #{partition_name}"
@@ -151,7 +152,7 @@ def create_range_partition_safely(partition_name, table_name, lower_bound, upper
return
end
- create_range_partition(partition_name, table_name, lower_bound, upper_bound)
+ create_range_partition(partition_name, table_name, lower_bound, upper_bound, schema: schema)
end
def create_sync_trigger(source_table, target_table, unique_key)
diff --git a/lib/gitlab/database/schema_helpers.rb b/lib/gitlab/database/schema_helpers.rb
index 8e544307d8193..fee52024a97aa 100644
--- a/lib/gitlab/database/schema_helpers.rb
+++ b/lib/gitlab/database/schema_helpers.rb
@@ -69,9 +69,11 @@ def assert_not_in_transaction_block(scope:)
private
- def create_range_partition(partition_name, table_name, lower_bound, upper_bound)
+ def create_range_partition(partition_name, table_name, lower_bound, upper_bound, schema:)
+ raise ArgumentError, 'explicit schema is required but currently missing' unless schema
+
execute(<<~SQL)
- CREATE TABLE #{partition_name} PARTITION OF #{table_name}
+ CREATE TABLE #{schema}.#{partition_name} PARTITION OF #{table_name}
FOR VALUES FROM (#{lower_bound}) TO (#{upper_bound})
SQL
end
diff --git a/spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb b/spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb
index 586b57d200219..60fd039da332d 100644
--- a/spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb
+++ b/spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb
@@ -241,7 +241,7 @@
describe '#drop_partitioned_table_for' do
let(:expected_tables) do
- %w[000000 201912 202001 202002].map { |suffix| "#{partitioned_table}_#{suffix}" }.unshift(partitioned_table)
+ %w[000000 201912 202001 202002].map { |suffix| "partitions_dynamic.#{partitioned_table}_#{suffix}" }.unshift(partitioned_table)
end
context 'when the table is not whitelisted' do
diff --git a/spec/support/helpers/partitioning_helpers.rb b/spec/support/helpers/partitioning_helpers.rb
index 98a13915d7698..7aedc6953aa21 100644
--- a/spec/support/helpers/partitioning_helpers.rb
+++ b/spec/support/helpers/partitioning_helpers.rb
@@ -8,8 +8,8 @@ def expect_table_partitioned_by(table, columns, part_type: :range)
expect(columns_with_part_type).to match_array(actual_columns)
end
- def expect_range_partition_of(partition_name, table_name, min_value, max_value)
- definition = find_partition_definition(partition_name)
+ def expect_range_partition_of(partition_name, table_name, min_value, max_value, schema: 'partitions_dynamic')
+ definition = find_partition_definition(partition_name, schema: schema)
expect(definition).not_to be_nil
expect(definition['base_table']).to eq(table_name.to_s)
@@ -40,7 +40,7 @@ def find_partitioned_columns(table)
SQL
end
- def find_partition_definition(partition)
+ def find_partition_definition(partition, schema: 'partitions_dynamic')
connection.select_one(<<~SQL)
select
parent_class.relname as base_table,
@@ -48,7 +48,10 @@ def find_partition_definition(partition)
from pg_class
inner join pg_inherits i on pg_class.oid = inhrelid
inner join pg_class parent_class on parent_class.oid = inhparent
- where pg_class.relname = '#{partition}' and pg_class.relispartition;
+ inner join pg_namespace ON pg_namespace.oid = pg_class.relnamespace
+ where pg_namespace.nspname = '#{schema}'
+ and pg_class.relname = '#{partition}'
+ and pg_class.relispartition
SQL
end
end
--
GitLab