From 613bb9c2bac7f48720a3e12e9581dc78cac5272f Mon Sep 17 00:00:00 2001
From: Pedro Pombeiro <noreply@pedro.pombei.ro>
Date: Fri, 10 Nov 2023 11:01:37 +0000
Subject: [PATCH] Add supporting classes for ClickHouse migrations
Inspired by the ActiveRecord migration classes
---
.../main/20230705124511_create_events.sql | 16 --
...230707151359_create_ci_finished_builds.sql | 33 ---
..._aggregated_queueing_delay_percentiles.sql | 11 -
...2_create_contribution_analytics_events.sql | 13 -
...ion_analytics_events_materialized_view.sql | 16 --
.../20230808070520_create_events_cursor.sql | 9 -
...gregated_queueing_delay_percentiles_mv.sql | 12 -
.../migrate/20230705124511_create_events.rb | 30 +++
...0230707151359_create_ci_finished_builds.rb | 47 ++++
...s_aggregated_queueing_delay_percentiles.rb | 25 ++
...32_create_contribution_analytics_events.rb | 27 ++
...tion_analytics_events_materialized_view.rb | 30 +++
.../20230808070520_create_sync_cursors.rb | 23 ++
...ggregated_queueing_delay_percentiles_mv.rb | 26 ++
...2300_modify_ci_finished_builds_settings.rb | 15 ++
lib/click_house/migration.rb | 89 +++++++
.../migration_support/migration_context.rb | 94 +++++++
.../migration_support/migration_error.rb | 54 ++++
lib/click_house/migration_support/migrator.rb | 160 ++++++++++++
.../migration_support/schema_migration.rb | 71 ++++++
lib/tasks/gitlab/click_house/migration.rake | 64 +++++
rubocop/rubocop-code_reuse.yml | 1 +
.../migration_context_spec.rb | 233 ++++++++++++++++++
.../drop_table/1_create_some_table.rb | 14 ++
.../drop_table/2_drop_some_table.rb | 11 +
.../duplicate_name/1_create_some_table.rb | 14 ++
.../duplicate_name/2_create_some_table.rb | 14 ++
.../duplicate_version/1_create_some_table.rb | 14 ++
.../duplicate_version/1_drop_some_table.rb | 11 +
.../1_migration_with_error.rb | 9 +
.../1_create_some_table_on_main_db.rb | 15 ++
.../2_create_some_table_on_another_db.rb | 16 ++
.../3_change_some_table_on_main_db.rb | 11 +
.../1_create_some_table.rb | 14 ++
.../1_create_some_table.rb | 16 ++
.../1_create_some_table.rb | 20 ++
spec/support/database/click_house/hooks.rb | 48 ++--
spec/support/helpers/click_house_helpers.rb | 74 ++++++
.../gitlab/click_house/migration_rake_spec.rb | 118 +++++++++
spec/tooling/quality/test_level_spec.rb | 4 +-
tooling/quality/test_level.rb | 1 +
41 files changed, 1396 insertions(+), 127 deletions(-)
delete mode 100644 db/click_house/main/20230705124511_create_events.sql
delete mode 100644 db/click_house/main/20230707151359_create_ci_finished_builds.sql
delete mode 100644 db/click_house/main/20230719101806_create_ci_finished_builds_aggregated_queueing_delay_percentiles.sql
delete mode 100644 db/click_house/main/20230724064832_create_contribution_analytics_events.sql
delete mode 100644 db/click_house/main/20230724064918_contribution_analytics_events_materialized_view.sql
delete mode 100644 db/click_house/main/20230808070520_create_events_cursor.sql
delete mode 100644 db/click_house/main/20230808140217_create_ci_finished_builds_aggregated_queueing_delay_percentiles_mv.sql
create mode 100644 db/click_house/migrate/20230705124511_create_events.rb
create mode 100644 db/click_house/migrate/20230707151359_create_ci_finished_builds.rb
create mode 100644 db/click_house/migrate/20230719101806_create_ci_finished_builds_aggregated_queueing_delay_percentiles.rb
create mode 100644 db/click_house/migrate/20230724064832_create_contribution_analytics_events.rb
create mode 100644 db/click_house/migrate/20230724064918_create_contribution_analytics_events_materialized_view.rb
create mode 100644 db/click_house/migrate/20230808070520_create_sync_cursors.rb
create mode 100644 db/click_house/migrate/20230808140217_create_ci_finished_builds_aggregated_queueing_delay_percentiles_mv.rb
create mode 100644 db/click_house/migrate/20231106202300_modify_ci_finished_builds_settings.rb
create mode 100644 lib/click_house/migration.rb
create mode 100644 lib/click_house/migration_support/migration_context.rb
create mode 100644 lib/click_house/migration_support/migration_error.rb
create mode 100644 lib/click_house/migration_support/migrator.rb
create mode 100644 lib/click_house/migration_support/schema_migration.rb
create mode 100644 lib/tasks/gitlab/click_house/migration.rake
create mode 100644 spec/click_house/migration_support/migration_context_spec.rb
create mode 100644 spec/fixtures/click_house/migrations/drop_table/1_create_some_table.rb
create mode 100644 spec/fixtures/click_house/migrations/drop_table/2_drop_some_table.rb
create mode 100644 spec/fixtures/click_house/migrations/duplicate_name/1_create_some_table.rb
create mode 100644 spec/fixtures/click_house/migrations/duplicate_name/2_create_some_table.rb
create mode 100644 spec/fixtures/click_house/migrations/duplicate_version/1_create_some_table.rb
create mode 100644 spec/fixtures/click_house/migrations/duplicate_version/1_drop_some_table.rb
create mode 100644 spec/fixtures/click_house/migrations/migration_with_error/1_migration_with_error.rb
create mode 100644 spec/fixtures/click_house/migrations/migrations_over_multiple_databases/1_create_some_table_on_main_db.rb
create mode 100644 spec/fixtures/click_house/migrations/migrations_over_multiple_databases/2_create_some_table_on_another_db.rb
create mode 100644 spec/fixtures/click_house/migrations/migrations_over_multiple_databases/3_change_some_table_on_main_db.rb
create mode 100644 spec/fixtures/click_house/migrations/plain_table_creation/1_create_some_table.rb
create mode 100644 spec/fixtures/click_house/migrations/plain_table_creation_on_invalid_database/1_create_some_table.rb
create mode 100644 spec/fixtures/click_house/migrations/table_creation_with_down_method/1_create_some_table.rb
create mode 100644 spec/support/helpers/click_house_helpers.rb
create mode 100644 spec/tasks/gitlab/click_house/migration_rake_spec.rb
diff --git a/db/click_house/main/20230705124511_create_events.sql b/db/click_house/main/20230705124511_create_events.sql
deleted file mode 100644
index 8af45443e4ce4..0000000000000
--- a/db/click_house/main/20230705124511_create_events.sql
+++ /dev/null
@@ -1,16 +0,0 @@
-CREATE TABLE events
-(
- id UInt64 DEFAULT 0,
- path String DEFAULT '',
- author_id UInt64 DEFAULT 0,
- target_id UInt64 DEFAULT 0,
- target_type LowCardinality(String) DEFAULT '',
- action UInt8 DEFAULT 0,
- deleted UInt8 DEFAULT 0,
- created_at DateTime64(6, 'UTC') DEFAULT now(),
- updated_at DateTime64(6, 'UTC') DEFAULT now()
-)
-ENGINE = ReplacingMergeTree(updated_at, deleted)
-PRIMARY KEY (id)
-ORDER BY (id)
-PARTITION BY toYear(created_at)
diff --git a/db/click_house/main/20230707151359_create_ci_finished_builds.sql b/db/click_house/main/20230707151359_create_ci_finished_builds.sql
deleted file mode 100644
index 9fd17e1968fd1..0000000000000
--- a/db/click_house/main/20230707151359_create_ci_finished_builds.sql
+++ /dev/null
@@ -1,33 +0,0 @@
--- source table for CI analytics, almost useless on it's own, but it's a basis for creating materialized views
-CREATE TABLE ci_finished_builds
-(
- id UInt64 DEFAULT 0,
- project_id UInt64 DEFAULT 0,
- pipeline_id UInt64 DEFAULT 0,
- status LowCardinality(String) DEFAULT '',
-
- --- Fields to calculate timings
- created_at DateTime64(6, 'UTC') DEFAULT now(),
- queued_at DateTime64(6, 'UTC') DEFAULT now(),
- finished_at DateTime64(6, 'UTC') DEFAULT now(),
- started_at DateTime64(6, 'UTC') DEFAULT now(),
-
- runner_id UInt64 DEFAULT 0,
- runner_manager_system_xid String DEFAULT '',
-
- --- Runner fields
- runner_run_untagged Boolean DEFAULT FALSE,
- runner_type UInt8 DEFAULT 0,
- runner_manager_version LowCardinality(String) DEFAULT '',
- runner_manager_revision LowCardinality(String) DEFAULT '',
- runner_manager_platform LowCardinality(String) DEFAULT '',
- runner_manager_architecture LowCardinality(String) DEFAULT '',
-
- --- Materialized columns
- duration Int64 MATERIALIZED age('ms', started_at, finished_at),
- queueing_duration Int64 MATERIALIZED age('ms', queued_at, started_at)
- --- This table is incomplete, we'll add more fields before starting the data migration
-)
-ENGINE = ReplacingMergeTree -- Using ReplacingMergeTree just in case we accidentally insert the same data twice
-ORDER BY (status, runner_type, project_id, finished_at, id)
-PARTITION BY toYear(finished_at);
diff --git a/db/click_house/main/20230719101806_create_ci_finished_builds_aggregated_queueing_delay_percentiles.sql b/db/click_house/main/20230719101806_create_ci_finished_builds_aggregated_queueing_delay_percentiles.sql
deleted file mode 100644
index 0b05c3a37f63a..0000000000000
--- a/db/click_house/main/20230719101806_create_ci_finished_builds_aggregated_queueing_delay_percentiles.sql
+++ /dev/null
@@ -1,11 +0,0 @@
-CREATE TABLE ci_finished_builds_aggregated_queueing_delay_percentiles
-(
- status LowCardinality(String) DEFAULT '',
- runner_type UInt8 DEFAULT 0,
- started_at_bucket DateTime64(6, 'UTC') DEFAULT now(),
-
- count_builds AggregateFunction(count),
- queueing_duration_quantile AggregateFunction(quantile, Int64)
-)
-ENGINE = AggregatingMergeTree()
-ORDER BY (started_at_bucket, status, runner_type);
diff --git a/db/click_house/main/20230724064832_create_contribution_analytics_events.sql b/db/click_house/main/20230724064832_create_contribution_analytics_events.sql
deleted file mode 100644
index 7867897e897d3..0000000000000
--- a/db/click_house/main/20230724064832_create_contribution_analytics_events.sql
+++ /dev/null
@@ -1,13 +0,0 @@
-CREATE TABLE contribution_analytics_events
-(
- id UInt64 DEFAULT 0,
- path String DEFAULT '',
- author_id UInt64 DEFAULT 0,
- target_type LowCardinality(String) DEFAULT '',
- action UInt8 DEFAULT 0,
- created_at Date DEFAULT toDate(now()),
- updated_at DateTime64(6, 'UTC') DEFAULT now()
-)
- ENGINE = MergeTree
- ORDER BY (path, created_at, author_id, id)
- PARTITION BY toYear(created_at);
diff --git a/db/click_house/main/20230724064918_contribution_analytics_events_materialized_view.sql b/db/click_house/main/20230724064918_contribution_analytics_events_materialized_view.sql
deleted file mode 100644
index 669b03ce0f3b3..0000000000000
--- a/db/click_house/main/20230724064918_contribution_analytics_events_materialized_view.sql
+++ /dev/null
@@ -1,16 +0,0 @@
-CREATE MATERIALIZED VIEW contribution_analytics_events_mv
-TO contribution_analytics_events
-AS
-SELECT
- id,
- argMax(path, events.updated_at) as path,
- argMax(author_id, events.updated_at) as author_id,
- argMax(target_type, events.updated_at) as target_type,
- argMax(action, events.updated_at) as action,
- argMax(date(created_at), events.updated_at) as created_at,
- max(events.updated_at) as updated_at
-FROM events
-where (("events"."action" = 5 AND "events"."target_type" = '')
- OR ("events"."action" IN (1, 3, 7, 12)
- AND "events"."target_type" IN ('MergeRequest', 'Issue')))
-GROUP BY id
diff --git a/db/click_house/main/20230808070520_create_events_cursor.sql b/db/click_house/main/20230808070520_create_events_cursor.sql
deleted file mode 100644
index effc3c64f6057..0000000000000
--- a/db/click_house/main/20230808070520_create_events_cursor.sql
+++ /dev/null
@@ -1,9 +0,0 @@
-CREATE TABLE sync_cursors
-(
- table_name LowCardinality(String) DEFAULT '',
- primary_key_value UInt64 DEFAULT 0,
- recorded_at DateTime64(6, 'UTC') DEFAULT now()
-)
-ENGINE = ReplacingMergeTree(recorded_at)
-ORDER BY (table_name)
-PRIMARY KEY (table_name)
diff --git a/db/click_house/main/20230808140217_create_ci_finished_builds_aggregated_queueing_delay_percentiles_mv.sql b/db/click_house/main/20230808140217_create_ci_finished_builds_aggregated_queueing_delay_percentiles_mv.sql
deleted file mode 100644
index 504e2d876091a..0000000000000
--- a/db/click_house/main/20230808140217_create_ci_finished_builds_aggregated_queueing_delay_percentiles_mv.sql
+++ /dev/null
@@ -1,12 +0,0 @@
-CREATE MATERIALIZED VIEW ci_finished_builds_aggregated_queueing_delay_percentiles_mv
-TO ci_finished_builds_aggregated_queueing_delay_percentiles
-AS
-SELECT
- status,
- runner_type,
- toStartOfInterval(started_at, INTERVAL 5 minute) AS started_at_bucket,
-
- countState(*) as count_builds,
- quantileState(queueing_duration) AS queueing_duration_quantile
-FROM ci_finished_builds
-GROUP BY status, runner_type, started_at_bucket
diff --git a/db/click_house/migrate/20230705124511_create_events.rb b/db/click_house/migrate/20230705124511_create_events.rb
new file mode 100644
index 0000000000000..cd60ade5d4d9f
--- /dev/null
+++ b/db/click_house/migrate/20230705124511_create_events.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+class CreateEvents < ClickHouse::Migration
+ def up
+ execute <<~SQL
+ CREATE TABLE IF NOT EXISTS events
+ (
+ id UInt64 DEFAULT 0,
+ path String DEFAULT '',
+ author_id UInt64 DEFAULT 0,
+ target_id UInt64 DEFAULT 0,
+ target_type LowCardinality(String) DEFAULT '',
+ action UInt8 DEFAULT 0,
+ deleted UInt8 DEFAULT 0,
+ created_at DateTime64(6, 'UTC') DEFAULT now(),
+ updated_at DateTime64(6, 'UTC') DEFAULT now()
+ )
+ ENGINE = ReplacingMergeTree(updated_at, deleted)
+ PRIMARY KEY (id)
+ ORDER BY (id)
+ PARTITION BY toYear(created_at)
+ SQL
+ end
+
+ def down
+ execute <<~SQL
+ DROP TABLE events
+ SQL
+ end
+end
diff --git a/db/click_house/migrate/20230707151359_create_ci_finished_builds.rb b/db/click_house/migrate/20230707151359_create_ci_finished_builds.rb
new file mode 100644
index 0000000000000..39521af8d9958
--- /dev/null
+++ b/db/click_house/migrate/20230707151359_create_ci_finished_builds.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+class CreateCiFinishedBuilds < ClickHouse::Migration
+ def up
+ execute <<~SQL
+ -- source table for CI analytics, almost useless on it's own, but it's a basis for creating materialized views
+ CREATE TABLE IF NOT EXISTS ci_finished_builds
+ (
+ id UInt64 DEFAULT 0,
+ project_id UInt64 DEFAULT 0,
+ pipeline_id UInt64 DEFAULT 0,
+ status LowCardinality(String) DEFAULT '',
+
+ --- Fields to calculate timings
+ created_at DateTime64(6, 'UTC') DEFAULT now(),
+ queued_at DateTime64(6, 'UTC') DEFAULT now(),
+ finished_at DateTime64(6, 'UTC') DEFAULT now(),
+ started_at DateTime64(6, 'UTC') DEFAULT now(),
+
+ runner_id UInt64 DEFAULT 0,
+ runner_manager_system_xid String DEFAULT '',
+
+ --- Runner fields
+ runner_run_untagged Boolean DEFAULT FALSE,
+ runner_type UInt8 DEFAULT 0,
+ runner_manager_version LowCardinality(String) DEFAULT '',
+ runner_manager_revision LowCardinality(String) DEFAULT '',
+ runner_manager_platform LowCardinality(String) DEFAULT '',
+ runner_manager_architecture LowCardinality(String) DEFAULT '',
+
+ --- Materialized columns
+ duration Int64 MATERIALIZED age('ms', started_at, finished_at),
+ queueing_duration Int64 MATERIALIZED age('ms', queued_at, started_at)
+ --- This table is incomplete, we'll add more fields before starting the data migration
+ )
+ ENGINE = ReplacingMergeTree -- Using ReplacingMergeTree just in case we accidentally insert the same data twice
+ ORDER BY (status, runner_type, project_id, finished_at, id)
+ PARTITION BY toYear(finished_at)
+ SQL
+ end
+
+ def down
+ execute <<~SQL
+ DROP TABLE ci_finished_builds
+ SQL
+ end
+end
diff --git a/db/click_house/migrate/20230719101806_create_ci_finished_builds_aggregated_queueing_delay_percentiles.rb b/db/click_house/migrate/20230719101806_create_ci_finished_builds_aggregated_queueing_delay_percentiles.rb
new file mode 100644
index 0000000000000..47934d8fe0281
--- /dev/null
+++ b/db/click_house/migrate/20230719101806_create_ci_finished_builds_aggregated_queueing_delay_percentiles.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+class CreateCiFinishedBuildsAggregatedQueueingDelayPercentiles < ClickHouse::Migration
+ def up
+ execute <<~SQL
+ CREATE TABLE IF NOT EXISTS ci_finished_builds_aggregated_queueing_delay_percentiles
+ (
+ status LowCardinality(String) DEFAULT '',
+ runner_type UInt8 DEFAULT 0,
+ started_at_bucket DateTime64(6, 'UTC') DEFAULT now(),
+
+ count_builds AggregateFunction(count),
+ queueing_duration_quantile AggregateFunction(quantile, Int64)
+ )
+ ENGINE = AggregatingMergeTree()
+ ORDER BY (started_at_bucket, status, runner_type)
+ SQL
+ end
+
+ def down
+ execute <<~SQL
+ DROP TABLE ci_finished_builds_aggregated_queueing_delay_percentiles
+ SQL
+ end
+end
diff --git a/db/click_house/migrate/20230724064832_create_contribution_analytics_events.rb b/db/click_house/migrate/20230724064832_create_contribution_analytics_events.rb
new file mode 100644
index 0000000000000..2606ae3adc98f
--- /dev/null
+++ b/db/click_house/migrate/20230724064832_create_contribution_analytics_events.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+class CreateContributionAnalyticsEvents < ClickHouse::Migration
+ def up
+ execute <<~SQL
+ CREATE TABLE IF NOT EXISTS contribution_analytics_events
+ (
+ id UInt64 DEFAULT 0,
+ path String DEFAULT '',
+ author_id UInt64 DEFAULT 0,
+ target_type LowCardinality(String) DEFAULT '',
+ action UInt8 DEFAULT 0,
+ created_at Date DEFAULT toDate(now()),
+ updated_at DateTime64(6, 'UTC') DEFAULT now()
+ )
+ ENGINE = MergeTree
+ ORDER BY (path, created_at, author_id, id)
+ PARTITION BY toYear(created_at);
+ SQL
+ end
+
+ def down
+ execute <<~SQL
+ DROP TABLE contribution_analytics_events
+ SQL
+ end
+end
diff --git a/db/click_house/migrate/20230724064918_create_contribution_analytics_events_materialized_view.rb b/db/click_house/migrate/20230724064918_create_contribution_analytics_events_materialized_view.rb
new file mode 100644
index 0000000000000..956a26d80f3e9
--- /dev/null
+++ b/db/click_house/migrate/20230724064918_create_contribution_analytics_events_materialized_view.rb
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+class CreateContributionAnalyticsEventsMaterializedView < ClickHouse::Migration
+ def up
+ execute <<~SQL
+ CREATE MATERIALIZED VIEW IF NOT EXISTS contribution_analytics_events_mv
+ TO contribution_analytics_events
+ AS
+ SELECT
+ id,
+ argMax(path, events.updated_at) as path,
+ argMax(author_id, events.updated_at) as author_id,
+ argMax(target_type, events.updated_at) as target_type,
+ argMax(action, events.updated_at) as action,
+ argMax(date(created_at), events.updated_at) as created_at,
+ max(events.updated_at) as updated_at
+ FROM events
+ WHERE (("events"."action" = 5 AND "events"."target_type" = '')
+ OR ("events"."action" IN (1, 3, 7, 12)
+ AND "events"."target_type" IN ('MergeRequest', 'Issue')))
+ GROUP BY id
+ SQL
+ end
+
+ def down
+ execute <<~SQL
+ DROP VIEW contribution_analytics_events_mv
+ SQL
+ end
+end
diff --git a/db/click_house/migrate/20230808070520_create_sync_cursors.rb b/db/click_house/migrate/20230808070520_create_sync_cursors.rb
new file mode 100644
index 0000000000000..7583f8ec0c523
--- /dev/null
+++ b/db/click_house/migrate/20230808070520_create_sync_cursors.rb
@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+class CreateSyncCursors < ClickHouse::Migration
+ def up
+ execute <<~SQL
+ CREATE TABLE IF NOT EXISTS sync_cursors
+ (
+ table_name LowCardinality(String) DEFAULT '',
+ primary_key_value UInt64 DEFAULT 0,
+ recorded_at DateTime64(6, 'UTC') DEFAULT now()
+ )
+ ENGINE = ReplacingMergeTree(recorded_at)
+ ORDER BY (table_name)
+ PRIMARY KEY (table_name)
+ SQL
+ end
+
+ def down
+ execute <<~SQL
+ DROP TABLE sync_cursors
+ SQL
+ end
+end
diff --git a/db/click_house/migrate/20230808140217_create_ci_finished_builds_aggregated_queueing_delay_percentiles_mv.rb b/db/click_house/migrate/20230808140217_create_ci_finished_builds_aggregated_queueing_delay_percentiles_mv.rb
new file mode 100644
index 0000000000000..cc029d48436c7
--- /dev/null
+++ b/db/click_house/migrate/20230808140217_create_ci_finished_builds_aggregated_queueing_delay_percentiles_mv.rb
@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+class CreateCiFinishedBuildsAggregatedQueueingDelayPercentilesMv < ClickHouse::Migration
+ def up
+ execute <<~SQL
+ CREATE MATERIALIZED VIEW IF NOT EXISTS ci_finished_builds_aggregated_queueing_delay_percentiles_mv
+ TO ci_finished_builds_aggregated_queueing_delay_percentiles
+ AS
+ SELECT
+ status,
+ runner_type,
+ toStartOfInterval(started_at, INTERVAL 5 minute) AS started_at_bucket,
+
+ countState(*) as count_builds,
+ quantileState(queueing_duration) AS queueing_duration_quantile
+ FROM ci_finished_builds
+ GROUP BY status, runner_type, started_at_bucket
+ SQL
+ end
+
+ def down
+ execute <<~SQL
+ DROP VIEW ci_finished_builds_aggregated_queueing_delay_percentiles_mv
+ SQL
+ end
+end
diff --git a/db/click_house/migrate/20231106202300_modify_ci_finished_builds_settings.rb b/db/click_house/migrate/20231106202300_modify_ci_finished_builds_settings.rb
new file mode 100644
index 0000000000000..d9951725c9ba2
--- /dev/null
+++ b/db/click_house/migrate/20231106202300_modify_ci_finished_builds_settings.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+class ModifyCiFinishedBuildsSettings < ClickHouse::Migration
+ def up
+ execute <<~SQL
+ ALTER TABLE ci_finished_builds MODIFY SETTING use_async_block_ids_cache = true
+ SQL
+ end
+
+ def down
+ execute <<~SQL
+ ALTER TABLE ci_finished_builds MODIFY SETTING use_async_block_ids_cache = false
+ SQL
+ end
+end
diff --git a/lib/click_house/migration.rb b/lib/click_house/migration.rb
new file mode 100644
index 0000000000000..410a7ec86bce0
--- /dev/null
+++ b/lib/click_house/migration.rb
@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module ClickHouse
+ class Migration
+ cattr_accessor :verbose, :client_configuration
+ attr_accessor :name, :version
+
+ class << self
+ attr_accessor :delegate
+ end
+
+ def initialize(name = self.class.name, version = nil)
+ @name = name
+ @version = version
+ end
+
+ self.client_configuration = ClickHouse::Client.configuration
+ self.verbose = true
+ # instantiate the delegate object after initialize is defined
+ self.delegate = new
+
+ MIGRATION_FILENAME_REGEXP = /\A([0-9]+)_([_a-z0-9]*)\.?([_a-z0-9]*)?\.rb\z/
+
+ def database
+ self.class.constants.include?(:SCHEMA) ? self.class.const_get(:SCHEMA, false) : :main
+ end
+
+ def execute(query)
+ ClickHouse::Client.execute(query, database, self.class.client_configuration)
+ end
+
+ def up
+ self.class.delegate = self
+
+ return unless self.class.respond_to?(:up)
+
+ self.class.up
+ end
+
+ def down
+ self.class.delegate = self
+
+ return unless self.class.respond_to?(:down)
+
+ self.class.down
+ end
+
+ # Execute this migration in the named direction
+ def migrate(direction)
+ return unless respond_to?(direction)
+
+ case direction
+ when :up then announce 'migrating'
+ when :down then announce 'reverting'
+ end
+
+ time = Benchmark.measure do
+ exec_migration(direction)
+ end
+
+ case direction
+ when :up then announce format("migrated (%.4fs)", time.real)
+ write
+ when :down then announce format("reverted (%.4fs)", time.real)
+ write
+ end
+ end
+
+ private
+
+ def exec_migration(direction)
+ # noinspection RubyCaseWithoutElseBlockInspection
+ case direction
+ when :up then up
+ when :down then down
+ end
+ end
+
+ def write(text = '')
+ $stdout.puts(text) if verbose
+ end
+
+ def announce(message)
+ text = "#{version} #{name}: #{message}"
+ length = [0, 75 - text.length].max
+ write format('== %s %s', text, '=' * length)
+ end
+ end
+end
diff --git a/lib/click_house/migration_support/migration_context.rb b/lib/click_house/migration_support/migration_context.rb
new file mode 100644
index 0000000000000..6e4dd2a97c244
--- /dev/null
+++ b/lib/click_house/migration_support/migration_context.rb
@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+
+module ClickHouse
+ module MigrationSupport
+ # MigrationContext sets the context in which a migration is run.
+ #
+ # A migration context requires the path to the migrations is set
+ # in the +migrations_paths+ parameter. Optionally a +schema_migration+
+ # class can be provided. For most applications, +SchemaMigration+ is
+ # sufficient. Multiple database applications need a +SchemaMigration+
+ # per primary database.
+ class MigrationContext
+ attr_reader :migrations_paths, :schema_migration
+
+ def initialize(migrations_paths, schema_migration)
+ @migrations_paths = migrations_paths
+ @schema_migration = schema_migration
+ end
+
+ def up(target_version = nil, &block)
+ selected_migrations = block ? migrations.select(&block) : migrations
+
+ migrate(:up, selected_migrations, target_version)
+ end
+
+ def down(target_version = nil, &block)
+ selected_migrations = block ? migrations.select(&block) : migrations
+
+ migrate(:down, selected_migrations, target_version)
+ end
+
+ private
+
+ def migrate(direction, selected_migrations, target_version = nil)
+ ClickHouse::MigrationSupport::Migrator.new(
+ direction,
+ selected_migrations,
+ schema_migration,
+ target_version
+ ).migrate
+ end
+
+ def migrations
+ migrations = migration_files.map do |file|
+ version, name, scope = parse_migration_filename(file)
+
+ raise ClickHouse::MigrationSupport::IllegalMigrationNameError, file unless version
+
+ version = version.to_i
+ name = name.camelize
+
+ MigrationProxy.new(name, version, file, scope)
+ end
+
+ migrations.sort_by(&:version)
+ end
+
+ def migration_files
+ paths = Array(migrations_paths)
+ Dir[*paths.flat_map { |path| "#{path}/**/[0-9]*_*.rb" }]
+ end
+
+ def parse_migration_filename(filename)
+ File.basename(filename).scan(ClickHouse::Migration::MIGRATION_FILENAME_REGEXP).first
+ end
+ end
+
+ # MigrationProxy is used to defer loading of the actual migration classes
+ # until they are needed
+ MigrationProxy = Struct.new(:name, :version, :filename, :scope) do
+ def initialize(name, version, filename, scope)
+ super
+ @migration = nil
+ end
+
+ def basename
+ File.basename(filename)
+ end
+
+ delegate :migrate, :announce, :write, :database, to: :migration
+
+ private
+
+ def migration
+ @migration ||= load_migration
+ end
+
+ def load_migration
+ require(File.expand_path(filename))
+ name.constantize.new(name, version)
+ end
+ end
+ end
+end
diff --git a/lib/click_house/migration_support/migration_error.rb b/lib/click_house/migration_support/migration_error.rb
new file mode 100644
index 0000000000000..0638d487e3729
--- /dev/null
+++ b/lib/click_house/migration_support/migration_error.rb
@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+module ClickHouse
+ module MigrationSupport
+ class MigrationError < StandardError
+ def initialize(message = nil)
+ message = "\n\n#{message}\n\n" if message
+ super
+ end
+ end
+
+ class IllegalMigrationNameError < MigrationError
+ def initialize(name = nil)
+ if name
+ super("Illegal name for migration file: #{name}\n\t(only lower case letters, numbers, and '_' allowed).")
+ else
+ super('Illegal name for migration.')
+ end
+ end
+ end
+
+ IrreversibleMigration = Class.new(MigrationError)
+
+ class DuplicateMigrationVersionError < MigrationError
+ def initialize(version = nil)
+ if version
+ super("Multiple migrations have the version number #{version}.")
+ else
+ super('Duplicate migration version error.')
+ end
+ end
+ end
+
+ class DuplicateMigrationNameError < MigrationError
+ def initialize(name = nil)
+ if name
+ super("Multiple migrations have the name #{name}.")
+ else
+ super('Duplicate migration name.')
+ end
+ end
+ end
+
+ class UnknownMigrationVersionError < MigrationError
+ def initialize(version = nil)
+ if version
+ super("No migration with version number #{version}.")
+ else
+ super('Unknown migration version.')
+ end
+ end
+ end
+ end
+end
diff --git a/lib/click_house/migration_support/migrator.rb b/lib/click_house/migration_support/migrator.rb
new file mode 100644
index 0000000000000..5c67b3a5ff1f3
--- /dev/null
+++ b/lib/click_house/migration_support/migrator.rb
@@ -0,0 +1,160 @@
+# frozen_string_literal: true
+
+module ClickHouse
+ module MigrationSupport
+ class Migrator
+ class << self
+ attr_accessor :migrations_paths
+ end
+
+ attr_accessor :logger
+
+ self.migrations_paths = ["db/click_house/migrate"]
+
+ def initialize(direction, migrations, schema_migration, target_version = nil, logger = Gitlab::AppLogger)
+ @direction = direction
+ @target_version = target_version
+ @migrated_versions = {}
+ @migrations = migrations
+ @schema_migration = schema_migration
+ @logger = logger
+
+ validate(@migrations)
+
+ migrations.map(&:database).uniq.each do |database|
+ @schema_migration.create_table(database)
+ end
+ end
+
+ def current_version
+ @migrated_versions.values.flatten.max || 0
+ end
+
+ def current_migration
+ migrations.detect { |m| m.version == current_version }
+ end
+ alias_method :current, :current_migration
+
+ def run
+ run_without_lock
+ end
+
+ def migrate
+ migrate_without_lock
+ end
+
+ def runnable
+ runnable = migrations[start..finish]
+
+ if up?
+ runnable.reject { |m| ran?(m) }
+ else
+ # skip the last migration if we're headed down, but not ALL the way down
+ runnable.pop if target
+ runnable.find_all { |m| ran?(m) }
+ end
+ end
+
+ def migrations
+ down? ? @migrations.reverse : @migrations.sort_by(&:version)
+ end
+
+ def pending_migrations(database)
+ already_migrated = migrated(database)
+
+ migrations.reject { |m| already_migrated.include?(m.version) }
+ end
+
+ def migrated(database)
+ @migrated_versions[database] || load_migrated(database)
+ end
+
+ def load_migrated(database)
+ @migrated_versions[database] = Set.new(@schema_migration.all_versions(database).map(&:to_i))
+ end
+
+ private
+
+ # Used for running a specific migration.
+ def run_without_lock
+ migration = migrations.detect { |m| m.version == @target_version }
+
+ raise ClickHouse::MigrationSupport::UnknownMigrationVersionError, @target_version if migration.nil?
+
+ execute_migration(migration)
+ end
+
+ # Used for running multiple migrations up to or down to a certain value.
+ def migrate_without_lock
+ raise ClickHouse::MigrationSupport::UnknownMigrationVersionError, @target_version if invalid_target?
+
+ runnable.each(&method(:execute_migration)) # rubocop: disable Performance/MethodObjectAsBlock -- Execute through proxy
+ end
+
+ def ran?(migration)
+ migrated(migration.database).include?(migration.version.to_i)
+ end
+
+ # Return true if a valid version is not provided.
+ def invalid_target?
+ return unless @target_version
+ return if @target_version == 0
+
+ !target
+ end
+
+ def execute_migration(migration)
+ database = migration.database
+
+ return if down? && migrated(database).exclude?(migration.version.to_i)
+ return if up? && migrated(database).include?(migration.version.to_i)
+
+ logger.info "Migrating to #{migration.name} (#{migration.version})" if logger
+
+ migration.migrate(@direction)
+ record_version_state_after_migrating(database, migration.version)
+ rescue StandardError => e
+ msg = "An error has occurred, all later migrations canceled:\n\n#{e}"
+ raise StandardError, msg, e.backtrace
+ end
+
+ def target
+ migrations.detect { |m| m.version == @target_version }
+ end
+
+ def finish
+ migrations.index(target) || (migrations.size - 1)
+ end
+
+ def start
+ up? ? 0 : (migrations.index(current) || 0)
+ end
+
+ def validate(migrations)
+ name, = migrations.group_by(&:name).find { |_, v| v.length > 1 }
+ raise ClickHouse::MigrationSupport::DuplicateMigrationNameError, name if name
+
+ version, = migrations.group_by(&:version).find { |_, v| v.length > 1 }
+ raise ClickHouse::MigrationSupport::DuplicateMigrationVersionError, version if version
+ end
+
+ def record_version_state_after_migrating(database, version)
+ if down?
+ migrated(database).delete(version)
+ @schema_migration.create!(database, version: version.to_s, active: 0)
+ else
+ migrated(database) << version
+ @schema_migration.create!(database, version: version.to_s)
+ end
+ end
+
+ def up?
+ @direction == :up
+ end
+
+ def down?
+ @direction == :down
+ end
+ end
+ end
+end
diff --git a/lib/click_house/migration_support/schema_migration.rb b/lib/click_house/migration_support/schema_migration.rb
new file mode 100644
index 0000000000000..e82debbad0de2
--- /dev/null
+++ b/lib/click_house/migration_support/schema_migration.rb
@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module ClickHouse
+ module MigrationSupport
+ class SchemaMigration
+ class_attribute :table_name_prefix, instance_writer: false, default: ''
+ class_attribute :table_name_suffix, instance_writer: false, default: ''
+ class_attribute :schema_migrations_table_name, instance_accessor: false, default: 'schema_migrations'
+
+ class << self
+ TABLE_EXISTS_QUERY = <<~SQL.squish
+ SELECT 1 FROM system.tables
+ WHERE name = {table_name: String} AND database = {database_name: String}
+ SQL
+
+ def primary_key
+ 'version'
+ end
+
+ def table_name
+ "#{table_name_prefix}#{schema_migrations_table_name}#{table_name_suffix}"
+ end
+
+ def table_exists?(database, configuration = ClickHouse::Migration.client_configuration)
+ database_name = configuration.databases[database]&.database
+ return false unless database_name
+
+ placeholders = { table_name: table_name, database_name: database_name }
+ query = ClickHouse::Client::Query.new(raw_query: TABLE_EXISTS_QUERY, placeholders: placeholders)
+
+ ClickHouse::Client.select(query, database, configuration).any?
+ end
+
+ def create_table(database, configuration = ClickHouse::Migration.client_configuration)
+ return if table_exists?(database, configuration)
+
+ query = <<~SQL
+ CREATE TABLE #{table_name} (
+ version LowCardinality(String),
+ active UInt8 NOT NULL DEFAULT 1,
+ applied_at DateTime64(6, 'UTC') NOT NULL DEFAULT now64()
+ )
+ ENGINE = ReplacingMergeTree(applied_at)
+ PRIMARY KEY(version)
+ ORDER BY (version)
+ SQL
+
+ ClickHouse::Client.execute(query, database, configuration)
+ end
+
+ def all_versions(database)
+ query = <<~SQL
+ SELECT version FROM #{table_name} FINAL
+ WHERE active = 1
+ ORDER BY (version)
+ SQL
+
+ ClickHouse::Client.select(query, database, ClickHouse::Migration.client_configuration).pluck('version')
+ end
+
+ def create!(database, **args)
+ insert_sql = <<~SQL
+ INSERT INTO #{table_name} (#{args.keys.join(',')}) VALUES (#{args.values.join(',')})
+ SQL
+
+ ClickHouse::Client.execute(insert_sql, database, ClickHouse::Migration.client_configuration)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/tasks/gitlab/click_house/migration.rake b/lib/tasks/gitlab/click_house/migration.rake
new file mode 100644
index 0000000000000..2c4bce65d8041
--- /dev/null
+++ b/lib/tasks/gitlab/click_house/migration.rake
@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+namespace :gitlab do
+ namespace :clickhouse do
+ task :prepare_schema_migration_table, [:database] => :environment do |_t, args|
+ require_relative '../../../../lib/click_house/migration_support/schema_migration'
+
+ ClickHouse::MigrationSupport::SchemaMigration.create_table(args.database&.to_sym || :main)
+ end
+
+ desc 'GitLab | ClickHouse | Migrate'
+ task migrate: [:prepare_schema_migration_table] do
+ migrate(:up)
+ end
+
+ desc 'GitLab | ClickHouse | Rollback'
+ task rollback: [:prepare_schema_migration_table] do
+ migrate(:down)
+ end
+
+ private
+
+ def check_target_version
+ return unless target_version
+
+ version = ENV['VERSION']
+
+ return if ClickHouse::Migration::MIGRATION_FILENAME_REGEXP.match?(version) || /\A\d+\z/.match?(version)
+
+ raise "Invalid format of target version: `VERSION=#{version}`"
+ end
+
+ def target_version
+ ENV['VERSION'].to_i if ENV['VERSION'] && !ENV['VERSION'].empty?
+ end
+
+ def verbose
+ ENV['VERBOSE'] ? ENV['VERBOSE'] != 'false' : true
+ end
+
+ def migrate(direction)
+ require_relative '../../../../lib/click_house/migration_support/schema_migration'
+ require_relative '../../../../lib/click_house/migration_support/migration_context'
+ require_relative '../../../../lib/click_house/migration_support/migrator'
+
+ check_target_version
+
+ scope = ENV['SCOPE']
+ verbose_was = ClickHouse::Migration.verbose
+ ClickHouse::Migration.verbose = verbose
+
+ migrations_paths = ClickHouse::MigrationSupport::Migrator.migrations_paths
+ schema_migration = ClickHouse::MigrationSupport::SchemaMigration
+ migration_context = ClickHouse::MigrationSupport::MigrationContext.new(migrations_paths, schema_migration)
+ migrations_ran = migration_context.public_send(direction, target_version) do |migration|
+ scope.blank? || scope == migration.scope
+ end
+
+ puts('No migrations ran.') unless migrations_ran&.any?
+ ensure
+ ClickHouse::Migration.verbose = verbose_was
+ end
+ end
+end
diff --git a/rubocop/rubocop-code_reuse.yml b/rubocop/rubocop-code_reuse.yml
index f96de5caf994d..2bd3339368da8 100644
--- a/rubocop/rubocop-code_reuse.yml
+++ b/rubocop/rubocop-code_reuse.yml
@@ -24,6 +24,7 @@ CodeReuse/ActiveRecord:
- danger/**/*.rb
- lib/backup/**/*.rb
- lib/banzai/**/*.rb
+ - lib/click_house/migration_support/**/*.rb
- lib/gitlab/background_migration/**/*.rb
- lib/gitlab/cycle_analytics/**/*.rb
- lib/gitlab/counters/**/*.rb
diff --git a/spec/click_house/migration_support/migration_context_spec.rb b/spec/click_house/migration_support/migration_context_spec.rb
new file mode 100644
index 0000000000000..9df8391270dfe
--- /dev/null
+++ b/spec/click_house/migration_support/migration_context_spec.rb
@@ -0,0 +1,233 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+require_relative '../../../lib/click_house/migration_support/migration_error'
+
+RSpec.describe ClickHouse::MigrationSupport::MigrationContext,
+ click_house: :without_migrations, feature_category: :database do
+ include ClickHouseHelpers
+
+ # We don't need to delete data since we don't modify Postgres data
+ self.use_transactional_tests = false
+
+ let_it_be(:schema_migration) { ClickHouse::MigrationSupport::SchemaMigration }
+
+ let(:migrations_base_dir) { 'click_house/migrations' }
+ let(:migrations_dir) { expand_fixture_path("#{migrations_base_dir}/#{migrations_dirname}") }
+ let(:migration_context) { described_class.new(migrations_dir, schema_migration) }
+ let(:target_version) { nil }
+
+ after do
+ clear_consts(expand_fixture_path(migrations_base_dir))
+ end
+
+ describe 'performs migrations' do
+ subject(:migration) { migrate(target_version, migration_context) }
+
+ describe 'when creating a table' do
+ let(:migrations_dirname) { 'plain_table_creation' }
+
+ it 'creates a table' do
+ expect { migration }.to change { active_schema_migrations_count }.from(0).to(1)
+
+ table_schema = describe_table('some')
+ expect(schema_migrations).to contain_exactly(a_hash_including(version: '1', active: 1))
+ expect(table_schema).to match({
+ id: a_hash_including(type: 'UInt64'),
+ date: a_hash_including(type: 'Date')
+ })
+ end
+ end
+
+ describe 'when dropping a table' do
+ let(:migrations_dirname) { 'drop_table' }
+ let(:target_version) { 2 }
+
+ it 'drops table' do
+ migrate(1, migration_context)
+ expect(table_names).to include('some')
+
+ migration
+ expect(table_names).not_to include('some')
+ end
+ end
+
+ context 'when a migration raises an error' do
+ let(:migrations_dirname) { 'migration_with_error' }
+
+ it 'passes the error to caller as a StandardError' do
+ expect { migration }.to raise_error StandardError,
+ "An error has occurred, all later migrations canceled:\n\nA migration error happened"
+ expect(schema_migrations).to be_empty
+ end
+ end
+
+ context 'when a migration targets an unknown database' do
+ let(:migrations_dirname) { 'plain_table_creation_on_invalid_database' }
+
+ it 'raises ConfigurationError' do
+ expect { migration }.to raise_error ClickHouse::Client::ConfigurationError,
+ "The database 'unknown_database' is not configured"
+ end
+ end
+
+ context 'when migrations target multiple databases' do
+ let_it_be(:config) { ClickHouse::Client::Configuration.new }
+ let_it_be(:main_db_config) { [:main, config] }
+ let_it_be(:another_db_config) { [:another_db, config] }
+ let_it_be(:another_database_name) { 'gitlab_clickhouse_test_2' }
+
+ let(:migrations_dirname) { 'migrations_over_multiple_databases' }
+
+ before(:context) do
+ # Ensure we have a second database to run the test on
+ clone_database_configuration(:main, :another_db, another_database_name, config)
+
+ with_net_connect_allowed do
+ ClickHouse::Client.execute("CREATE DATABASE IF NOT EXISTS #{another_database_name}", :main, config)
+ end
+ end
+
+ after(:context) do
+ with_net_connect_allowed do
+ ClickHouse::Client.execute("DROP DATABASE #{another_database_name}", :another_db, config)
+ end
+ end
+
+ around do |example|
+ clear_db(configuration: config)
+
+ previous_config = ClickHouse::Migration.client_configuration
+ ClickHouse::Migration.client_configuration = config
+
+ example.run
+ ensure
+ ClickHouse::Migration.client_configuration = previous_config
+ end
+
+ def clone_database_configuration(source_db_identifier, target_db_identifier, target_db_name, target_config)
+ raw_config = Rails.application.config_for(:click_house)
+ raw_config.each do |database_identifier, db_config|
+ register_database(target_config, database_identifier, db_config)
+ end
+
+ target_db_config = raw_config[source_db_identifier].merge(database: target_db_name)
+ register_database(target_config, target_db_identifier, target_db_config)
+ target_config.http_post_proc = ClickHouse::Client.configuration.http_post_proc
+ target_config.json_parser = ClickHouse::Client.configuration.json_parser
+ target_config.logger = ::Logger.new(IO::NULL)
+ end
+
+ it 'registers migrations on respective database', :aggregate_failures do
+ expect { migrate(2, migration_context) }
+ .to change { active_schema_migrations_count(*main_db_config) }.from(0).to(1)
+ .and change { active_schema_migrations_count(*another_db_config) }.from(0).to(1)
+
+ expect(schema_migrations(*another_db_config)).to contain_exactly(a_hash_including(version: '2', active: 1))
+ expect(table_names(*main_db_config)).not_to include('some_on_another_db')
+ expect(table_names(*another_db_config)).not_to include('some')
+
+ expect(describe_table('some', *main_db_config)).to match({
+ id: a_hash_including(type: 'UInt64'),
+ date: a_hash_including(type: 'Date')
+ })
+ expect(describe_table('some_on_another_db', *another_db_config)).to match({
+ id: a_hash_including(type: 'UInt64'),
+ date: a_hash_including(type: 'Date')
+ })
+
+ expect { migrate(nil, migration_context) }
+ .to change { active_schema_migrations_count(*main_db_config) }.to(2)
+ .and not_change { active_schema_migrations_count(*another_db_config) }
+
+ expect(schema_migrations(*main_db_config)).to match([
+ a_hash_including(version: '1', active: 1),
+ a_hash_including(version: '3', active: 1)
+ ])
+ expect(schema_migrations(*another_db_config)).to match_array(a_hash_including(version: '2', active: 1))
+
+ expect(describe_table('some', *main_db_config)).to match({
+ id: a_hash_including(type: 'UInt64'),
+ timestamp: a_hash_including(type: 'Date')
+ })
+ end
+ end
+
+ context 'when target_version is incorrect' do
+ let(:target_version) { 2 }
+ let(:migrations_dirname) { 'plain_table_creation' }
+
+ it 'raises UnknownMigrationVersionError' do
+ expect { migration }.to raise_error ClickHouse::MigrationSupport::UnknownMigrationVersionError
+
+ expect(active_schema_migrations_count).to eq 0
+ end
+ end
+
+ context 'when migrations with duplicate name exist' do
+ let(:migrations_dirname) { 'duplicate_name' }
+
+ it 'raises DuplicateMigrationNameError' do
+ expect { migration }.to raise_error ClickHouse::MigrationSupport::DuplicateMigrationNameError
+
+ expect(active_schema_migrations_count).to eq 0
+ end
+ end
+
+ context 'when migrations with duplicate version exist' do
+ let(:migrations_dirname) { 'duplicate_version' }
+
+ it 'raises DuplicateMigrationVersionError' do
+ expect { migration }.to raise_error ClickHouse::MigrationSupport::DuplicateMigrationVersionError
+
+ expect(active_schema_migrations_count).to eq 0
+ end
+ end
+ end
+
+ describe 'performs rollbacks' do
+ subject(:migration) { rollback(target_version, migration_context) }
+
+ before do
+ migrate(nil, migration_context)
+ end
+
+ context 'when migrating back all the way to 0' do
+ let(:target_version) { 0 }
+
+ context 'when down method is present' do
+ let(:migrations_dirname) { 'table_creation_with_down_method' }
+
+ it 'removes migration and performs down method' do
+ expect(table_names).to include('some')
+
+ expect { migration }.to change { active_schema_migrations_count }.from(1).to(0)
+
+ expect(table_names).not_to include('some')
+ expect(schema_migrations).to contain_exactly(a_hash_including(version: '1', active: 0))
+ end
+ end
+
+ context 'when down method is missing' do
+ let(:migrations_dirname) { 'plain_table_creation' }
+
+ it 'removes migration ignoring missing down method' do
+ expect { migration }.to change { active_schema_migrations_count }.from(1).to(0)
+ .and not_change { table_names & %w[some] }.from(%w[some])
+ end
+ end
+ end
+
+ context 'when target_version is incorrect' do
+ let(:target_version) { -1 }
+ let(:migrations_dirname) { 'plain_table_creation' }
+
+ it 'raises UnknownMigrationVersionError' do
+ expect { migration }.to raise_error ClickHouse::MigrationSupport::UnknownMigrationVersionError
+
+ expect(active_schema_migrations_count).to eq 1
+ end
+ end
+ end
+end
diff --git a/spec/fixtures/click_house/migrations/drop_table/1_create_some_table.rb b/spec/fixtures/click_house/migrations/drop_table/1_create_some_table.rb
new file mode 100644
index 0000000000000..14ef80cbdb743
--- /dev/null
+++ b/spec/fixtures/click_house/migrations/drop_table/1_create_some_table.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+# rubocop: disable Gitlab/NamespacedClass -- Fixtures do not need to be namespaced
+class CreateSomeTable < ClickHouse::Migration
+ def up
+ execute <<~SQL
+ CREATE TABLE some (
+ id UInt64,
+ date Date
+ ) ENGINE = Memory
+ SQL
+ end
+end
+# rubocop: enable Gitlab/NamespacedClass
diff --git a/spec/fixtures/click_house/migrations/drop_table/2_drop_some_table.rb b/spec/fixtures/click_house/migrations/drop_table/2_drop_some_table.rb
new file mode 100644
index 0000000000000..82045b08e21ad
--- /dev/null
+++ b/spec/fixtures/click_house/migrations/drop_table/2_drop_some_table.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+# rubocop: disable Gitlab/NamespacedClass -- Fixtures do not need to be namespaced
+class DropSomeTable < ClickHouse::Migration
+ def up
+ execute <<~SQL
+ DROP TABLE some
+ SQL
+ end
+end
+# rubocop: enable Gitlab/NamespacedClass
diff --git a/spec/fixtures/click_house/migrations/duplicate_name/1_create_some_table.rb b/spec/fixtures/click_house/migrations/duplicate_name/1_create_some_table.rb
new file mode 100644
index 0000000000000..14ef80cbdb743
--- /dev/null
+++ b/spec/fixtures/click_house/migrations/duplicate_name/1_create_some_table.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+# rubocop: disable Gitlab/NamespacedClass -- Fixtures do not need to be namespaced
+class CreateSomeTable < ClickHouse::Migration
+ def up
+ execute <<~SQL
+ CREATE TABLE some (
+ id UInt64,
+ date Date
+ ) ENGINE = Memory
+ SQL
+ end
+end
+# rubocop: enable Gitlab/NamespacedClass
diff --git a/spec/fixtures/click_house/migrations/duplicate_name/2_create_some_table.rb b/spec/fixtures/click_house/migrations/duplicate_name/2_create_some_table.rb
new file mode 100644
index 0000000000000..be6c1905502aa
--- /dev/null
+++ b/spec/fixtures/click_house/migrations/duplicate_name/2_create_some_table.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+# rubocop: disable Gitlab/NamespacedClass -- Fixtures do not need to be namespaced
+class CreateSomeTable2 < ClickHouse::Migration
+ def up
+ execute <<~SQL
+ CREATE TABLE some (
+ id UInt64,
+ date Date
+ ) ENGINE = Memory
+ SQL
+ end
+end
+# rubocop: enable Gitlab/NamespacedClass
diff --git a/spec/fixtures/click_house/migrations/duplicate_version/1_create_some_table.rb b/spec/fixtures/click_house/migrations/duplicate_version/1_create_some_table.rb
new file mode 100644
index 0000000000000..14ef80cbdb743
--- /dev/null
+++ b/spec/fixtures/click_house/migrations/duplicate_version/1_create_some_table.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+# rubocop: disable Gitlab/NamespacedClass -- Fixtures do not need to be namespaced
+class CreateSomeTable < ClickHouse::Migration
+ def up
+ execute <<~SQL
+ CREATE TABLE some (
+ id UInt64,
+ date Date
+ ) ENGINE = Memory
+ SQL
+ end
+end
+# rubocop: enable Gitlab/NamespacedClass
diff --git a/spec/fixtures/click_house/migrations/duplicate_version/1_drop_some_table.rb b/spec/fixtures/click_house/migrations/duplicate_version/1_drop_some_table.rb
new file mode 100644
index 0000000000000..82045b08e21ad
--- /dev/null
+++ b/spec/fixtures/click_house/migrations/duplicate_version/1_drop_some_table.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+# rubocop: disable Gitlab/NamespacedClass -- Fixtures do not need to be namespaced
+class DropSomeTable < ClickHouse::Migration
+ def up
+ execute <<~SQL
+ DROP TABLE some
+ SQL
+ end
+end
+# rubocop: enable Gitlab/NamespacedClass
diff --git a/spec/fixtures/click_house/migrations/migration_with_error/1_migration_with_error.rb b/spec/fixtures/click_house/migrations/migration_with_error/1_migration_with_error.rb
new file mode 100644
index 0000000000000..b8ae3df20851d
--- /dev/null
+++ b/spec/fixtures/click_house/migrations/migration_with_error/1_migration_with_error.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+# rubocop: disable Gitlab/NamespacedClass -- Fixtures do not need to be namespaced
+class MigrationWithError < ClickHouse::Migration
+ def up
+ raise ClickHouse::Client::DatabaseError, 'A migration error happened'
+ end
+end
+# rubocop: enable Gitlab/NamespacedClass
diff --git a/spec/fixtures/click_house/migrations/migrations_over_multiple_databases/1_create_some_table_on_main_db.rb b/spec/fixtures/click_house/migrations/migrations_over_multiple_databases/1_create_some_table_on_main_db.rb
new file mode 100644
index 0000000000000..98d71d9507b07
--- /dev/null
+++ b/spec/fixtures/click_house/migrations/migrations_over_multiple_databases/1_create_some_table_on_main_db.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+# rubocop: disable Gitlab/NamespacedClass -- Fixtures do not need to be namespaced
+class CreateSomeTableOnMainDb < ClickHouse::Migration
+ def up
+ execute <<~SQL
+ CREATE TABLE some (
+ id UInt64,
+ date Date
+ ) ENGINE = MergeTree
+ PRIMARY KEY(id)
+ SQL
+ end
+end
+# rubocop: enable Gitlab/NamespacedClass
diff --git a/spec/fixtures/click_house/migrations/migrations_over_multiple_databases/2_create_some_table_on_another_db.rb b/spec/fixtures/click_house/migrations/migrations_over_multiple_databases/2_create_some_table_on_another_db.rb
new file mode 100644
index 0000000000000..b8cd86a67f58a
--- /dev/null
+++ b/spec/fixtures/click_house/migrations/migrations_over_multiple_databases/2_create_some_table_on_another_db.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+# rubocop: disable Gitlab/NamespacedClass -- Fixtures do not need to be namespaced
+class CreateSomeTableOnAnotherDb < ClickHouse::Migration
+ SCHEMA = :another_db
+
+ def up
+ execute <<~SQL
+ CREATE TABLE some_on_another_db (
+ id UInt64,
+ date Date
+ ) ENGINE = Memory
+ SQL
+ end
+end
+# rubocop: enable Gitlab/NamespacedClass
diff --git a/spec/fixtures/click_house/migrations/migrations_over_multiple_databases/3_change_some_table_on_main_db.rb b/spec/fixtures/click_house/migrations/migrations_over_multiple_databases/3_change_some_table_on_main_db.rb
new file mode 100644
index 0000000000000..9112ab79fc5f4
--- /dev/null
+++ b/spec/fixtures/click_house/migrations/migrations_over_multiple_databases/3_change_some_table_on_main_db.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+# rubocop: disable Gitlab/NamespacedClass -- Fixtures do not need to be namespaced
+class ChangeSomeTableOnMainDb < ClickHouse::Migration
+ def up
+ execute <<~SQL
+ ALTER TABLE some RENAME COLUMN date to timestamp
+ SQL
+ end
+end
+# rubocop: enable Gitlab/NamespacedClass
diff --git a/spec/fixtures/click_house/migrations/plain_table_creation/1_create_some_table.rb b/spec/fixtures/click_house/migrations/plain_table_creation/1_create_some_table.rb
new file mode 100644
index 0000000000000..14ef80cbdb743
--- /dev/null
+++ b/spec/fixtures/click_house/migrations/plain_table_creation/1_create_some_table.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+# rubocop: disable Gitlab/NamespacedClass -- Fixtures do not need to be namespaced
+class CreateSomeTable < ClickHouse::Migration
+ def up
+ execute <<~SQL
+ CREATE TABLE some (
+ id UInt64,
+ date Date
+ ) ENGINE = Memory
+ SQL
+ end
+end
+# rubocop: enable Gitlab/NamespacedClass
diff --git a/spec/fixtures/click_house/migrations/plain_table_creation_on_invalid_database/1_create_some_table.rb b/spec/fixtures/click_house/migrations/plain_table_creation_on_invalid_database/1_create_some_table.rb
new file mode 100644
index 0000000000000..ee900ef24c5cb
--- /dev/null
+++ b/spec/fixtures/click_house/migrations/plain_table_creation_on_invalid_database/1_create_some_table.rb
@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+# rubocop: disable Gitlab/NamespacedClass -- Fixtures do not need to be namespaced
+class CreateSomeTable < ClickHouse::Migration
+ SCHEMA = :unknown_database
+
+ def up
+ execute <<~SQL
+ CREATE TABLE some (
+ id UInt64,
+ date Date
+ ) ENGINE = Memory
+ SQL
+ end
+end
+# rubocop: enable Gitlab/NamespacedClass
diff --git a/spec/fixtures/click_house/migrations/table_creation_with_down_method/1_create_some_table.rb b/spec/fixtures/click_house/migrations/table_creation_with_down_method/1_create_some_table.rb
new file mode 100644
index 0000000000000..7ac92b9ee38b9
--- /dev/null
+++ b/spec/fixtures/click_house/migrations/table_creation_with_down_method/1_create_some_table.rb
@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+# rubocop: disable Gitlab/NamespacedClass -- Fixtures do not need to be namespaced
+class CreateSomeTable < ClickHouse::Migration
+ def up
+ execute <<~SQL
+ CREATE TABLE some (
+ id UInt64,
+ date Date
+ ) ENGINE = Memory
+ SQL
+ end
+
+ def down
+ execute <<~SQL
+ DROP TABLE some
+ SQL
+ end
+end
+# rubocop: enable Gitlab/NamespacedClass
diff --git a/spec/support/database/click_house/hooks.rb b/spec/support/database/click_house/hooks.rb
index b970d3daf84dc..c13778f9c3679 100644
--- a/spec/support/database/click_house/hooks.rb
+++ b/spec/support/database/click_house/hooks.rb
@@ -9,6 +9,8 @@ def truncate_tables
"(SELECT '#{table}' AS table FROM #{table} LIMIT 1)"
end.join(' UNION ALL ')
+ next if query.empty?
+
tables_with_data = ClickHouse::Client.select(query, db).pluck('table')
tables_with_data.each do |table|
ClickHouse::Client.execute("TRUNCATE TABLE #{table}", db)
@@ -16,20 +18,27 @@ def truncate_tables
end
end
- def ensure_schema
- return if @ensure_schema
-
- ClickHouse::Client.configuration.databases.each_key do |db|
+ def clear_db(configuration = ClickHouse::Client.configuration)
+ configuration.databases.each_key do |db|
# drop all tables
- lookup_tables(db).each do |table|
- ClickHouse::Client.execute("DROP TABLE IF EXISTS #{table}", db)
+ lookup_tables(db, configuration).each do |table|
+ ClickHouse::Client.execute("DROP TABLE IF EXISTS #{table}", db, configuration)
end
- # run the schema SQL files
- Dir[Rails.root.join("db/click_house/#{db}/*.sql")].each do |file|
- ClickHouse::Client.execute(File.read(file), db)
- end
+ ClickHouse::MigrationSupport::SchemaMigration.create_table(db, configuration)
end
+ end
+
+ def ensure_schema
+ return if @ensure_schema
+
+ clear_db
+
+ # run the schema SQL files
+ migrations_paths = ClickHouse::MigrationSupport::Migrator.migrations_paths
+ schema_migration = ClickHouse::MigrationSupport::SchemaMigration
+ migration_context = ClickHouse::MigrationSupport::MigrationContext.new(migrations_paths, schema_migration)
+ migration_context.up
@ensure_schema = true
end
@@ -38,11 +47,11 @@ def ensure_schema
def tables_for(db)
@tables ||= {}
- @tables[db] ||= lookup_tables(db)
+ @tables[db] ||= lookup_tables(db) - [ClickHouse::MigrationSupport::SchemaMigration.table_name]
end
- def lookup_tables(db)
- ClickHouse::Client.select('SHOW TABLES', db).pluck('name')
+ def lookup_tables(db, configuration = ClickHouse::Client.configuration)
+ ClickHouse::Client.select('SHOW TABLES', db, configuration).pluck('name')
end
end
# rubocop: enable Gitlab/NamespacedClass
@@ -52,10 +61,19 @@ def lookup_tables(db)
config.around(:each, :click_house) do |example|
with_net_connect_allowed do
- click_house_test_runner.ensure_schema
- click_house_test_runner.truncate_tables
+ was_verbose = ClickHouse::Migration.verbose
+ ClickHouse::Migration.verbose = false
+
+ if example.example.metadata[:click_house] == :without_migrations
+ click_house_test_runner.clear_db
+ else
+ click_house_test_runner.ensure_schema
+ click_house_test_runner.truncate_tables
+ end
example.run
+ ensure
+ ClickHouse::Migration.verbose = was_verbose
end
end
end
diff --git a/spec/support/helpers/click_house_helpers.rb b/spec/support/helpers/click_house_helpers.rb
new file mode 100644
index 0000000000000..93d5b1d6e01c9
--- /dev/null
+++ b/spec/support/helpers/click_house_helpers.rb
@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+module ClickHouseHelpers
+ private
+
+ def migrate(target_version, migration_context)
+ quietly { migration_context.up(target_version) }
+ end
+
+ def rollback(target_version, migration_context)
+ quietly { migration_context.down(target_version) }
+ end
+
+ def table_names(database = :main, configuration = ClickHouse::Client.configuration)
+ ClickHouse::Client.select('SHOW TABLES', database, configuration).pluck('name')
+ end
+
+ def active_schema_migrations_count(database = :main, configuration = ClickHouse::Client.configuration)
+ query = <<~SQL
+ SELECT COUNT(*) AS count FROM schema_migrations FINAL WHERE active = 1
+ SQL
+
+ ClickHouse::Client.select(query, database, configuration).first['count']
+ end
+
+ def describe_table(table_name, database = :main, configuration = ClickHouse::Client.configuration)
+ ClickHouse::Client
+ .select("DESCRIBE TABLE #{table_name} FORMAT JSON", database, configuration)
+ .map(&:symbolize_keys)
+ .index_by { |h| h[:name].to_sym }
+ end
+
+ def schema_migrations(database = :main, configuration = ClickHouse::Client.configuration)
+ ClickHouse::Client
+ .select('SELECT * FROM schema_migrations FINAL ORDER BY version ASC', database, configuration)
+ .map(&:symbolize_keys)
+ end
+
+ def clear_db(configuration: ClickHouse::Client.configuration)
+ ClickHouseTestRunner.new.clear_db(configuration)
+ end
+
+ def register_database(config, database_identifier, db_config)
+ config.register_database(
+ database_identifier,
+ database: db_config[:database],
+ url: db_config[:url],
+ username: db_config[:username],
+ password: db_config[:password],
+ variables: db_config[:variables] || {}
+ )
+ end
+
+ def clear_consts(fixtures_path)
+ $LOADED_FEATURES.select { |file| file.include? fixtures_path }.each do |file|
+ const = File.basename(file)
+ .scan(ClickHouse::Migration::MIGRATION_FILENAME_REGEXP)[0][1]
+ .camelcase
+ .safe_constantize
+
+ Object.send(:remove_const, const.to_s) if const
+ $LOADED_FEATURES.delete(file)
+ end
+ end
+
+ def quietly(&_block)
+ was_verbose = ClickHouse::Migration.verbose
+ ClickHouse::Migration.verbose = false
+
+ yield
+ ensure
+ ClickHouse::Migration.verbose = was_verbose
+ end
+end
diff --git a/spec/tasks/gitlab/click_house/migration_rake_spec.rb b/spec/tasks/gitlab/click_house/migration_rake_spec.rb
new file mode 100644
index 0000000000000..6b834d52e9a63
--- /dev/null
+++ b/spec/tasks/gitlab/click_house/migration_rake_spec.rb
@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe 'gitlab:clickhouse', click_house: :without_migrations, feature_category: :database do
+ include ClickHouseHelpers
+
+ # We don't need to delete data since we don't modify Postgres data
+ self.use_transactional_tests = false
+
+ let(:migrations_base_dir) { 'click_house/migrations' }
+ let(:migrations_dirname) { '' }
+ let(:migrations_dir) { expand_fixture_path("#{migrations_base_dir}/#{migrations_dirname}") }
+
+ before(:all) do
+ Rake.application.rake_require 'tasks/gitlab/click_house/migration'
+ end
+
+ before do
+ stub_env('VERBOSE', 'false')
+ end
+
+ describe 'migrate' do
+ subject(:migration) { run_rake_task('gitlab:clickhouse:migrate') }
+
+ let(:target_version) { nil }
+
+ around do |example|
+ ClickHouse::MigrationSupport::Migrator.migrations_paths = [migrations_dir]
+
+ example.run
+
+ clear_consts(expand_fixture_path(migrations_base_dir))
+ end
+
+ before do
+ stub_env('VERSION', target_version) if target_version
+ end
+
+ describe 'when creating a table' do
+ let(:migrations_dirname) { 'plain_table_creation' }
+
+ it 'creates a table' do
+ expect { migration }.to change { active_schema_migrations_count }.from(0).to(1)
+
+ expect(describe_table('some')).to match({
+ id: a_hash_including(type: 'UInt64'),
+ date: a_hash_including(type: 'Date')
+ })
+ end
+ end
+
+ describe 'when dropping a table' do
+ let(:migrations_dirname) { 'drop_table' }
+ let(:target_version) { 2 }
+
+ it 'drops table' do
+ stub_env('VERSION', 1)
+ run_rake_task('gitlab:clickhouse:migrate')
+
+ expect(table_names).to include('some')
+
+ stub_env('VERSION', target_version)
+ migration
+ expect(table_names).not_to include('some')
+ end
+ end
+
+ describe 'with VERSION is invalid' do
+ let(:migrations_dirname) { 'plain_table_creation' }
+ let(:target_version) { 'invalid' }
+
+ it { expect { migration }.to raise_error RuntimeError, 'Invalid format of target version: `VERSION=invalid`' }
+ end
+ end
+
+ describe 'rollback' do
+ subject(:migration) { run_rake_task('gitlab:clickhouse:rollback') }
+
+ let(:schema_migration) { ClickHouse::MigrationSupport::SchemaMigration }
+
+ around do |example|
+ ClickHouse::MigrationSupport::Migrator.migrations_paths = [migrations_dir]
+ migrate(nil, ClickHouse::MigrationSupport::MigrationContext.new(migrations_dir, schema_migration))
+
+ example.run
+
+ clear_consts(expand_fixture_path(migrations_base_dir))
+ end
+
+ context 'when migrating back all the way to 0' do
+ let(:target_version) { 0 }
+
+ context 'when down method is present' do
+ let(:migrations_dirname) { 'table_creation_with_down_method' }
+
+ it 'removes migration' do
+ expect(table_names).to include('some')
+
+ migration
+ expect(table_names).not_to include('some')
+ end
+ end
+ end
+ end
+
+ %w[gitlab:clickhouse:migrate].each do |task|
+ context "when running #{task}" do
+ it "does run gitlab:clickhouse:prepare_schema_migration_table before" do
+ expect(Rake::Task['gitlab:clickhouse:prepare_schema_migration_table']).to receive(:execute).and_return(true)
+ expect(Rake::Task[task]).to receive(:execute).and_return(true)
+
+ Rake::Task['gitlab:clickhouse:prepare_schema_migration_table'].reenable
+ run_rake_task(task)
+ end
+ end
+ end
+end
diff --git a/spec/tooling/quality/test_level_spec.rb b/spec/tooling/quality/test_level_spec.rb
index 6ccd2e46f7bb3..d7d04015b4806 100644
--- a/spec/tooling/quality/test_level_spec.rb
+++ b/spec/tooling/quality/test_level_spec.rb
@@ -46,7 +46,7 @@
context 'when level is unit' do
it 'returns a pattern' do
expect(subject.pattern(:unit))
- .to eq("spec/{bin,channels,components,config,contracts,db,dependencies,elastic,elastic_integration,experiments,factories,finders,frontend,graphql,haml_lint,helpers,initializers,lib,metrics_server,models,policies,presenters,rack_servers,replicators,routing,rubocop,scripts,serializers,services,sidekiq,sidekiq_cluster,spam,support_specs,tasks,uploaders,validators,views,workers,tooling}{,/**/}*_spec.rb")
+ .to eq("spec/{bin,channels,click_house,components,config,contracts,db,dependencies,elastic,elastic_integration,experiments,factories,finders,frontend,graphql,haml_lint,helpers,initializers,lib,metrics_server,models,policies,presenters,rack_servers,replicators,routing,rubocop,scripts,serializers,services,sidekiq,sidekiq_cluster,spam,support_specs,tasks,uploaders,validators,views,workers,tooling}{,/**/}*_spec.rb")
end
end
@@ -121,7 +121,7 @@
context 'when level is unit' do
it 'returns a regexp' do
expect(subject.regexp(:unit))
- .to eq(%r{spec/(bin|channels|components|config|contracts|db|dependencies|elastic|elastic_integration|experiments|factories|finders|frontend|graphql|haml_lint|helpers|initializers|lib|metrics_server|models|policies|presenters|rack_servers|replicators|routing|rubocop|scripts|serializers|services|sidekiq|sidekiq_cluster|spam|support_specs|tasks|uploaders|validators|views|workers|tooling)/})
+ .to eq(%r{spec/(bin|channels|click_house|components|config|contracts|db|dependencies|elastic|elastic_integration|experiments|factories|finders|frontend|graphql|haml_lint|helpers|initializers|lib|metrics_server|models|policies|presenters|rack_servers|replicators|routing|rubocop|scripts|serializers|services|sidekiq|sidekiq_cluster|spam|support_specs|tasks|uploaders|validators|views|workers|tooling)/})
end
end
diff --git a/tooling/quality/test_level.rb b/tooling/quality/test_level.rb
index 20e00763f6543..050eb4f4dafd6 100644
--- a/tooling/quality/test_level.rb
+++ b/tooling/quality/test_level.rb
@@ -18,6 +18,7 @@ class TestLevel
unit: %w[
bin
channels
+ click_house
components
config
contracts
--
GitLab