From 61c73870e5b8d4a884424d30952289d6e6f00f7f Mon Sep 17 00:00:00 2001
From: Krasimir Angelov <kangelov@gitlab.com>
Date: Mon, 17 Oct 2022 11:51:40 +0000
Subject: [PATCH] Add CI job to verify execution of migrations in two steps

In order to veryfy post-deployment migrations add a new CI job that
executes migrations in two steps, similar to what happens during
zero-downtime deployments.

Related to
https://gitlab.com/gitlab-com/gl-infra/production/-/issues/7762#note_1106257962.
---
 .gitlab/ci/rails.gitlab-ci.yml               |  15 +++
 scripts/migration_schema_validator.rb        | 117 +++++++++++++++++++
 scripts/post_deployment_migrations_validator |  31 +++++
 scripts/validate_migration_schema            | 116 +-----------------
 4 files changed, 164 insertions(+), 115 deletions(-)
 create mode 100644 scripts/migration_schema_validator.rb
 create mode 100755 scripts/post_deployment_migrations_validator

diff --git a/.gitlab/ci/rails.gitlab-ci.yml b/.gitlab/ci/rails.gitlab-ci.yml
index 672d2301c3f3e..c60f85634b650 100644
--- a/.gitlab/ci/rails.gitlab-ci.yml
+++ b/.gitlab/ci/rails.gitlab-ci.yml
@@ -236,6 +236,21 @@ db:check-migrations-single-db:
     - .single-db
     - .rails:rules:single-db
 
+db:post_deployment_migrations_validator:
+  extends:
+    - .db-job-base
+    - .rails:rules:ee-and-foss-mr-with-migration
+  script:
+    - git fetch origin $CI_MERGE_REQUEST_TARGET_BRANCH_NAME:$CI_MERGE_REQUEST_TARGET_BRANCH_NAME --depth 20
+    - scripts/post_deployment_migrations_validator
+  allow_failure: true
+
+db:post_deployment_migrations_validator-single-db:
+  extends:
+    - db:post_deployment_migrations_validator
+    - .single-db
+    - .rails:rules:single-db
+
 db:migrate-non-superuser:
   extends:
     - .db-job-base
diff --git a/scripts/migration_schema_validator.rb b/scripts/migration_schema_validator.rb
new file mode 100644
index 0000000000000..08b904ce46ce1
--- /dev/null
+++ b/scripts/migration_schema_validator.rb
@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+require 'open3'
+
+class MigrationSchemaValidator
+  FILENAME = 'db/structure.sql'
+
+  MIGRATION_DIRS = %w[db/migrate db/post_migrate].freeze
+
+  SCHEMA_VERSION_DIR = 'db/schema_migrations'
+
+  VERSION_DIGITS = 14
+
+  def validate!
+    if committed_migrations.empty?
+      puts "\e[32m No migrations found, skipping schema validation\e[0m"
+      return
+    end
+
+    validate_schema_on_rollback!
+    validate_schema_on_migrate!
+    validate_schema_version_files!
+  end
+
+  private
+
+  def validate_schema_on_rollback!
+    committed_migrations.reverse_each do |filename|
+      version = find_migration_version(filename)
+
+      run("scripts/db_tasks db:migrate:down VERSION=#{version}")
+      run("scripts/db_tasks db:schema:dump")
+    end
+
+    git_command = "git diff #{diff_target} -- #{FILENAME}"
+    base_message = "rollback of added migrations does not revert #{FILENAME} to previous state"
+
+    validate_clean_output!(git_command, base_message)
+  end
+
+  def validate_schema_on_migrate!
+    run("scripts/db_tasks db:migrate")
+    run("scripts/db_tasks db:schema:dump")
+
+    git_command = "git diff -- #{FILENAME}"
+    base_message = "the committed #{FILENAME} does not match the one generated by running added migrations"
+
+    validate_clean_output!(git_command, base_message)
+  end
+
+  def validate_schema_version_files!
+    git_command = "git add -A -n #{SCHEMA_VERSION_DIR}"
+    base_message = "the committed files in #{SCHEMA_VERSION_DIR} do not match those expected by the added migrations"
+
+    validate_clean_output!(git_command, base_message)
+  end
+
+  def committed_migrations
+    @committed_migrations ||= begin
+      git_command = "git diff --name-only --diff-filter=A #{diff_target} -- #{MIGRATION_DIRS.join(' ')}"
+
+      run(git_command).split("\n")
+    end
+  end
+
+  def diff_target
+    @diff_target ||= pipeline_for_merged_results? ? target_branch : merge_base
+  end
+
+  def merge_base
+    run("git merge-base #{target_branch} #{source_ref}")
+  end
+
+  def target_branch
+    ENV['CI_MERGE_REQUEST_TARGET_BRANCH_NAME'] || ENV['TARGET'] || ENV['CI_DEFAULT_BRANCH'] || 'master'
+  end
+
+  def source_ref
+    ENV['CI_COMMIT_SHA'] || 'HEAD'
+  end
+
+  def pipeline_for_merged_results?
+    ENV.key?('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA')
+  end
+
+  def find_migration_version(filename)
+    file_basename = File.basename(filename)
+    version_match = /\A(?<version>\d{#{VERSION_DIGITS}})_/o.match(file_basename)
+
+    die "#{filename} has an invalid migration version" if version_match.nil?
+
+    version_match[:version]
+  end
+
+  def validate_clean_output!(command, base_message)
+    command_output = run(command)
+
+    return if command_output.empty?
+
+    die "#{base_message}:\n#{command_output}"
+  end
+
+  def die(message, error_code: 1)
+    puts "\e[31mError: #{message}\e[0m"
+    exit error_code
+  end
+
+  def run(cmd)
+    puts "\e[32m$ #{cmd}\e[37m"
+    stdout_str, stderr_str, status = Open3.capture3(cmd)
+    puts "#{stdout_str}#{stderr_str}\e[0m"
+
+    die "command failed: #{stderr_str}" unless status.success?
+
+    stdout_str.chomp
+  end
+end
diff --git a/scripts/post_deployment_migrations_validator b/scripts/post_deployment_migrations_validator
new file mode 100755
index 0000000000000..3df2f772197bd
--- /dev/null
+++ b/scripts/post_deployment_migrations_validator
@@ -0,0 +1,31 @@
+#!/usr/bin/env ruby
+
+# frozen_string_literal: true
+
+require_relative 'migration_schema_validator'
+
+class PostDeploymentMigrationsValidator < MigrationSchemaValidator
+  def validate!
+    if committed_migrations.empty?
+      puts "\e[32m No migrations found, skipping post-deployment migrations validation\e[0m"
+      return
+    end
+
+    rollback_commited_migrations
+
+    run("SKIP_POST_DEPLOYMENT_MIGRATIONS=true scripts/db_tasks db:migrate")
+    run("scripts/db_tasks db:migrate")
+  end
+
+  private
+
+  def rollback_commited_migrations
+    committed_migrations.reverse_each do |filename|
+      version = find_migration_version(filename)
+
+      run("scripts/db_tasks db:migrate:down VERSION=#{version}")
+    end
+  end
+end
+
+PostDeploymentMigrationsValidator.new.validate!
diff --git a/scripts/validate_migration_schema b/scripts/validate_migration_schema
index 5c38985184403..c6f93b855ec12 100755
--- a/scripts/validate_migration_schema
+++ b/scripts/validate_migration_schema
@@ -2,120 +2,6 @@
 
 # frozen_string_literal: true
 
-require 'open3'
-
-class MigrationSchemaValidator
-  FILENAME = 'db/structure.sql'
-
-  MIGRATION_DIRS = %w[db/migrate db/post_migrate].freeze
-
-  SCHEMA_VERSION_DIR = 'db/schema_migrations'
-
-  VERSION_DIGITS = 14
-
-  def validate!
-    if committed_migrations.empty?
-      puts "\e[32m No migrations found, skipping schema validation\e[0m"
-      return
-    end
-
-    validate_schema_on_rollback!
-    validate_schema_on_migrate!
-    validate_schema_version_files!
-  end
-
-  private
-
-  def validate_schema_on_rollback!
-    committed_migrations.reverse_each do |filename|
-      version = find_migration_version(filename)
-
-      run("scripts/db_tasks db:migrate:down VERSION=#{version}")
-      run("scripts/db_tasks db:schema:dump")
-    end
-
-    git_command = "git diff #{diff_target} -- #{FILENAME}"
-    base_message = "rollback of added migrations does not revert #{FILENAME} to previous state"
-
-    validate_clean_output!(git_command, base_message)
-  end
-
-  def validate_schema_on_migrate!
-    run("scripts/db_tasks db:migrate")
-    run("scripts/db_tasks db:schema:dump")
-
-    git_command = "git diff -- #{FILENAME}"
-    base_message = "the committed #{FILENAME} does not match the one generated by running added migrations"
-
-    validate_clean_output!(git_command, base_message)
-  end
-
-  def validate_schema_version_files!
-    git_command = "git add -A -n #{SCHEMA_VERSION_DIR}"
-    base_message = "the committed files in #{SCHEMA_VERSION_DIR} do not match those expected by the added migrations"
-
-    validate_clean_output!(git_command, base_message)
-  end
-
-  def committed_migrations
-    @committed_migrations ||= begin
-      git_command = "git diff --name-only --diff-filter=A #{diff_target} -- #{MIGRATION_DIRS.join(' ')}"
-
-      run(git_command).split("\n")
-    end
-  end
-
-  def diff_target
-    @diff_target ||= pipeline_for_merged_results? ? target_branch : merge_base
-  end
-
-  def merge_base
-    run("git merge-base #{target_branch} #{source_ref}")
-  end
-
-  def target_branch
-    ENV['CI_MERGE_REQUEST_TARGET_BRANCH_NAME'] || ENV['TARGET'] || ENV['CI_DEFAULT_BRANCH'] || 'master'
-  end
-
-  def source_ref
-    ENV['CI_COMMIT_SHA'] || 'HEAD'
-  end
-
-  def pipeline_for_merged_results?
-    ENV.key?('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA')
-  end
-
-  def find_migration_version(filename)
-    file_basename = File.basename(filename)
-    version_match = /\A(?<version>\d{#{VERSION_DIGITS}})_/o.match(file_basename)
-
-    die "#{filename} has an invalid migration version" if version_match.nil?
-
-    version_match[:version]
-  end
-
-  def validate_clean_output!(command, base_message)
-    command_output = run(command)
-
-    return if command_output.empty?
-
-    die "#{base_message}:\n#{command_output}"
-  end
-
-  def die(message, error_code: 1)
-    puts "\e[31mError: #{message}\e[0m"
-    exit error_code
-  end
-
-  def run(cmd)
-    puts "\e[32m$ #{cmd}\e[37m"
-    stdout_str, stderr_str, status = Open3.capture3(cmd)
-    puts "#{stdout_str}#{stderr_str}\e[0m"
-
-    die "command failed: #{stderr_str}" unless status.success?
-
-    stdout_str.chomp
-  end
-end
+require_relative 'migration_schema_validator'
 
 MigrationSchemaValidator.new.validate!
-- 
GitLab