Adds a new Danger check for renamed/dropped columns

This check detects the use of `remove_column` or
`cleanup_concurrent_column_rename` and adds a
comment to the changed lines, indicating the need
for ignoring the columns in the proper model

Changelog: changed

danger/ignored_model_columns/Dangerfile

+# frozen_string_literal: true
+
+ignored_model_columns.add_comment_for_ignored_model_columns

danger/plugins/ignored_model_columns.rb

+# frozen_string_literal: true
+
+require_relative '../../tooling/danger/ignored_model_columns'
+
+module Danger
+  class IgnoredModelColumns < ::Danger::Plugin
+    include Tooling::Danger::IgnoredModelColumns
+  end
+end

spec/tooling/danger/ignored_model_columns_spec.rb

+# frozen_string_literal: true
+
+require 'danger'
+require 'gitlab/dangerfiles/spec_helper'
+
+require_relative '../../../tooling/danger/ignored_model_columns'
+require_relative '../../../tooling/danger/project_helper'
+
+RSpec.describe Tooling::Danger::IgnoredModelColumns, feature_category: :tooling do
+  subject(:ignored_model_columns) { fake_danger.new(helper: fake_helper) }
+
+  let(:fake_danger) { DangerSpecHelper.fake_danger.include(described_class) }
+  let(:fake_project_helper) { instance_double(Tooling::Danger::ProjectHelper) }
+  let(:comment) { described_class::COMMENT.chomp }
+  let(:file_diff) do
+    File.read(File.expand_path("../fixtures/#{fixture}", __dir__)).split("\n")
+  end
+
+  include_context "with dangerfile"
+
+  describe '#add_comment_for_ignored_model_columns' do
+    let(:file_lines) { file_diff.map { |line| line.delete_prefix('+').delete_prefix('-') } }
+
+    before do
+      allow(ignored_model_columns).to receive(:project_helper).and_return(fake_project_helper)
+      allow(ignored_model_columns.project_helper).to receive(:file_lines).and_return(file_lines)
+      allow(ignored_model_columns.helper).to receive(:all_changed_files).and_return([filename])
+      allow(ignored_model_columns.helper).to receive(:changed_lines).with(filename).and_return(file_diff)
+    end
+
+    context 'when table column is renamed in a regular migration' do
+      let(:filename) { 'db/migrate/rename_my_column_migration.rb' }
+      let(:fixture) { 'rename_column_migration.txt' }
+      let(:matching_lines) { [7, 11, 15, 19, 23, 27, 31, 35, 39] }
+
+      it 'adds comment at the correct line' do
+        matching_lines.each do |line_number|
+          expect(ignored_model_columns).to receive(:markdown).with("\n#{comment}", file: filename, line: line_number)
+        end
+
+        ignored_model_columns.add_comment_for_ignored_model_columns
+      end
+    end
+
+    context 'when table column is renamed in a post migration' do
+      let(:filename) { 'db/post_migrate/remove_column_migration.rb' }
+      let(:fixture) { 'remove_column_migration.txt' }
+      let(:matching_lines) { [7, 8, 16, 24, 32, 40, 48, 56, 64, 72] }
+
+      it 'adds comment at the correct line' do
+        matching_lines.each do |line_number|
+          expect(ignored_model_columns).to receive(:markdown).with("\n#{comment}", file: filename, line: line_number)
+        end
+
+        ignored_model_columns.add_comment_for_ignored_model_columns
+      end
+    end
+
+    context 'when table cleanup is performed in a post migration' do
+      let(:filename) { 'db/post_migrate/cleanup_conversion_big_int_migration.rb' }
+      let(:fixture) { 'cleanup_conversion_migration.txt' }
+      let(:matching_lines) { [7, 11, 15, 19, 23, 27, 31, 35, 39] }
+
+      it 'adds comment at the correct line' do
+        matching_lines.each do |line_number|
+          expect(ignored_model_columns).to receive(:markdown).with("\n#{comment}", file: filename, line: line_number)
+        end
+
+        ignored_model_columns.add_comment_for_ignored_model_columns
+      end
+    end
+
+    context 'when a regular migration does not rename table column' do
+      let(:filename) { 'db/migrate/my_migration.rb' }
+      let(:file_diff) do
+        [
+          "+    undo_cleanup_concurrent_column_rename(:my_table, :old_column, :new_column)",
+          "-    cleanup_concurrent_column_rename(:my_table, :new_column, :old_column)"
+        ]
+      end
+
+      let(:file_lines) do
+        [
+          '  def up',
+          '    undo_cleanup_concurrent_column_rename(:my_table, :old_column, :new_column)',
+          '  end'
+        ]
+      end
+
+      it 'does not add comment' do
+        expect(ignored_model_columns).not_to receive(:markdown)
+
+        ignored_model_columns.add_comment_for_ignored_model_columns
+      end
+    end
+
+    context 'when a post migration does not remove table column' do
+      let(:filename) { 'db/migrate/my_migration.rb' }
+      let(:file_diff) do
+        [
+          "+    add_column(:my_table, :my_column, :type)",
+          "-    remove_column(:my_table, :my_column)"
+        ]
+      end
+
+      let(:file_lines) do
+        [
+          '  def up',
+          '    add_column(:my_table, :my_column, :type)',
+          '  end'
+        ]
+      end
+
+      it 'does not add comment' do
+        expect(ignored_model_columns).not_to receive(:markdown)
+
+        ignored_model_columns.add_comment_for_ignored_model_columns
+      end
+    end
+
+    context 'when a post migration does not convert table column' do
+      let(:filename) { 'db/migrate/my_migration.rb' }
+      let(:file_diff) do
+        [
+          "+    restore_conversion_of_integer_to_bigint(TABLE, COLUMNS)",
+          "-    cleanup_conversion_of_integer_to_bigint(TABLE, COLUMNS)"
+        ]
+      end
+
+      let(:file_lines) do
+        [
+          '  def up',
+          '    cleanup_conversion_of_integer_to_bigint(TABLE, COLUMNS)',
+          '  end'
+        ]
+      end
+
+      it 'does not add comment' do
+        expect(ignored_model_columns).not_to receive(:markdown)
+
+        ignored_model_columns.add_comment_for_ignored_model_columns
+      end
+    end
+  end
+end

spec/tooling/fixtures/cleanup_conversion_migration.txt

++# frozen_string_literal: true
++
++class TestMigration < Gitlab::Database::Migration[2.1]
++  disable_ddl_transaction!
++
++  def up
++    cleanup_conversion_of_integer_to_bigint :my_table, :my_column
++  end
++
++  def up
++    cleanup_conversion_of_integer_to_bigint 'my_table', 'my_column'
++  end
++
++  def up
++    cleanup_conversion_of_integer_to_bigint "my_table", "my_column", "new_column"
++  end
++
++  def up
++    cleanup_conversion_of_integer_to_bigint TABLE_NAME, MY_COLUMN
++  end
++
++  def up
++    cleanup_conversion_of_integer_to_bigint(:my_table, :my_column)
++  end
++
++  def up
++    cleanup_conversion_of_integer_to_bigint('my_table', 'my_column')
++  end
++
++  def up
++    cleanup_conversion_of_integer_to_bigint("my_table", "my_column")
++  end
++
++  def up
++    cleanup_conversion_of_integer_to_bigint(TABLE_NAME, MY_COLUMN)
++  end
++
++  def up
++    cleanup_conversion_of_integer_to_bigint(
++      :my_table,
++      :my_column
++    )
++  end
++end

spec/tooling/fixtures/remove_column_migration.txt

++# frozen_string_literal: true
++
++class TestMigration < Gitlab::Database::Migration[2.1]
++  disable_ddl_transaction!
++
++  def up
++    remove_column :my_table, :my_column
++    remove_column :my_other_table, :my_column
++  end
++
++  def down
++    remove_column :my_table, :my_column
++  end
++
++  def up
++    remove_column 'my_table', 'my_column'
++  end
++
++  def down
++    remove_column 'my_table', 'my_column'
++  end
++
++  def up
++    remove_column "my_table", "my_column", "new_column"
++  end
++
++  def down
++    remove_column "my_table", "my_column", "new_column"
++  end
++
++  def up
++    remove_column TABLE_NAME, MY_COLUMN
++  end
++
++  def down
++    remove_column TABLE_NAME, MY_COLUMN
++  end
++
++  def up
++    remove_column(:my_table, :my_column)
++  end
++
++  def down
++    remove_column(:my_table, :my_column)
++  end
++
++  def up
++    remove_column('my_table', 'my_column')
++  end
++
++  def down
++    remove_column('my_table', 'my_column')
++  end
++
++  def up
++    remove_column("my_table", "my_column")
++  end
++
++  def down
++    remove_column("my_table", "my_column")
++  end
++
++  def up
++    remove_column(TABLE_NAME, MY_COLUMN)
++  end
++
++  def down
++    remove_column(TABLE_NAME, MY_COLUMN)
++  end
++
++  def up
++    remove_column(
++      :my_table,
++      :my_column
++    )
++  end
++
++  def down
++    remove_column(
++      :my_table,
++      :my_column
++    )
++  end
++end

spec/tooling/fixtures/rename_column_migration.txt

++# frozen_string_literal: true
++
++class TestMigration < Gitlab::Database::Migration[2.1]
++  disable_ddl_transaction!
++
++  def up
++    cleanup_concurrent_column_rename :my_table, :old_column, :new_column
++  end
++
++  def up
++    cleanup_concurrent_column_rename 'my_table', 'old_column', 'new_column'
++  end
++
++  def up
++    cleanup_concurrent_column_rename "my_table", "old_column", "new_column"
++  end
++
++  def up
++    cleanup_concurrent_column_rename TABLE_NAME, OLD_COLUMN_NAME, NEW_COLUMN_NAME
++  end
++
++  def up
++    cleanup_concurrent_column_rename(:my_table, :old_column, :new_column)
++  end
++
++  def up
++    cleanup_concurrent_column_rename('my_table', 'old_column', 'new_column')
++  end
++
++  def up
++    cleanup_concurrent_column_rename("my_table", "old_column", "new_column")
++  end
++
++  def up
++    cleanup_concurrent_column_rename(TABLE_NAME, OLD_COLUMN_NAME, NEW_COLUMN_NAME)
++  end
++
++  def up
++    cleanup_concurrent_column_rename(
++      :my_table,
++      :old_column,
++      :new_column
++    )
++  end
++end

tooling/danger/ignored_model_columns.rb

+# frozen_string_literal: true
+
+require_relative 'suggestor'
+
+module Tooling
+  module Danger
+    module IgnoredModelColumns
+      include ::Tooling::Danger::Suggestor
+
+      METHODS = %w[remove_column cleanup_concurrent_column_rename cleanup_conversion_of_integer_to_bigint].freeze
+      MIGRATION_FILES_REGEX = %r{^db/(post_)?migrate}
+      MIGRATION_METHODS_REGEX = /^\+\s*(.*\.)?(#{METHODS.join('|')})[(\s]/
+      UP_METHOD_REGEX = /^.+(def\sup)/
+      END_METHOD_REGEX = /^.+(end)/
+      DOC_URL = "https://docs.gitlab.com/ee/development/database/avoiding_downtime_in_migrations.html"
+
+      COMMENT = <<~COMMENT.freeze
+        Column operations, like [dropping](#{DOC_URL}#dropping-columns), [renaming](#{DOC_URL}#renaming-columns) or
+        [primary key conversion](#{DOC_URL}#migrating-integer-primary-keys-to-bigint), require columns to be ignored in
+        the model. This step is necessary because Rails caches the columns and re-uses it in various places across the
+        application. Please ensure that columns are properly ignored in the model.
+      COMMENT
+
+      def add_comment_for_ignored_model_columns
+        migration_files.each do |filename|
+          add_suggestion(filename: filename, regex: MIGRATION_METHODS_REGEX, comment_text: COMMENT)
+        end
+      end
+
+      private
+
+      # This method was overwritten to make use of +up_method_lines+.
+      # It's necessary to only match lines that are inside the +up+ block in a migration file.
+      #
+      # @return [Integer, Nil] the line number - only if the line is from within a +up+ method
+      def find_line_number(file_lines, searched_line, exclude_indexes: [])
+        lines_to_search = up_method_lines(file_lines)
+
+        _, index = file_lines.each_with_index.find do |file_line, index|
+          next unless lines_to_search.include?(index)
+
+          file_line == searched_line && !exclude_indexes.include?(index) # rubocop:disable Rails/NegateInclude
+        end
+
+        index
+      end
+
+      # Return the line numbers from within the up method
+      #
+      # @example
+      #         line 0 def up
+      #         line 1   cleanup_conversion_of_integer_to_bigint():my_table, :my_column)
+      #         line 2   remove_column(:my_table, :my_column)
+      #         line 3   other_method
+      #         line 4 end
+      #
+      #         => [1, 2, 3]
+      def up_method_lines(file_lines)
+        capture_up_block = false
+        up_method_content_lines = []
+
+        file_lines.each_with_index do |content, line_number|
+          capture_up_block = false if capture_up_block && END_METHOD_REGEX.match?(content)
+          up_method_content_lines << line_number if capture_up_block
+          capture_up_block = true if UP_METHOD_REGEX.match?(content)
+        end
+
+        up_method_content_lines
+      end
+
+      def migration_files
+        helper.all_changed_files.grep(MIGRATION_FILES_REGEX)
+      end
+    end
+  end
+end