diff --git a/danger/ignored_model_columns/Dangerfile b/danger/ignored_model_columns/Dangerfile new file mode 100644 index 0000000000000000000000000000000000000000..e11566f90f2a6a4d19c90798b09b826d566c7547 --- /dev/null +++ b/danger/ignored_model_columns/Dangerfile @@ -0,0 +1,3 @@ +# frozen_string_literal: true + +ignored_model_columns.add_comment_for_ignored_model_columns diff --git a/danger/plugins/ignored_model_columns.rb b/danger/plugins/ignored_model_columns.rb new file mode 100644 index 0000000000000000000000000000000000000000..8e0024a94d642515dc061d53d5b01e7a052f4a8d --- /dev/null +++ b/danger/plugins/ignored_model_columns.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +require_relative '../../tooling/danger/ignored_model_columns' + +module Danger + class IgnoredModelColumns < ::Danger::Plugin + include Tooling::Danger::IgnoredModelColumns + end +end diff --git a/spec/tooling/danger/ignored_model_columns_spec.rb b/spec/tooling/danger/ignored_model_columns_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..737b6cce0778c9f76a0b9c864584e065d9c87005 --- /dev/null +++ b/spec/tooling/danger/ignored_model_columns_spec.rb @@ -0,0 +1,145 @@ +# frozen_string_literal: true + +require 'danger' +require 'gitlab/dangerfiles/spec_helper' + +require_relative '../../../tooling/danger/ignored_model_columns' +require_relative '../../../tooling/danger/project_helper' + +RSpec.describe Tooling::Danger::IgnoredModelColumns, feature_category: :tooling do + subject(:ignored_model_columns) { fake_danger.new(helper: fake_helper) } + + let(:fake_danger) { DangerSpecHelper.fake_danger.include(described_class) } + let(:fake_project_helper) { instance_double(Tooling::Danger::ProjectHelper) } + let(:comment) { described_class::COMMENT.chomp } + let(:file_diff) do + File.read(File.expand_path("../fixtures/#{fixture}", __dir__)).split("\n") + end + + include_context "with dangerfile" + + describe '#add_comment_for_ignored_model_columns' do + let(:file_lines) { file_diff.map { |line| line.delete_prefix('+').delete_prefix('-') } } + + before do + allow(ignored_model_columns).to receive(:project_helper).and_return(fake_project_helper) + allow(ignored_model_columns.project_helper).to receive(:file_lines).and_return(file_lines) + allow(ignored_model_columns.helper).to receive(:all_changed_files).and_return([filename]) + allow(ignored_model_columns.helper).to receive(:changed_lines).with(filename).and_return(file_diff) + end + + context 'when table column is renamed in a regular migration' do + let(:filename) { 'db/migrate/rename_my_column_migration.rb' } + let(:fixture) { 'rename_column_migration.txt' } + let(:matching_lines) { [7, 11, 15, 19, 23, 27, 31, 35, 39] } + + it 'adds comment at the correct line' do + matching_lines.each do |line_number| + expect(ignored_model_columns).to receive(:markdown).with("\n#{comment}", file: filename, line: line_number) + end + + ignored_model_columns.add_comment_for_ignored_model_columns + end + end + + context 'when table column is renamed in a post migration' do + let(:filename) { 'db/post_migrate/remove_column_migration.rb' } + let(:fixture) { 'remove_column_migration.txt' } + let(:matching_lines) { [7, 8, 16, 24, 32, 40, 48, 56, 64, 72] } + + it 'adds comment at the correct line' do + matching_lines.each do |line_number| + expect(ignored_model_columns).to receive(:markdown).with("\n#{comment}", file: filename, line: line_number) + end + + ignored_model_columns.add_comment_for_ignored_model_columns + end + end + + context 'when table cleanup is performed in a post migration' do + let(:filename) { 'db/post_migrate/cleanup_conversion_big_int_migration.rb' } + let(:fixture) { 'cleanup_conversion_migration.txt' } + let(:matching_lines) { [7, 11, 15, 19, 23, 27, 31, 35, 39] } + + it 'adds comment at the correct line' do + matching_lines.each do |line_number| + expect(ignored_model_columns).to receive(:markdown).with("\n#{comment}", file: filename, line: line_number) + end + + ignored_model_columns.add_comment_for_ignored_model_columns + end + end + + context 'when a regular migration does not rename table column' do + let(:filename) { 'db/migrate/my_migration.rb' } + let(:file_diff) do + [ + "+ undo_cleanup_concurrent_column_rename(:my_table, :old_column, :new_column)", + "- cleanup_concurrent_column_rename(:my_table, :new_column, :old_column)" + ] + end + + let(:file_lines) do + [ + ' def up', + ' undo_cleanup_concurrent_column_rename(:my_table, :old_column, :new_column)', + ' end' + ] + end + + it 'does not add comment' do + expect(ignored_model_columns).not_to receive(:markdown) + + ignored_model_columns.add_comment_for_ignored_model_columns + end + end + + context 'when a post migration does not remove table column' do + let(:filename) { 'db/migrate/my_migration.rb' } + let(:file_diff) do + [ + "+ add_column(:my_table, :my_column, :type)", + "- remove_column(:my_table, :my_column)" + ] + end + + let(:file_lines) do + [ + ' def up', + ' add_column(:my_table, :my_column, :type)', + ' end' + ] + end + + it 'does not add comment' do + expect(ignored_model_columns).not_to receive(:markdown) + + ignored_model_columns.add_comment_for_ignored_model_columns + end + end + + context 'when a post migration does not convert table column' do + let(:filename) { 'db/migrate/my_migration.rb' } + let(:file_diff) do + [ + "+ restore_conversion_of_integer_to_bigint(TABLE, COLUMNS)", + "- cleanup_conversion_of_integer_to_bigint(TABLE, COLUMNS)" + ] + end + + let(:file_lines) do + [ + ' def up', + ' cleanup_conversion_of_integer_to_bigint(TABLE, COLUMNS)', + ' end' + ] + end + + it 'does not add comment' do + expect(ignored_model_columns).not_to receive(:markdown) + + ignored_model_columns.add_comment_for_ignored_model_columns + end + end + end +end diff --git a/spec/tooling/fixtures/cleanup_conversion_migration.txt b/spec/tooling/fixtures/cleanup_conversion_migration.txt new file mode 100644 index 0000000000000000000000000000000000000000..14a7937b4695b8913989da3305b669591c899985 --- /dev/null +++ b/spec/tooling/fixtures/cleanup_conversion_migration.txt @@ -0,0 +1,44 @@ ++# frozen_string_literal: true ++ ++class TestMigration < Gitlab::Database::Migration[2.1] ++ disable_ddl_transaction! ++ ++ def up ++ cleanup_conversion_of_integer_to_bigint :my_table, :my_column ++ end ++ ++ def up ++ cleanup_conversion_of_integer_to_bigint 'my_table', 'my_column' ++ end ++ ++ def up ++ cleanup_conversion_of_integer_to_bigint "my_table", "my_column", "new_column" ++ end ++ ++ def up ++ cleanup_conversion_of_integer_to_bigint TABLE_NAME, MY_COLUMN ++ end ++ ++ def up ++ cleanup_conversion_of_integer_to_bigint(:my_table, :my_column) ++ end ++ ++ def up ++ cleanup_conversion_of_integer_to_bigint('my_table', 'my_column') ++ end ++ ++ def up ++ cleanup_conversion_of_integer_to_bigint("my_table", "my_column") ++ end ++ ++ def up ++ cleanup_conversion_of_integer_to_bigint(TABLE_NAME, MY_COLUMN) ++ end ++ ++ def up ++ cleanup_conversion_of_integer_to_bigint( ++ :my_table, ++ :my_column ++ ) ++ end ++end diff --git a/spec/tooling/fixtures/remove_column_migration.txt b/spec/tooling/fixtures/remove_column_migration.txt new file mode 100644 index 0000000000000000000000000000000000000000..885f0060d920fa628d8729dea313cf63d5c1ae4d --- /dev/null +++ b/spec/tooling/fixtures/remove_column_migration.txt @@ -0,0 +1,84 @@ ++# frozen_string_literal: true ++ ++class TestMigration < Gitlab::Database::Migration[2.1] ++ disable_ddl_transaction! ++ ++ def up ++ remove_column :my_table, :my_column ++ remove_column :my_other_table, :my_column ++ end ++ ++ def down ++ remove_column :my_table, :my_column ++ end ++ ++ def up ++ remove_column 'my_table', 'my_column' ++ end ++ ++ def down ++ remove_column 'my_table', 'my_column' ++ end ++ ++ def up ++ remove_column "my_table", "my_column", "new_column" ++ end ++ ++ def down ++ remove_column "my_table", "my_column", "new_column" ++ end ++ ++ def up ++ remove_column TABLE_NAME, MY_COLUMN ++ end ++ ++ def down ++ remove_column TABLE_NAME, MY_COLUMN ++ end ++ ++ def up ++ remove_column(:my_table, :my_column) ++ end ++ ++ def down ++ remove_column(:my_table, :my_column) ++ end ++ ++ def up ++ remove_column('my_table', 'my_column') ++ end ++ ++ def down ++ remove_column('my_table', 'my_column') ++ end ++ ++ def up ++ remove_column("my_table", "my_column") ++ end ++ ++ def down ++ remove_column("my_table", "my_column") ++ end ++ ++ def up ++ remove_column(TABLE_NAME, MY_COLUMN) ++ end ++ ++ def down ++ remove_column(TABLE_NAME, MY_COLUMN) ++ end ++ ++ def up ++ remove_column( ++ :my_table, ++ :my_column ++ ) ++ end ++ ++ def down ++ remove_column( ++ :my_table, ++ :my_column ++ ) ++ end ++end diff --git a/spec/tooling/fixtures/rename_column_migration.txt b/spec/tooling/fixtures/rename_column_migration.txt new file mode 100644 index 0000000000000000000000000000000000000000..e79029219a53c65af42aaffc44563d853cdbc060 --- /dev/null +++ b/spec/tooling/fixtures/rename_column_migration.txt @@ -0,0 +1,45 @@ ++# frozen_string_literal: true ++ ++class TestMigration < Gitlab::Database::Migration[2.1] ++ disable_ddl_transaction! ++ ++ def up ++ cleanup_concurrent_column_rename :my_table, :old_column, :new_column ++ end ++ ++ def up ++ cleanup_concurrent_column_rename 'my_table', 'old_column', 'new_column' ++ end ++ ++ def up ++ cleanup_concurrent_column_rename "my_table", "old_column", "new_column" ++ end ++ ++ def up ++ cleanup_concurrent_column_rename TABLE_NAME, OLD_COLUMN_NAME, NEW_COLUMN_NAME ++ end ++ ++ def up ++ cleanup_concurrent_column_rename(:my_table, :old_column, :new_column) ++ end ++ ++ def up ++ cleanup_concurrent_column_rename('my_table', 'old_column', 'new_column') ++ end ++ ++ def up ++ cleanup_concurrent_column_rename("my_table", "old_column", "new_column") ++ end ++ ++ def up ++ cleanup_concurrent_column_rename(TABLE_NAME, OLD_COLUMN_NAME, NEW_COLUMN_NAME) ++ end ++ ++ def up ++ cleanup_concurrent_column_rename( ++ :my_table, ++ :old_column, ++ :new_column ++ ) ++ end ++end diff --git a/tooling/danger/ignored_model_columns.rb b/tooling/danger/ignored_model_columns.rb new file mode 100644 index 0000000000000000000000000000000000000000..50379eed85ccf2842e728c009ce05e3787614ced --- /dev/null +++ b/tooling/danger/ignored_model_columns.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +require_relative 'suggestor' + +module Tooling + module Danger + module IgnoredModelColumns + include ::Tooling::Danger::Suggestor + + METHODS = %w[remove_column cleanup_concurrent_column_rename cleanup_conversion_of_integer_to_bigint].freeze + MIGRATION_FILES_REGEX = %r{^db/(post_)?migrate} + MIGRATION_METHODS_REGEX = /^\+\s*(.*\.)?(#{METHODS.join('|')})[(\s]/ + UP_METHOD_REGEX = /^.+(def\sup)/ + END_METHOD_REGEX = /^.+(end)/ + DOC_URL = "https://docs.gitlab.com/ee/development/database/avoiding_downtime_in_migrations.html" + + COMMENT = <<~COMMENT.freeze + Column operations, like [dropping](#{DOC_URL}#dropping-columns), [renaming](#{DOC_URL}#renaming-columns) or + [primary key conversion](#{DOC_URL}#migrating-integer-primary-keys-to-bigint), require columns to be ignored in + the model. This step is necessary because Rails caches the columns and re-uses it in various places across the + application. Please ensure that columns are properly ignored in the model. + COMMENT + + def add_comment_for_ignored_model_columns + migration_files.each do |filename| + add_suggestion(filename: filename, regex: MIGRATION_METHODS_REGEX, comment_text: COMMENT) + end + end + + private + + # This method was overwritten to make use of +up_method_lines+. + # It's necessary to only match lines that are inside the +up+ block in a migration file. + # + # @return [Integer, Nil] the line number - only if the line is from within a +up+ method + def find_line_number(file_lines, searched_line, exclude_indexes: []) + lines_to_search = up_method_lines(file_lines) + + _, index = file_lines.each_with_index.find do |file_line, index| + next unless lines_to_search.include?(index) + + file_line == searched_line && !exclude_indexes.include?(index) # rubocop:disable Rails/NegateInclude + end + + index + end + + # Return the line numbers from within the up method + # + # @example + # line 0 def up + # line 1 cleanup_conversion_of_integer_to_bigint():my_table, :my_column) + # line 2 remove_column(:my_table, :my_column) + # line 3 other_method + # line 4 end + # + # => [1, 2, 3] + def up_method_lines(file_lines) + capture_up_block = false + up_method_content_lines = [] + + file_lines.each_with_index do |content, line_number| + capture_up_block = false if capture_up_block && END_METHOD_REGEX.match?(content) + up_method_content_lines << line_number if capture_up_block + capture_up_block = true if UP_METHOD_REGEX.match?(content) + end + + up_method_content_lines + end + + def migration_files + helper.all_changed_files.grep(MIGRATION_FILES_REGEX) + end + end + end +end