diff --git a/danger/ci_tables/Dangerfile b/danger/ci_tables/Dangerfile new file mode 100644 index 0000000000000000000000000000000000000000..1d4601d33b248b352e5f21348a41cb4ee860b235 --- /dev/null +++ b/danger/ci_tables/Dangerfile @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +SEE_DB_DOC = "See the [database dictionary documentation](https://docs.gitlab.com/ee/development/database/database_dictionary.html)." + +PARTITIONING_COMMENT = <<~SUGGEST_COMMENT +When adding new CI tables, consider [partitioning](https://docs.gitlab.com/ee/development/cicd/cicd_tables.html) the table +from the start if it references any of the larger CI tables: `ci_pipelines`, `ci_stages`, `ci_builds`, `p_ci_builds_metadata`, `ci_job_artifacts`, `ci_pipeline_variables`. +SUGGEST_COMMENT + +def check_database_dictionary_yaml(database_dictionary) + return unless database_dictionary.ci_schema? + # `p_` prefix is used by the partitioned tables, so we can assume that the table is already partitioned + return if database_dictionary.table_name.to_s.start_with?('p_') + + mr_line = database_dictionary.raw.lines.find_index { |line| line.start_with?('table_name:') } + return unless mr_line + + markdown(PARTITIONING_COMMENT, file: database_dictionary.path, line: mr_line.succ) +rescue Psych::Exception + # YAML could not be parsed, fail the build. + fail "#{helper.html_link(database_ditionary.path)} isn't valid YAML! #{SEE_DB_DOC}" # rubocop:disable Style/SignalException +rescue StandardError => e + warn "There was a problem trying to check the database dictionary file. Exception: #{e.class.name} - #{e.message}" +end + +def added_database_dictionary_files + database_dictionary.database_dictionary_files(change_type: :added) +end + +added_database_dictionary_files.each do |database_dictionary| + check_database_dictionary_yaml(database_dictionary) +end diff --git a/danger/plugins/database_dictionary.rb b/danger/plugins/database_dictionary.rb new file mode 100644 index 0000000000000000000000000000000000000000..b6a3c320891e9aedea05e39a52b4969d827d8d39 --- /dev/null +++ b/danger/plugins/database_dictionary.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +require_relative '../../tooling/danger/database_dictionary' + +module Danger + class DatabaseDictionary < Plugin + # Put the helper code somewhere it can be tested + include Tooling::Danger::DatabaseDictionary + end +end diff --git a/spec/tooling/danger/database_dictionary_spec.rb b/spec/tooling/danger/database_dictionary_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..1a771a6cec04f3d1bacdd6660a79d28515cfbc0f --- /dev/null +++ b/spec/tooling/danger/database_dictionary_spec.rb @@ -0,0 +1,152 @@ +# frozen_string_literal: true + +require 'gitlab-dangerfiles' +require 'gitlab/dangerfiles/spec_helper' + +require_relative '../../../tooling/danger/database_dictionary' + +RSpec.describe Tooling::Danger::DatabaseDictionary, feature_category: :shared do + include_context "with dangerfile" + + let(:fake_danger) { DangerSpecHelper.fake_danger.include(described_class) } + + subject(:database_dictionary) { fake_danger.new(helper: fake_helper) } + + describe '#database_dictionary_files' do + let(:database_dictionary_files) do + [ + 'db/docs/ci_pipelines.yml', + 'db/docs/projects.yml' + ] + end + + let(:other_files) do + [ + 'app/models/model.rb', + 'app/assets/javascripts/file.js' + ] + end + + shared_examples 'an array of Found objects' do |change_type| + it 'returns an array of Found objects' do + expect(database_dictionary.database_dictionary_files(change_type: change_type)) + .to contain_exactly( + an_instance_of(described_class::Found), + an_instance_of(described_class::Found) + ) + + expect(database_dictionary.database_dictionary_files(change_type: change_type).map(&:path)) + .to eq(database_dictionary_files) + end + end + + shared_examples 'an empty array' do |change_type| + it 'returns an array of Found objects' do + expect(database_dictionary.database_dictionary_files(change_type: change_type)).to be_empty + end + end + + describe 'retrieves added database dictionary files' do + context 'with added added database dictionary files' do + let(:added_files) { database_dictionary_files } + + include_examples 'an array of Found objects', :added + end + + context 'without added added database dictionary files' do + let(:added_files) { other_files } + + include_examples 'an empty array', :added + end + end + + describe 'retrieves modified database dictionary files' do + context 'with modified modified database dictionary files' do + let(:modified_files) { database_dictionary_files } + + include_examples 'an array of Found objects', :modified + end + + context 'without modified modified database dictionary files' do + let(:modified_files) { other_files } + + include_examples 'an empty array', :modified + end + end + + describe 'retrieves deleted database dictionary files' do + context 'with deleted deleted database dictionary files' do + let(:deleted_files) { database_dictionary_files } + + include_examples 'an array of Found objects', :deleted + end + + context 'without deleted deleted database dictionary files' do + let(:deleted_files) { other_files } + + include_examples 'an empty array', :deleted + end + end + end + + describe described_class::Found do + let(:database_dictionary_path) { 'db/docs/ci_pipelines.yml' } + let(:gitlab_schema) { 'gitlab_ci' } + + let(:yaml) do + { + + 'table_name' => 'ci_pipelines', + 'classes' => ['Ci::Pipeline'], + 'feature_categories' => ['continuous_integration'], + 'description' => 'TODO', + 'introduced_by_url' => 'https://gitlab.com/gitlab-org/gitlab/-/commit/c6ae290cea4b88ecaa9cfe0bc9d88e8fd32070c1', + 'milestone' => '9.0', + 'gitlab_schema' => gitlab_schema + } + end + + let(:raw_yaml) { YAML.dump(yaml) } + + subject(:found) { described_class.new(database_dictionary_path) } + + before do + allow(File).to receive(:read).and_call_original + allow(File).to receive(:read).with(database_dictionary_path).and_return(raw_yaml) + end + + described_class::ATTRIBUTES.each do |attribute| + describe "##{attribute}" do + it 'returns value from the YAML' do + expect(found.public_send(attribute)).to eq(yaml[attribute]) + end + end + end + + describe '#raw' do + it 'returns the raw YAML' do + expect(found.raw).to eq(raw_yaml) + end + end + + describe '#ci_schema?' do + it { expect(found.ci_schema?).to be_truthy } + + context 'with main schema' do + let(:gitlab_schema) { 'gitlab_main' } + + it { expect(found.ci_schema?).to be_falsey } + end + end + + describe '#main_schema?' do + it { expect(found.main_schema?).to be_falsey } + + context 'with main schema' do + let(:gitlab_schema) { 'gitlab_main' } + + it { expect(found.main_schema?).to be_truthy } + end + end + end +end diff --git a/tooling/danger/database_dictionary.rb b/tooling/danger/database_dictionary.rb new file mode 100644 index 0000000000000000000000000000000000000000..8776532ff84c0314dff184b19ba9ad2029016907 --- /dev/null +++ b/tooling/danger/database_dictionary.rb @@ -0,0 +1,57 @@ +# frozen_string_literal: true + +require 'yaml' + +module Tooling + module Danger + module DatabaseDictionary + DICTIONARY_PATH_REGEXP = %r{db/docs/.*\.yml} + + # `change_type` can be: + # - :added + # - :modified + # - :deleted + def database_dictionary_files(change_type:) + files = helper.public_send("#{change_type}_files") # rubocop:disable GitlabSecurity/PublicSend + + files.filter_map { |path| Found.new(path) if path =~ DICTIONARY_PATH_REGEXP } + end + + class Found + ATTRIBUTES = %w[ + table_name classes feature_categories description introduced_by_url milestone gitlab_schema + ].freeze + + attr_reader :path + + def initialize(path) + @path = path + end + + ATTRIBUTES.each do |attribute| + define_method(attribute) do + yaml[attribute] + end + end + + def raw + @raw ||= File.read(path) + end + + def ci_schema? + gitlab_schema == 'gitlab_ci' + end + + def main_schema? + gitlab_schema == 'gitlab_main' + end + + private + + def yaml + @yaml ||= YAML.safe_load(raw) + end + end + end + end +end