diff --git a/keeps/helpers/groups.rb b/keeps/helpers/groups.rb index 2cbf2bc295757f6fe307108cd0315c7165cd5001..68faf0a37a21133b8535045e941597e1dfc58f08 100644 --- a/keeps/helpers/groups.rb +++ b/keeps/helpers/groups.rb @@ -19,6 +19,8 @@ def group_for_group_label(group_label) end def pick_reviewer(group, identifiers) + return if group['backend_engineers'].empty? + random_engineer = Digest::SHA256.hexdigest(identifiers.join).to_i(16) % group['backend_engineers'].size group['backend_engineers'][random_engineer] diff --git a/keeps/quarantine_flaky_tests.rb b/keeps/quarantine_flaky_tests.rb new file mode 100644 index 0000000000000000000000000000000000000000..b3cbe227b466a1ec330cdfc364e84110de755ae4 --- /dev/null +++ b/keeps/quarantine_flaky_tests.rb @@ -0,0 +1,113 @@ +# frozen_string_literal: true + +require 'gitlab-http' + +require_relative 'helpers/groups' + +module Keeps + # This is an implementation of a ::Gitlab::Housekeeper::Keep. This keep will fetch any `failure::flaky-test` issues + # with more than 1000 reports and quarantine these tests. + # + # You can run it individually with: + # + # ``` + # bundle exec gitlab-housekeeper -d \ + # -k Keeps::QuarantineFlakyTests + # ``` + class QuarantineFlakyTests < ::Gitlab::Housekeeper::Keep + MINIMUM_FLAKINESS_OCCURENCES = 1000 + FLAKY_TEST_ISSUES_URL = "https://gitlab.com/api/v4/projects/gitlab-org%2Fgitlab/issues/?order_by=updated_at&state=opened&labels%5B%5D=test&labels%5B%5D=failure%3A%3Aflaky-test¬%5Blabels%5D%5B%5D=QA¬%5Blabel_name%5D%5B%5D=quarantine&per_page=10" + FLAKY_TEST_ISSUE_NOTES_URL = "https://gitlab.com/api/v4/projects/gitlab-org%%2Fgitlab/issues/%<issue_iid>s/notes" + EXAMPLE_LINE_REGEX = /([\w'",])? do$/ + + def each_change + each_very_flaky_issue do |flaky_issue| + change = prepare_change(flaky_issue) + + yield(change) if change + end + end + + private + + def groups_helper + @groups_helper ||= ::Keeps::Helpers::Groups.new + end + + def prepare_change(flaky_issue) + match = flaky_issue['description'].match(%r{\| File URL \| \[`(?<filename>[\w\/\.]+)#L(?<line_number>\d+)`\]}) + return unless match + + filename = match[:filename] + line_number = match[:line_number].to_i + + match = flaky_issue['description'].match(%r{\| Description \| (?<description>.+) \|}) + return unless match + + description = match[:description] + + file = File.expand_path("../#{filename}", __dir__) + full_file_content = File.read(file) + issue_url = flaky_issue['web_url'] + + file_lines = full_file_content.lines + return unless file_lines[line_number - 1].match?(EXAMPLE_LINE_REGEX) + + file_lines[line_number - 1].sub!(EXAMPLE_LINE_REGEX, "\\1, quarantine: '#{issue_url}' do") + File.write(file, file_lines.join) + + change = ::Gitlab::Housekeeper::Change.new + change.title = "Quarantine a flaky test" + change.identifiers = [self.class.name.demodulize, filename, line_number.to_s] + change.changed_files = [filename] + change.description = <<~MARKDOWN + The #{description} test has been reported as flaky more then #{MINIMUM_FLAKINESS_OCCURENCES} times. + + This MR quarantines the test. This is a discussion starting point to let the responsible group know about the flakiness + so that they can take action: + + - accept the merge request and schedule to improve the test + - close the merge request in favor of another merge request to delete the test + + Relate to #{issue_url}. + MARKDOWN + + group_label = flaky_issue['labels'].grep(/group::/).first + change.labels = [ + 'maintenance::refactor', + 'test', + 'failure::flaky-test', + 'pipeline:expedite', + 'quarantine', + 'quarantine::flaky', + group_label + ].compact + + if change.reviewers.empty? && group_label + group_data = groups_helper.group_for_group_label(group_label) + + change.reviewers = groups_helper.pick_reviewer(group_data, change.identifiers) if group_data + end + + change + end + + def each_very_flaky_issue + flaky_test_issues = Gitlab::HTTP.get(FLAKY_TEST_ISSUES_URL) + + flaky_test_issues_above_threshold = flaky_test_issues.select do |flaky_test_issue| + Gitlab::HTTP.get(format(FLAKY_TEST_ISSUE_NOTES_URL, { issue_iid: flaky_test_issue['iid'] }), + headers: { 'PRIVATE-TOKEN': ENV['HOUSEKEEPER_GITLAB_API_TOKEN'] }).find do |note| + match = note['body'].match(/### Flakiness reports \((?<reports_count>\d+)\)/) + next unless match + + match[:reports_count].to_i >= MINIMUM_FLAKINESS_OCCURENCES + end + end + + flaky_test_issues_above_threshold.map do |issue| + yield(issue) + end + end + end +end