Merge branch 'auto-quarantine-mrs-with-gitlab-housekeeper' into 'master'

Automatic creation of quarantine MRs for flaky tests

See merge request https://gitlab.com/gitlab-org/gitlab/-/merge_requests/147137



Merged-by: David Dieulivol <ddieulivol@gitlab.com>
Approved-by: David Dieulivol <ddieulivol@gitlab.com>
Reviewed-by: Rémy Coutable <remy@rymai.me>
Reviewed-by: David Dieulivol <ddieulivol@gitlab.com>
Co-authored-by: Rémy Coutable <remy@rymai.me>

keeps/helpers/groups.rb

@@ -19,6 +19,8 @@ def group_for_group_label(group_label)
       end
 
       def pick_reviewer(group, identifiers)
+        return if group['backend_engineers'].empty?
+
         random_engineer = Digest::SHA256.hexdigest(identifiers.join).to_i(16) % group['backend_engineers'].size
 
         group['backend_engineers'][random_engineer]

keeps/quarantine_flaky_tests.rb

+# frozen_string_literal: true
+
+require 'gitlab-http'
+
+require_relative 'helpers/groups'
+
+module Keeps
+  # This is an implementation of a ::Gitlab::Housekeeper::Keep. This keep will fetch any `failure::flaky-test` issues
+  # with more than 1000 reports and quarantine these tests.
+  #
+  # You can run it individually with:
+  #
+  # ```
+  # bundle exec gitlab-housekeeper -d \
+  #   -k Keeps::QuarantineFlakyTests
+  # ```
+  class QuarantineFlakyTests < ::Gitlab::Housekeeper::Keep
+    MINIMUM_FLAKINESS_OCCURENCES = 1000
+    FLAKY_TEST_ISSUES_URL = "https://gitlab.com/api/v4/projects/gitlab-org%2Fgitlab/issues/?order_by=updated_at&state=opened&labels%5B%5D=test&labels%5B%5D=failure%3A%3Aflaky-test&not%5Blabels%5D%5B%5D=QA&not%5Blabel_name%5D%5B%5D=quarantine&per_page=10"
+    FLAKY_TEST_ISSUE_NOTES_URL = "https://gitlab.com/api/v4/projects/gitlab-org%%2Fgitlab/issues/%<issue_iid>s/notes"
+    EXAMPLE_LINE_REGEX = /([\w'",])? do$/
+
+    def each_change
+      each_very_flaky_issue do |flaky_issue|
+        change = prepare_change(flaky_issue)
+
+        yield(change) if change
+      end
+    end
+
+    private
+
+    def groups_helper
+      @groups_helper ||= ::Keeps::Helpers::Groups.new
+    end
+
+    def prepare_change(flaky_issue)
+      match = flaky_issue['description'].match(%r{\| File URL \| \[`(?<filename>[\w\/\.]+)#L(?<line_number>\d+)`\]})
+      return unless match
+
+      filename = match[:filename]
+      line_number = match[:line_number].to_i
+
+      match = flaky_issue['description'].match(%r{\| Description \| (?<description>.+) \|})
+      return unless match
+
+      description = match[:description]
+
+      file = File.expand_path("../#{filename}", __dir__)
+      full_file_content = File.read(file)
+      issue_url = flaky_issue['web_url']
+
+      file_lines = full_file_content.lines
+      return unless file_lines[line_number - 1].match?(EXAMPLE_LINE_REGEX)
+
+      file_lines[line_number - 1].sub!(EXAMPLE_LINE_REGEX, "\\1, quarantine: '#{issue_url}' do")
+      File.write(file, file_lines.join)
+
+      change = ::Gitlab::Housekeeper::Change.new
+      change.title = "Quarantine a flaky test"
+      change.identifiers = [self.class.name.demodulize, filename, line_number.to_s]
+      change.changed_files = [filename]
+      change.description = <<~MARKDOWN
+      The #{description} test has been reported as flaky more then #{MINIMUM_FLAKINESS_OCCURENCES} times.
+
+      This MR quarantines the test. This is a discussion starting point to let the responsible group know about the flakiness
+      so that they can take action:
+
+      - accept the merge request and schedule to improve the test
+      - close the merge request in favor of another merge request to delete the test
+
+      Relate to #{issue_url}.
+      MARKDOWN
+
+      group_label = flaky_issue['labels'].grep(/group::/).first
+      change.labels = [
+        'maintenance::refactor',
+        'test',
+        'failure::flaky-test',
+        'pipeline:expedite',
+        'quarantine',
+        'quarantine::flaky',
+        group_label
+      ].compact
+
+      if change.reviewers.empty? && group_label
+        group_data = groups_helper.group_for_group_label(group_label)
+
+        change.reviewers = groups_helper.pick_reviewer(group_data, change.identifiers) if group_data
+      end
+
+      change
+    end
+
+    def each_very_flaky_issue
+      flaky_test_issues = Gitlab::HTTP.get(FLAKY_TEST_ISSUES_URL)
+
+      flaky_test_issues_above_threshold = flaky_test_issues.select do |flaky_test_issue|
+        Gitlab::HTTP.get(format(FLAKY_TEST_ISSUE_NOTES_URL, { issue_iid: flaky_test_issue['iid'] }),
+          headers: { 'PRIVATE-TOKEN': ENV['HOUSEKEEPER_GITLAB_API_TOKEN'] }).find do |note|
+          match = note['body'].match(/### Flakiness reports \((?<reports_count>\d+)\)/)
+          next unless match
+
+          match[:reports_count].to_i >= MINIMUM_FLAKINESS_OCCURENCES
+        end
+      end
+
+      flaky_test_issues_above_threshold.map do |issue|
+        yield(issue)
+      end
+    end
+  end
+end