diff --git a/.gitlab/ci/gitlab-gems.gitlab-ci.yml b/.gitlab/ci/gitlab-gems.gitlab-ci.yml index 3ac8374d9085f2e61301a693cf4bea0f21efb795..802ad9a5a8a924c3fd6f508aa9143c3895fa38c7 100644 --- a/.gitlab/ci/gitlab-gems.gitlab-ci.yml +++ b/.gitlab/ci/gitlab-gems.gitlab-ci.yml @@ -19,7 +19,7 @@ include: gem_name: "ipynbdiff" - local: .gitlab/ci/templates/gem.gitlab-ci.yml inputs: - gem_name: "rspec_flaky" + gem_name: "gitlab-rspec_flaky" - local: .gitlab/ci/templates/gem.gitlab-ci.yml inputs: gem_name: "gitlab-safe_request_store" diff --git a/Gemfile b/Gemfile index a52c32f427844abb2b9742c6e03bc2d8d39b08c9..07d533136f62a06ba8f3e9221dd974d3b553e794 100644 --- a/Gemfile +++ b/Gemfile @@ -483,7 +483,7 @@ end # Gems required in various pipelines group :development, :test, :monorepo do gem 'gitlab-rspec', path: 'gems/gitlab-rspec' # rubocop:todo Gemfile/MissingFeatureCategory - gem 'rspec_flaky', path: 'gems/rspec_flaky' # rubocop:todo Gemfile/MissingFeatureCategory + gem 'gitlab-rspec_flaky', path: 'gems/gitlab-rspec_flaky', feature_category: :tooling end group :test do diff --git a/Gemfile.lock b/Gemfile.lock index 4aee89105d26ca203db3af41e2e7135589e0bf63..d4533fe0184bbe6091d22349bef62ed286f75f39 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -40,6 +40,13 @@ PATH nokogiri (~> 1.15.4) railties (~> 7) +PATH + remote: gems/gitlab-rspec_flaky + specs: + gitlab-rspec_flaky (0.1.0) + activesupport (>= 6.1, < 8) + rspec (~> 3.0) + PATH remote: gems/gitlab-rspec specs: @@ -85,13 +92,6 @@ PATH diffy (~> 3.4) oj (~> 3.13.16) -PATH - remote: gems/rspec_flaky - specs: - rspec_flaky (0.1.0) - activesupport (>= 6.1, < 8) - rspec (~> 3.0) - PATH remote: vendor/gems/attr_encrypted specs: @@ -1884,6 +1884,7 @@ DEPENDENCIES gitlab-markup (~> 1.9.0) gitlab-net-dns (~> 0.9.2) gitlab-rspec! + gitlab-rspec_flaky! gitlab-safe_request_store! gitlab-schema-validation! gitlab-sdk @@ -2037,7 +2038,6 @@ DEPENDENCIES rspec-parameterized (~> 1.0) rspec-rails (~> 6.1.0) rspec-retry (~> 0.6.2) - rspec_flaky! rspec_junit_formatter rspec_profiling (~> 0.0.6) rubocop diff --git a/doc/development/testing_guide/flaky_tests.md b/doc/development/testing_guide/flaky_tests.md index ca81f7002f469dd92d570ab5cc7c03ab23ad95d5..1895b9bdb39ee4b36563a092ab78c183f4d7cc6d 100644 --- a/doc/development/testing_guide/flaky_tests.md +++ b/doc/development/testing_guide/flaky_tests.md @@ -233,7 +233,7 @@ Once a test is in quarantine, there are 3 choices: On our CI, we use [`RSpec::Retry`](https://github.com/NoRedInk/rspec-retry) to automatically retry a failing example a few times (see [`spec/spec_helper.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/spec/spec_helper.rb) for the precise retries count). -We also use a custom [`RspecFlaky::Listener`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/gems/rspec_flaky/lib/rspec_flaky/listener.rb). +We also use a custom [`Gitlab::RspecFlaky::Listener`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/listener.rb). This listener runs in the `update-tests-metadata` job in `maintenance` scheduled pipelines on the `master` branch, and saves flaky examples to `rspec/flaky/report-suite.json`. The report file is then retrieved by the `retrieve-tests-metadata` job in all pipelines. diff --git a/gems/rspec_flaky/.gitignore b/gems/gitlab-rspec_flaky/.gitignore similarity index 100% rename from gems/rspec_flaky/.gitignore rename to gems/gitlab-rspec_flaky/.gitignore diff --git a/gems/rspec_flaky/.gitlab-ci.yml b/gems/gitlab-rspec_flaky/.gitlab-ci.yml similarity index 59% rename from gems/rspec_flaky/.gitlab-ci.yml rename to gems/gitlab-rspec_flaky/.gitlab-ci.yml index 41fac86e7a52e0a93de7e1f8125e0991f77232cc..926771014baab8f54d6bcd42e3e5114e39233861 100644 --- a/gems/rspec_flaky/.gitlab-ci.yml +++ b/gems/gitlab-rspec_flaky/.gitlab-ci.yml @@ -1,4 +1,4 @@ include: - local: gems/gem.gitlab-ci.yml inputs: - gem_name: "rspec_flaky" + gem_name: "gitlab-rspec_flaky" diff --git a/gems/rspec_flaky/.rspec b/gems/gitlab-rspec_flaky/.rspec similarity index 100% rename from gems/rspec_flaky/.rspec rename to gems/gitlab-rspec_flaky/.rspec diff --git a/gems/rspec_flaky/.rubocop.yml b/gems/gitlab-rspec_flaky/.rubocop.yml similarity index 100% rename from gems/rspec_flaky/.rubocop.yml rename to gems/gitlab-rspec_flaky/.rubocop.yml diff --git a/gems/rspec_flaky/Gemfile b/gems/gitlab-rspec_flaky/Gemfile similarity index 66% rename from gems/rspec_flaky/Gemfile rename to gems/gitlab-rspec_flaky/Gemfile index 90bf29fb6478586484484d5ab46256b9c26228ee..634ff34a9b3e89c58aa1c5a1410b9222155632f6 100644 --- a/gems/rspec_flaky/Gemfile +++ b/gems/gitlab-rspec_flaky/Gemfile @@ -2,7 +2,7 @@ source "https://rubygems.org" -# Specify your gem's dependencies in rspec_flaky.gemspec +# Specify your gem's dependencies in gitlab-rspec_flaky.gemspec gemspec gem "gitlab-rspec", "~> 0.1", path: "../gitlab-rspec" diff --git a/gems/rspec_flaky/Gemfile.lock b/gems/gitlab-rspec_flaky/Gemfile.lock similarity index 98% rename from gems/rspec_flaky/Gemfile.lock rename to gems/gitlab-rspec_flaky/Gemfile.lock index 491d589d80c9acd180d72390869b5ef7c38d158b..7b5817b91e46db40d04cd6ba43d6f66b2f934a26 100644 --- a/gems/rspec_flaky/Gemfile.lock +++ b/gems/gitlab-rspec_flaky/Gemfile.lock @@ -9,7 +9,7 @@ PATH PATH remote: . specs: - rspec_flaky (0.1.0) + gitlab-rspec_flaky (0.1.0) activesupport (>= 6.1, < 8) rspec (~> 3.0) @@ -136,9 +136,9 @@ PLATFORMS DEPENDENCIES gitlab-rspec (~> 0.1)! + gitlab-rspec_flaky! gitlab-styles (~> 10.1.0) rspec-parameterized (~> 1.0) - rspec_flaky! rubocop (~> 1.50) rubocop-rspec (~> 2.22) diff --git a/gems/rspec_flaky/rspec_flaky.gemspec b/gems/gitlab-rspec_flaky/gitlab-rspec_flaky.gemspec similarity index 85% rename from gems/rspec_flaky/rspec_flaky.gemspec rename to gems/gitlab-rspec_flaky/gitlab-rspec_flaky.gemspec index 6ddbe4afd1e39e7e0d3433425be5cf2c8f1f9970..36bf070d15d173bb328b73b777037c38e569eeec 100644 --- a/gems/rspec_flaky/rspec_flaky.gemspec +++ b/gems/gitlab-rspec_flaky/gitlab-rspec_flaky.gemspec @@ -1,10 +1,10 @@ # frozen_string_literal: true -require_relative "lib/rspec_flaky/version" +require_relative "lib/gitlab/rspec_flaky/version" Gem::Specification.new do |spec| - spec.name = "rspec_flaky" - spec.version = RspecFlaky::Version::VERSION + spec.name = "gitlab-rspec_flaky" + spec.version = Gitlab::RspecFlaky::Version::VERSION spec.authors = ["Engineering Productivity"] spec.email = ["quality@gitlab.com"] @@ -12,7 +12,7 @@ Gem::Specification.new do |spec| spec.description = "This gem provide an RSpec listener that allows to detect flaky examples. See " \ "https://docs.gitlab.com/ee/development/testing_guide/flaky_tests.html#automatic-retries-and-flaky-tests-detection." - spec.homepage = "https://gitlab.com/gitlab-org/gitlab/-/tree/master/gems/rspec_flaky" + spec.homepage = "https://gitlab.com/gitlab-org/gitlab/-/tree/master/gems/gitlab-rspec_flaky" spec.license = "MIT" spec.required_ruby_version = ">= 3.0" spec.metadata["rubygems_mfa_required"] = "true" diff --git a/gems/rspec_flaky/lib/rspec_flaky.rb b/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky.rb similarity index 78% rename from gems/rspec_flaky/lib/rspec_flaky.rb rename to gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky.rb index 90fc6b1dc490bbda65a16114529a4cb0163e4ef5..4a1a0a0fe10293b5058e4a0f15d9848fedb304b0 100644 --- a/gems/rspec_flaky/lib/rspec_flaky.rb +++ b/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky.rb @@ -4,3 +4,8 @@ require_relative "rspec_flaky/config" require_relative "rspec_flaky/listener" require_relative "rspec_flaky/version" + +module Gitlab + module RspecFlaky + end +end diff --git a/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/config.rb b/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/config.rb new file mode 100644 index 0000000000000000000000000000000000000000..daf5b4f46cc49a7b6599c5f2c64b3e2f51a4880e --- /dev/null +++ b/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/config.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module Gitlab + module RspecFlaky + class Config + def self.generate_report? + !!(ENV['FLAKY_RSPEC_GENERATE_REPORT'] =~ /1|true/) + end + + def self.suite_flaky_examples_report_path + ENV['FLAKY_RSPEC_SUITE_REPORT_PATH'] || "rspec/flaky/suite-report.json" + end + + def self.flaky_examples_report_path + ENV['FLAKY_RSPEC_REPORT_PATH'] || "rspec/flaky/report.json" + end + + def self.new_flaky_examples_report_path + ENV['NEW_FLAKY_RSPEC_REPORT_PATH'] || "rspec/flaky/new-report.json" + end + end + end +end diff --git a/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/example.rb b/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/example.rb new file mode 100644 index 0000000000000000000000000000000000000000..bc3980e1e8de6f88c0fbfb8994d9c65ba6bee917 --- /dev/null +++ b/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/example.rb @@ -0,0 +1,70 @@ +# frozen_string_literal: true + +require 'forwardable' +require 'digest' + +module Gitlab + module RspecFlaky + # This is a wrapper class for RSpec::Core::Example + class Example + extend Forwardable + + def_delegators :execution_result, :status, :exception + + def initialize(rspec_example) + @rspec_example = rspec_example.respond_to?(:example) ? rspec_example.example : rspec_example + end + + def uid + @uid ||= Digest::MD5.hexdigest("#{description}-#{file}") # rubocop:disable Fips/MD5 -- MD5 is only used to compute and ID and we need to keep using for back-compat + end + + def example_id + rspec_example.id + end + + def file + metadata[:file_path] + end + + def line + metadata[:line_number] + end + + def description + metadata[:full_description] + end + + def attempts + rspec_example.respond_to?(:attempts) ? rspec_example.attempts : 1 + end + + def feature_category + metadata[:feature_category] + end + + def to_h + { + example_id: example_id, + file: file, + line: line, + description: description, + last_attempts_count: attempts, + feature_category: feature_category + } + end + + private + + attr_reader :rspec_example + + def metadata + rspec_example.metadata + end + + def execution_result + rspec_example.execution_result + end + end + end +end diff --git a/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/flaky_example.rb b/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/flaky_example.rb new file mode 100644 index 0000000000000000000000000000000000000000..420c60778429613980d6bf5e0da4758c3c2ea54e --- /dev/null +++ b/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/flaky_example.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +require 'ostruct' + +module Gitlab + module RspecFlaky + # This represents a flaky RSpec example and is mainly meant to be saved in a JSON file + class FlakyExample + ALLOWED_ATTRIBUTES = %i[ + example_id + file + line + description + first_flaky_at + last_flaky_at + last_flaky_job + last_attempts_count + flaky_reports + feature_category + ].freeze + + def initialize(example_hash) + @attributes = { + first_flaky_at: Time.now, + last_flaky_at: Time.now, + last_flaky_job: nil, + last_attempts_count: example_hash[:attempts], + flaky_reports: 0, + feature_category: example_hash[:feature_category] + }.merge(example_hash.slice(*ALLOWED_ATTRIBUTES)) + + %i[first_flaky_at last_flaky_at].each do |attr| + attributes[attr] = Time.parse(attributes[attr]) if attributes[attr].is_a?(String) + end + end + + def update!(example_hash) + attributes[:file] = example_hash[:file] + attributes[:line] = example_hash[:line] + attributes[:description] = example_hash[:description] + attributes[:first_flaky_at] ||= Time.now + attributes[:last_flaky_at] = Time.now + attributes[:flaky_reports] += 1 + attributes[:feature_category] = example_hash[:feature_category] + attributes[:last_attempts_count] = example_hash[:last_attempts_count] if example_hash[:last_attempts_count] + + return unless ENV['CI_JOB_URL'] + + attributes[:last_flaky_job] = (ENV['CI_JOB_URL']).to_s + end + + def to_h + attributes.dup + end + + private + + attr_reader :attributes + end + end +end diff --git a/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/flaky_examples_collection.rb b/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/flaky_examples_collection.rb new file mode 100644 index 0000000000000000000000000000000000000000..7250f7bf1645f86aa2557c4c76d1e00083b4f85d --- /dev/null +++ b/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/flaky_examples_collection.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +require 'active_support/hash_with_indifferent_access' +require 'delegate' + +require_relative 'flaky_example' + +module Gitlab + module RspecFlaky + class FlakyExamplesCollection < SimpleDelegator + def initialize(collection = {}) + raise ArgumentError, "`collection` must be a Hash, #{collection.class} given!" unless collection.is_a?(Hash) + + collection_of_flaky_examples = + collection.map do |uid, example| + [ + uid, + FlakyExample.new(example.to_h.symbolize_keys) + ] + end + + super(Hash[collection_of_flaky_examples]) + end + + def to_h + transform_values(&:to_h).deep_symbolize_keys + end + + def -(other) + raise ArgumentError, "`other` must respond to `#key?`, #{other.class} does not!" unless other.respond_to?(:key) + + self.class.new(reject { |uid, _| other.key?(uid) }) + end + end + end +end diff --git a/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/listener.rb b/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/listener.rb new file mode 100644 index 0000000000000000000000000000000000000000..6f4dce9df33fcc255c5c7dd6854751e0775e8312 --- /dev/null +++ b/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/listener.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +require 'json' + +require_relative 'config' +require_relative 'example' +require_relative 'flaky_example' +require_relative 'flaky_examples_collection' +require_relative 'report' + +module Gitlab + module RspecFlaky + class Listener + # - suite_flaky_examples: contains all the currently tracked flacky example + # for the whole RSpec suite + # - flaky_examples: contains the examples detected as flaky during the + # current RSpec run + attr_reader :suite_flaky_examples, :flaky_examples + + def initialize(suite_flaky_examples_json = nil) + @flaky_examples = FlakyExamplesCollection.new + @suite_flaky_examples = init_suite_flaky_examples(suite_flaky_examples_json) + end + + def example_passed(notification) + current_example = Example.new(notification.example) + + return unless current_example.attempts > 1 + + flaky_example = suite_flaky_examples.fetch(current_example.uid) do + FlakyExample.new(current_example.to_h) + end + flaky_example.update!(current_example.to_h) + + flaky_examples[current_example.uid] = flaky_example + end + + def dump_summary(_) + Report.new(flaky_examples).write(Config.flaky_examples_report_path) + + return unless new_flaky_examples.any? + + rails_logger_warn("\nNew flaky examples detected:\n") + rails_logger_warn(JSON.pretty_generate(new_flaky_examples.to_h)) + + Report.new(new_flaky_examples).write(Config.new_flaky_examples_report_path) + end + + private + + def new_flaky_examples + @new_flaky_examples ||= flaky_examples - suite_flaky_examples + end + + def init_suite_flaky_examples(suite_flaky_examples_json = nil) + if suite_flaky_examples_json + Report.load_json(suite_flaky_examples_json).flaky_examples + else + return {} unless File.exist?(Config.suite_flaky_examples_report_path) + + Report.load(Config.suite_flaky_examples_report_path).flaky_examples + end + end + + def rails_logger_warn(text) + target = defined?(Rails) ? Rails.logger : Kernel + + target.warn(text) + end + end + end +end diff --git a/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/report.rb b/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/report.rb new file mode 100644 index 0000000000000000000000000000000000000000..0fc669d7f0aca69f3d4213918fe00eb5a48fbe6c --- /dev/null +++ b/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/report.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +require 'json' +require 'time' +require 'fileutils' + +require_relative 'config' +require_relative 'flaky_examples_collection' + +module Gitlab + module RspecFlaky + # This class is responsible for loading/saving JSON reports, and pruning + # outdated examples. + class Report < SimpleDelegator + OUTDATED_DAYS_THRESHOLD = 7 + + attr_reader :flaky_examples + + def self.load(file_path) + load_json(File.read(file_path)) + end + + def self.load_json(json) + new(FlakyExamplesCollection.new(JSON.parse(json))) + end + + def initialize(flaky_examples) + unless flaky_examples.is_a?(FlakyExamplesCollection) + raise ArgumentError, + "`flaky_examples` must be a Gitlab::RspecFlaky::FlakyExamplesCollection, #{flaky_examples.class} given!" + end + + @flaky_examples = flaky_examples + super(flaky_examples) + end + + def write(file_path) + unless Config.generate_report? + Kernel.warn "! Generating reports is disabled. To enable it, please set the `FLAKY_RSPEC_GENERATE_REPORT=1` !" + return + end + + report_path_dir = File.dirname(file_path) + FileUtils.mkdir_p(report_path_dir) + + File.write(file_path, JSON.pretty_generate(flaky_examples.to_h)) + end + + def prune_outdated(days: OUTDATED_DAYS_THRESHOLD) + outdated_date_threshold = Time.now - (3600 * 24 * days) + recent_flaky_examples = flaky_examples.dup + .delete_if do |_uid, flaky_example| + last_flaky_at = flaky_example.to_h[:last_flaky_at] + last_flaky_at && last_flaky_at.to_i < outdated_date_threshold.to_i + end + + self.class.new(FlakyExamplesCollection.new(recent_flaky_examples)) + end + end + end +end diff --git a/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/version.rb b/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/version.rb new file mode 100644 index 0000000000000000000000000000000000000000..6fae14a1744ae4ac3e3f461b5da27a31708c5658 --- /dev/null +++ b/gems/gitlab-rspec_flaky/lib/gitlab/rspec_flaky/version.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module Gitlab + module RspecFlaky + module Version + VERSION = "0.1.0" + end + end +end diff --git a/gems/rspec_flaky/spec/rspec_flaky/config_spec.rb b/gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/config_spec.rb similarity index 96% rename from gems/rspec_flaky/spec/rspec_flaky/config_spec.rb rename to gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/config_spec.rb index 827249efefa027488f25e2341e9242644e1dd609..a3508b123bd7ee4a024433843bb8280cbe140c06 100644 --- a/gems/rspec_flaky/spec/rspec_flaky/config_spec.rb +++ b/gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/config_spec.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true -require 'rspec_flaky/config' +require 'gitlab/rspec_flaky/config' -RSpec.describe RspecFlaky::Config, :aggregate_failures do +RSpec.describe Gitlab::RspecFlaky::Config, :aggregate_failures do include StubENV before do diff --git a/gems/rspec_flaky/spec/rspec_flaky/example_spec.rb b/gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/example_spec.rb similarity index 97% rename from gems/rspec_flaky/spec/rspec_flaky/example_spec.rb rename to gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/example_spec.rb index 64d65c0e1705ca4d6bf55eac51ccd0ba374c2d10..8e38b93526a9e56445f50383356b9c18a95e7d35 100644 --- a/gems/rspec_flaky/spec/rspec_flaky/example_spec.rb +++ b/gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/example_spec.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true -require 'rspec_flaky/example' +require 'gitlab/rspec_flaky/example' -RSpec.describe RspecFlaky::Example do +RSpec.describe Gitlab::RspecFlaky::Example do let(:example_attrs) do { id: 'spec/foo/bar_spec.rb:2', diff --git a/gems/rspec_flaky/spec/rspec_flaky/flaky_example_spec.rb b/gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/flaky_example_spec.rb similarity index 97% rename from gems/rspec_flaky/spec/rspec_flaky/flaky_example_spec.rb rename to gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/flaky_example_spec.rb index 244ca275f1495b91c695f51ff932630636ed0061..2d4393f2831e1e91a27ac3e242a54d6e9db4c5df 100644 --- a/gems/rspec_flaky/spec/rspec_flaky/flaky_example_spec.rb +++ b/gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/flaky_example_spec.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true -require 'rspec_flaky/flaky_example' +require 'gitlab/rspec_flaky/flaky_example' -RSpec.describe RspecFlaky::FlakyExample, :aggregate_failures do +RSpec.describe Gitlab::RspecFlaky::FlakyExample, :aggregate_failures do include StubENV let(:example_attrs) do diff --git a/gems/rspec_flaky/spec/rspec_flaky/flaky_examples_collection_spec.rb b/gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/flaky_examples_collection_spec.rb similarity index 93% rename from gems/rspec_flaky/spec/rspec_flaky/flaky_examples_collection_spec.rb rename to gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/flaky_examples_collection_spec.rb index 260ebc7219248608bd00a765efd89b4756d0b4d0..16811ecb2d28661d9e5cde6a69fb431b70c3d084 100644 --- a/gems/rspec_flaky/spec/rspec_flaky/flaky_examples_collection_spec.rb +++ b/gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/flaky_examples_collection_spec.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true -require 'rspec_flaky/flaky_examples_collection' +require 'gitlab/rspec_flaky/flaky_examples_collection' -RSpec.describe RspecFlaky::FlakyExamplesCollection, :aggregate_failures, :freeze_time do +RSpec.describe Gitlab::RspecFlaky::FlakyExamplesCollection, :aggregate_failures, :freeze_time do let(:collection_hash) do { a: { example_id: 'spec/foo/bar_spec.rb:2' }, diff --git a/gems/rspec_flaky/spec/rspec_flaky/listener_spec.rb b/gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/listener_spec.rb similarity index 87% rename from gems/rspec_flaky/spec/rspec_flaky/listener_spec.rb rename to gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/listener_spec.rb index cbc5422a76329b5731cb0f156b2b712186373b55..b46044e45218ce531c3bcaf2dc103d97e24910b8 100644 --- a/gems/rspec_flaky/spec/rspec_flaky/listener_spec.rb +++ b/gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/listener_spec.rb @@ -1,8 +1,8 @@ # frozen_string_literal: true -require 'rspec_flaky/listener' +require 'gitlab/rspec_flaky/listener' -RSpec.describe RspecFlaky::Listener, :aggregate_failures do +RSpec.describe Gitlab::RspecFlaky::Listener, :aggregate_failures do include StubENV let(:already_flaky_example_uid) { '6e869794f4cfd2badd93eb68719371d1' } @@ -85,10 +85,10 @@ end it 'delegates the load to RspecFlaky::Report' do - report = RspecFlaky::Report - .new(RspecFlaky::FlakyExamplesCollection.new(suite_flaky_example_report)) + report = Gitlab::RspecFlaky::Report + .new(Gitlab::RspecFlaky::FlakyExamplesCollection.new(suite_flaky_example_report)) - expect(RspecFlaky::Report).to receive(:load).with(report_file_path).and_return(report) + expect(Gitlab::RspecFlaky::Report).to receive(:load).with(report_file_path).and_return(report) expect(described_class.new.suite_flaky_examples.to_h).to eq(report.flaky_examples.to_h) end end @@ -99,7 +99,7 @@ end it 'return an empty hash' do - expect(RspecFlaky::Report).not_to receive(:load) + expect(Gitlab::RspecFlaky::Report).not_to receive(:load) expect(described_class.new.suite_flaky_examples.to_h).to eq({}) end end @@ -134,7 +134,7 @@ end it 'changes the flaky examples hash' do - new_example = RspecFlaky::Example.new(rspec_example) + new_example = Gitlab::RspecFlaky::Example.new(rspec_example) travel_to(Time.now + 42) do the_future = Time.now @@ -161,7 +161,7 @@ end it 'changes the all flaky examples hash' do - new_example = RspecFlaky::Example.new(rspec_example) + new_example = Gitlab::RspecFlaky::Example.new(rspec_example) travel_to(Time.now + 42) do the_future = Time.now @@ -215,12 +215,12 @@ report1 = double report2 = double - expect(RspecFlaky::Report).to receive(:new).with(listener.flaky_examples).and_return(report1) - expect(report1).to receive(:write).with(RspecFlaky::Config.flaky_examples_report_path) + expect(Gitlab::RspecFlaky::Report).to receive(:new).with(listener.flaky_examples).and_return(report1) + expect(report1).to receive(:write).with(Gitlab::RspecFlaky::Config.flaky_examples_report_path) - expect(RspecFlaky::Report) + expect(Gitlab::RspecFlaky::Report) .to receive(:new).with(listener.__send__(:new_flaky_examples)).and_return(report2) - expect(report2).to receive(:write).with(RspecFlaky::Config.new_flaky_examples_report_path) + expect(report2).to receive(:write).with(Gitlab::RspecFlaky::Config.new_flaky_examples_report_path) listener.dump_summary(nil) end diff --git a/gems/rspec_flaky/spec/rspec_flaky/report_spec.rb b/gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/report_spec.rb similarity index 80% rename from gems/rspec_flaky/spec/rspec_flaky/report_spec.rb rename to gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/report_spec.rb index e1e9bd6a7b1819efc25550035a1d338e088f92ae..74d521b1ea1061c31d0bf6f36c517dfbd0d84f63 100644 --- a/gems/rspec_flaky/spec/rspec_flaky/report_spec.rb +++ b/gems/gitlab-rspec_flaky/spec/gitlab/rspec_flaky/report_spec.rb @@ -2,9 +2,9 @@ require 'tempfile' -require 'rspec_flaky/report' +require 'gitlab/rspec_flaky/report' -RSpec.describe RspecFlaky::Report, :aggregate_failures, :freeze_time do +RSpec.describe Gitlab::RspecFlaky::Report, :aggregate_failures, :freeze_time do let(:thirty_one_days) { 3600 * 24 * 31 } let(:collection_hash) do { @@ -31,7 +31,7 @@ } end - let(:flaky_examples) { RspecFlaky::FlakyExamplesCollection.new(collection_hash) } + let(:flaky_examples) { Gitlab::RspecFlaky::FlakyExamplesCollection.new(collection_hash) } let(:report) { described_class.new(flaky_examples) } before do @@ -67,7 +67,7 @@ end describe '#initialize' do - it 'accepts a RspecFlaky::FlakyExamplesCollection' do + it 'accepts a Gitlab::RspecFlaky::FlakyExamplesCollection' do expect { report }.not_to raise_error end @@ -76,7 +76,7 @@ described_class.new([1, 2, 3]) end.to raise_error(ArgumentError, - "`flaky_examples` must be a RspecFlaky::FlakyExamplesCollection, Array given!") + "`flaky_examples` must be a Gitlab::RspecFlaky::FlakyExamplesCollection, Array given!") end end @@ -95,9 +95,9 @@ FileUtils.rm_f(report_file_path) end - context 'when RspecFlaky::Config.generate_report? is false' do + context 'when Gitlab::RspecFlaky::Config.generate_report? is false' do before do - allow(RspecFlaky::Config).to receive(:generate_report?).and_return(false) + allow(Gitlab::RspecFlaky::Config).to receive(:generate_report?).and_return(false) end it 'does not write any report file' do @@ -107,12 +107,12 @@ end end - context 'when RspecFlaky::Config.generate_report? is true' do + context 'when Gitlab::RspecFlaky::Config.generate_report? is true' do before do - allow(RspecFlaky::Config).to receive(:generate_report?).and_return(true) + allow(Gitlab::RspecFlaky::Config).to receive(:generate_report?).and_return(true) end - it 'delegates the writes to RspecFlaky::Report' do + it 'delegates the writes to Gitlab::RspecFlaky::Report' do report.write(report_file_path) expect(File.exist?(report_file_path)).to be(true) diff --git a/gems/rspec_flaky/spec/spec_helper.rb b/gems/gitlab-rspec_flaky/spec/spec_helper.rb similarity index 93% rename from gems/rspec_flaky/spec/spec_helper.rb rename to gems/gitlab-rspec_flaky/spec/spec_helper.rb index 72d48ee6e6384c8469c806a0d402bbb9f665c482..12c60cb17ad7e449e67465e292453b0088beb330 100644 --- a/gems/rspec_flaky/spec/spec_helper.rb +++ b/gems/gitlab-rspec_flaky/spec/spec_helper.rb @@ -2,7 +2,7 @@ require "rspec-parameterized" require "gitlab/rspec/all" -require "rspec_flaky" +require "gitlab/rspec_flaky" RSpec.configure do |config| # Enable flags like --only-failures and --next-failure diff --git a/gems/rspec_flaky/lib/rspec_flaky/config.rb b/gems/rspec_flaky/lib/rspec_flaky/config.rb deleted file mode 100644 index ca57d1e08be3ea7d5ed233474fe596a8d044753e..0000000000000000000000000000000000000000 --- a/gems/rspec_flaky/lib/rspec_flaky/config.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -module RspecFlaky - class Config - def self.generate_report? - !!(ENV['FLAKY_RSPEC_GENERATE_REPORT'] =~ /1|true/) - end - - def self.suite_flaky_examples_report_path - ENV['FLAKY_RSPEC_SUITE_REPORT_PATH'] || "rspec/flaky/suite-report.json" - end - - def self.flaky_examples_report_path - ENV['FLAKY_RSPEC_REPORT_PATH'] || "rspec/flaky/report.json" - end - - def self.new_flaky_examples_report_path - ENV['NEW_FLAKY_RSPEC_REPORT_PATH'] || "rspec/flaky/new-report.json" - end - end -end diff --git a/gems/rspec_flaky/lib/rspec_flaky/example.rb b/gems/rspec_flaky/lib/rspec_flaky/example.rb deleted file mode 100644 index 4a128a151dc3224ecb25f6b156057f03ac5d0f59..0000000000000000000000000000000000000000 --- a/gems/rspec_flaky/lib/rspec_flaky/example.rb +++ /dev/null @@ -1,68 +0,0 @@ -# frozen_string_literal: true - -require 'forwardable' -require 'digest' - -module RspecFlaky - # This is a wrapper class for RSpec::Core::Example - class Example - extend Forwardable - - def_delegators :execution_result, :status, :exception - - def initialize(rspec_example) - @rspec_example = rspec_example.respond_to?(:example) ? rspec_example.example : rspec_example - end - - def uid - @uid ||= Digest::MD5.hexdigest("#{description}-#{file}") # rubocop:disable Fips/MD5 - end - - def example_id - rspec_example.id - end - - def file - metadata[:file_path] - end - - def line - metadata[:line_number] - end - - def description - metadata[:full_description] - end - - def attempts - rspec_example.respond_to?(:attempts) ? rspec_example.attempts : 1 - end - - def feature_category - metadata[:feature_category] - end - - def to_h - { - example_id: example_id, - file: file, - line: line, - description: description, - last_attempts_count: attempts, - feature_category: feature_category - } - end - - private - - attr_reader :rspec_example - - def metadata - rspec_example.metadata - end - - def execution_result - rspec_example.execution_result - end - end -end diff --git a/gems/rspec_flaky/lib/rspec_flaky/flaky_example.rb b/gems/rspec_flaky/lib/rspec_flaky/flaky_example.rb deleted file mode 100644 index 35d1f34d2a28185ae812460690d9bc267a4cc64f..0000000000000000000000000000000000000000 --- a/gems/rspec_flaky/lib/rspec_flaky/flaky_example.rb +++ /dev/null @@ -1,59 +0,0 @@ -# frozen_string_literal: true - -require 'ostruct' - -module RspecFlaky - # This represents a flaky RSpec example and is mainly meant to be saved in a JSON file - class FlakyExample - ALLOWED_ATTRIBUTES = %i[ - example_id - file - line - description - first_flaky_at - last_flaky_at - last_flaky_job - last_attempts_count - flaky_reports - feature_category - ].freeze - - def initialize(example_hash) - @attributes = { - first_flaky_at: Time.now, - last_flaky_at: Time.now, - last_flaky_job: nil, - last_attempts_count: example_hash[:attempts], - flaky_reports: 0, - feature_category: example_hash[:feature_category] - }.merge(example_hash.slice(*ALLOWED_ATTRIBUTES)) - - %i[first_flaky_at last_flaky_at].each do |attr| - attributes[attr] = Time.parse(attributes[attr]) if attributes[attr].is_a?(String) - end - end - - def update!(example_hash) - attributes[:file] = example_hash[:file] - attributes[:line] = example_hash[:line] - attributes[:description] = example_hash[:description] - attributes[:first_flaky_at] ||= Time.now - attributes[:last_flaky_at] = Time.now - attributes[:flaky_reports] += 1 - attributes[:feature_category] = example_hash[:feature_category] - attributes[:last_attempts_count] = example_hash[:last_attempts_count] if example_hash[:last_attempts_count] - - return unless ENV['CI_JOB_URL'] - - attributes[:last_flaky_job] = (ENV['CI_JOB_URL']).to_s - end - - def to_h - attributes.dup - end - - private - - attr_reader :attributes - end -end diff --git a/gems/rspec_flaky/lib/rspec_flaky/flaky_examples_collection.rb b/gems/rspec_flaky/lib/rspec_flaky/flaky_examples_collection.rb deleted file mode 100644 index f03fe63d11b509efca48d9d96b6ef301dea29408..0000000000000000000000000000000000000000 --- a/gems/rspec_flaky/lib/rspec_flaky/flaky_examples_collection.rb +++ /dev/null @@ -1,34 +0,0 @@ -# frozen_string_literal: true - -require 'active_support/hash_with_indifferent_access' -require 'delegate' - -require_relative 'flaky_example' - -module RspecFlaky - class FlakyExamplesCollection < SimpleDelegator - def initialize(collection = {}) - raise ArgumentError, "`collection` must be a Hash, #{collection.class} given!" unless collection.is_a?(Hash) - - collection_of_flaky_examples = - collection.map do |uid, example| - [ - uid, - FlakyExample.new(example.to_h.symbolize_keys) - ] - end - - super(Hash[collection_of_flaky_examples]) - end - - def to_h - transform_values(&:to_h).deep_symbolize_keys - end - - def -(other) - raise ArgumentError, "`other` must respond to `#key?`, #{other.class} does not!" unless other.respond_to?(:key) - - self.class.new(reject { |uid, _| other.key?(uid) }) - end - end -end diff --git a/gems/rspec_flaky/lib/rspec_flaky/listener.rb b/gems/rspec_flaky/lib/rspec_flaky/listener.rb deleted file mode 100644 index c2deb1c327a64766e0cc62d36c3ad16427b85489..0000000000000000000000000000000000000000 --- a/gems/rspec_flaky/lib/rspec_flaky/listener.rb +++ /dev/null @@ -1,70 +0,0 @@ -# frozen_string_literal: true - -require 'json' - -require_relative 'config' -require_relative 'example' -require_relative 'flaky_example' -require_relative 'flaky_examples_collection' -require_relative 'report' - -module RspecFlaky - class Listener - # - suite_flaky_examples: contains all the currently tracked flacky example - # for the whole RSpec suite - # - flaky_examples: contains the examples detected as flaky during the - # current RSpec run - attr_reader :suite_flaky_examples, :flaky_examples - - def initialize(suite_flaky_examples_json = nil) - @flaky_examples = FlakyExamplesCollection.new - @suite_flaky_examples = init_suite_flaky_examples(suite_flaky_examples_json) - end - - def example_passed(notification) - current_example = Example.new(notification.example) - - return unless current_example.attempts > 1 - - flaky_example = suite_flaky_examples.fetch(current_example.uid) do - FlakyExample.new(current_example.to_h) - end - flaky_example.update!(current_example.to_h) - - flaky_examples[current_example.uid] = flaky_example - end - - def dump_summary(_) - Report.new(flaky_examples).write(Config.flaky_examples_report_path) - - return unless new_flaky_examples.any? - - rails_logger_warn("\nNew flaky examples detected:\n") - rails_logger_warn(JSON.pretty_generate(new_flaky_examples.to_h)) - - Report.new(new_flaky_examples).write(Config.new_flaky_examples_report_path) - end - - private - - def new_flaky_examples - @new_flaky_examples ||= flaky_examples - suite_flaky_examples - end - - def init_suite_flaky_examples(suite_flaky_examples_json = nil) - if suite_flaky_examples_json - Report.load_json(suite_flaky_examples_json).flaky_examples - else - return {} unless File.exist?(Config.suite_flaky_examples_report_path) - - Report.load(Config.suite_flaky_examples_report_path).flaky_examples - end - end - - def rails_logger_warn(text) - target = defined?(Rails) ? Rails.logger : Kernel - - target.warn(text) - end - end -end diff --git a/gems/rspec_flaky/lib/rspec_flaky/report.rb b/gems/rspec_flaky/lib/rspec_flaky/report.rb deleted file mode 100644 index cc213d336ae7e52c19020f0798f2f0f958126e99..0000000000000000000000000000000000000000 --- a/gems/rspec_flaky/lib/rspec_flaky/report.rb +++ /dev/null @@ -1,59 +0,0 @@ -# frozen_string_literal: true - -require 'json' -require 'time' -require 'fileutils' - -require_relative 'config' -require_relative 'flaky_examples_collection' - -module RspecFlaky - # This class is responsible for loading/saving JSON reports, and pruning - # outdated examples. - class Report < SimpleDelegator - OUTDATED_DAYS_THRESHOLD = 7 - - attr_reader :flaky_examples - - def self.load(file_path) - load_json(File.read(file_path)) - end - - def self.load_json(json) - new(FlakyExamplesCollection.new(JSON.parse(json))) - end - - def initialize(flaky_examples) - unless flaky_examples.is_a?(FlakyExamplesCollection) - raise ArgumentError, - "`flaky_examples` must be a RspecFlaky::FlakyExamplesCollection, #{flaky_examples.class} given!" - end - - @flaky_examples = flaky_examples - super(flaky_examples) - end - - def write(file_path) - unless Config.generate_report? - Kernel.warn "! Generating reports is disabled. To enable it, please set the `FLAKY_RSPEC_GENERATE_REPORT=1` !" - return - end - - report_path_dir = File.dirname(file_path) - FileUtils.mkdir_p(report_path_dir) - - File.write(file_path, JSON.pretty_generate(flaky_examples.to_h)) - end - - def prune_outdated(days: OUTDATED_DAYS_THRESHOLD) - outdated_date_threshold = Time.now - (3600 * 24 * days) - recent_flaky_examples = flaky_examples.dup - .delete_if do |_uid, flaky_example| - last_flaky_at = flaky_example.to_h[:last_flaky_at] - last_flaky_at && last_flaky_at.to_i < outdated_date_threshold.to_i - end - - self.class.new(FlakyExamplesCollection.new(recent_flaky_examples)) - end - end -end diff --git a/gems/rspec_flaky/lib/rspec_flaky/version.rb b/gems/rspec_flaky/lib/rspec_flaky/version.rb deleted file mode 100644 index ec507d734c873e78ca2342c820ac7b5a49a4a0fa..0000000000000000000000000000000000000000 --- a/gems/rspec_flaky/lib/rspec_flaky/version.rb +++ /dev/null @@ -1,7 +0,0 @@ -# frozen_string_literal: true - -module RspecFlaky - module Version - VERSION = "0.1.0" - end -end diff --git a/scripts/flaky_examples/prune-old-flaky-examples b/scripts/flaky_examples/prune-old-flaky-examples index fc31f0f699679f9fa7346a18be9832a9a8cb2c3e..61ea44fe55c70f699ee3a4f42a06c15f130dc549 100755 --- a/scripts/flaky_examples/prune-old-flaky-examples +++ b/scripts/flaky_examples/prune-old-flaky-examples @@ -6,7 +6,7 @@ require 'bundler/inline' gemfile do source 'https://rubygems.org' - gem 'rspec_flaky', path: 'gems/rspec_flaky' + gem 'gitlab-rspec_flaky', path: 'gems/gitlab-rspec_flaky' end report_file = ARGV.shift @@ -16,12 +16,12 @@ unless report_file end new_report_file = ARGV.shift || report_file -report = RspecFlaky::Report.load(report_file) +report = Gitlab::RspecFlaky::Report.load(report_file) puts "Loading #{report_file}..." puts "Current report has #{report.size} entries." new_report = report.prune_outdated puts "New report has #{new_report.size} entries: #{report.size - new_report.size} entries older than " \ - "#{RspecFlaky::Report::OUTDATED_DAYS_THRESHOLD} days were removed." + "#{Gitlab::RspecFlaky::Report::OUTDATED_DAYS_THRESHOLD} days were removed." puts "Saved #{new_report_file}." if new_report.write(new_report_file) diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index a3bc21ff11e2050425b30507e2082deccf115bd8..7317b512ae426b1ea08572cd1a70b476f7e02e16 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -38,7 +38,7 @@ require 'parslet/rig/rspec' require 'axe-rspec' -require 'rspec_flaky' +require 'gitlab/rspec_flaky' rspec_profiling_is_configured = ENV['RSPEC_PROFILING_POSTGRES_URL'].present? || @@ -224,9 +224,9 @@ config.exceptions_to_hard_fail = [DeprecationToolkitEnv::DeprecationBehaviors::SelectiveRaise::RaiseDisallowedDeprecation] end - if RspecFlaky::Config.generate_report? + if Gitlab::RspecFlaky::Config.generate_report? config.reporter.register_listener( - RspecFlaky::Listener.new, + Gitlab::RspecFlaky::Listener.new, :example_passed, :dump_summary) end