diff --git a/.gitlab/ci/gitlab-gems.gitlab-ci.yml b/.gitlab/ci/gitlab-gems.gitlab-ci.yml
index f188e1f97f8ad4fc128821281695eeec33715d88..c996f96b823b227c864df64ef19a245e651e9ba6 100644
--- a/.gitlab/ci/gitlab-gems.gitlab-ci.yml
+++ b/.gitlab/ci/gitlab-gems.gitlab-ci.yml
@@ -4,16 +4,19 @@ include:
gem_name: "activerecord-gitlab"
- local: .gitlab/ci/templates/gem.gitlab-ci.yml
inputs:
- gem_name: "gitlab-rspec"
+ gem_name: "click_house-client"
- local: .gitlab/ci/templates/gem.gitlab-ci.yml
inputs:
- gem_name: "gitlab-utils"
+ gem_name: "gitlab-ipynbdiff"
- local: .gitlab/ci/templates/gem.gitlab-ci.yml
inputs:
- gem_name: "click_house-client"
+ gem_name: "gitlab-rspec"
- local: .gitlab/ci/templates/gem.gitlab-ci.yml
inputs:
gem_name: "gitlab-schema-validation"
- local: .gitlab/ci/templates/gem.gitlab-ci.yml
inputs:
- gem_name: "gitlab-ipynbdiff"
+ gem_name: "gitlab-utils"
+ - local: .gitlab/ci/templates/gem.gitlab-ci.yml
+ inputs:
+ gem_name: "rspec_flaky"
diff --git a/.rubocop.yml b/.rubocop.yml
index 9c0fb76f68f0c44bd425c493df9a4519148cbcbb..f6d0625a5301bf922b1a758e6ced7b8590d59890 100644
--- a/.rubocop.yml
+++ b/.rubocop.yml
@@ -395,7 +395,6 @@ Gitlab/Json:
Exclude:
- 'qa/**/*'
- 'scripts/**/*'
- - 'tooling/rspec_flaky/**/*'
- 'lib/quality/**/*'
- 'tooling/danger/**/*'
diff --git a/.rubocop_todo/fips/md5.yml b/.rubocop_todo/fips/md5.yml
index d50aaf7950256e973e851cf821dbe8104eacb327..4a1994635b7b99cf151c2c3ede232b002ed11339 100644
--- a/.rubocop_todo/fips/md5.yml
+++ b/.rubocop_todo/fips/md5.yml
@@ -17,5 +17,3 @@ Fips/MD5:
- 'spec/lib/gitlab/ci/trace/remote_checksum_spec.rb'
- 'spec/models/concerns/checksummable_spec.rb'
- 'spec/services/gravatar_service_spec.rb'
- - 'spec/tooling/rspec_flaky/example_spec.rb'
- - 'tooling/rspec_flaky/example.rb'
diff --git a/.rubocop_todo/layout/line_length.yml b/.rubocop_todo/layout/line_length.yml
index 72f43c7779e8ad1849b6d8bfd7a01b65d02b9246..0ffb5e3bdd858d1ab47566d2d9d84731062b1457 100644
--- a/.rubocop_todo/layout/line_length.yml
+++ b/.rubocop_todo/layout/line_length.yml
@@ -3103,7 +3103,6 @@ Layout/LineLength:
- 'scripts/api/cancel_pipeline.rb'
- 'scripts/api/get_job_id.rb'
- 'scripts/changed-feature-flags'
- - 'scripts/flaky_examples/prune-old-flaky-examples'
- 'scripts/lint_templates_bash.rb'
- 'scripts/no-dir-check'
- 'scripts/perf/query_limiting_report.rb'
@@ -5067,9 +5066,6 @@ Layout/LineLength:
- 'spec/tooling/lib/tooling/kubernetes_client_spec.rb'
- 'spec/tooling/lib/tooling/test_map_generator_spec.rb'
- 'spec/tooling/quality/test_level_spec.rb'
- - 'spec/tooling/rspec_flaky/config_spec.rb'
- - 'spec/tooling/rspec_flaky/flaky_examples_collection_spec.rb'
- - 'spec/tooling/rspec_flaky/report_spec.rb'
- 'spec/uploaders/ci/pipeline_artifact_uploader_spec.rb'
- 'spec/uploaders/file_mover_spec.rb'
- 'spec/uploaders/file_uploader_spec.rb'
@@ -5180,5 +5176,3 @@ Layout/LineLength:
- 'tooling/lib/tooling/kubernetes_client.rb'
- 'tooling/merge_request_rspec_failure_rake_task.rb'
- 'tooling/quality/test_level.rb'
- - 'tooling/rspec_flaky/listener.rb'
- - 'tooling/rspec_flaky/report.rb'
diff --git a/.rubocop_todo/lint/non_atomic_file_operation.yml b/.rubocop_todo/lint/non_atomic_file_operation.yml
index cd700870448f68e5a147d6771cb4173e1e5c2591..d9d415de9231cc737e366a34adeec9ded6066ef3 100644
--- a/.rubocop_todo/lint/non_atomic_file_operation.yml
+++ b/.rubocop_todo/lint/non_atomic_file_operation.yml
@@ -41,5 +41,3 @@ Lint/NonAtomicFileOperation:
- 'spec/services/bulk_imports/repository_bundle_export_service_spec.rb'
- 'spec/services/bulk_imports/uploads_export_service_spec.rb'
- 'spec/support/database/query_recorder.rb'
- - 'spec/tooling/rspec_flaky/report_spec.rb'
- - 'tooling/rspec_flaky/report.rb'
diff --git a/.rubocop_todo/lint/redundant_cop_disable_directive.yml b/.rubocop_todo/lint/redundant_cop_disable_directive.yml
index b35f876ed503676c8073b140bfaddcba36146806..7a07242ab29cf93aa985d243ab9251f700dc9c31 100644
--- a/.rubocop_todo/lint/redundant_cop_disable_directive.yml
+++ b/.rubocop_todo/lint/redundant_cop_disable_directive.yml
@@ -316,4 +316,3 @@ Lint/RedundantCopDisableDirective:
- 'tooling/lib/tooling/helm3_client.rb'
- 'tooling/lib/tooling/kubernetes_client.rb'
- 'tooling/quality/test_level.rb'
- - 'tooling/rspec_flaky/listener.rb'
diff --git a/.rubocop_todo/lint/unused_block_argument.yml b/.rubocop_todo/lint/unused_block_argument.yml
index 01510a8443d9b2ab7d9184fdc9cb47d2113ab12b..05e30a7f9add646e9212bf2006d1347fdf0da40b 100644
--- a/.rubocop_todo/lint/unused_block_argument.yml
+++ b/.rubocop_todo/lint/unused_block_argument.yml
@@ -426,6 +426,5 @@ Lint/UnusedBlockArgument:
- 'spec/support/shared_examples/uploaders/object_storage_shared_examples.rb'
- 'spec/support/shared_examples/workers/concerns/reenqueuer_shared_examples.rb'
- 'spec/tooling/lib/tooling/find_codeowners_spec.rb'
- - 'spec/tooling/rspec_flaky/config_spec.rb'
- 'spec/workers/projects/git_garbage_collect_worker_spec.rb'
- 'tooling/lib/tooling/find_codeowners.rb'
diff --git a/.rubocop_todo/rspec/expect_in_hook.yml b/.rubocop_todo/rspec/expect_in_hook.yml
index 3ae4d9ee7f4db491e937d88ba2e4e2e8977ff1ba..b0369d5180d28b3ec584827948b5cd7001db8223 100644
--- a/.rubocop_todo/rspec/expect_in_hook.yml
+++ b/.rubocop_todo/rspec/expect_in_hook.yml
@@ -466,7 +466,6 @@ RSpec/ExpectInHook:
- 'spec/tasks/gitlab/praefect_rake_spec.rb'
- 'spec/lib/gitlab/task_helpers_spec.rb'
- 'spec/tooling/danger/feature_flag_spec.rb'
- - 'spec/tooling/rspec_flaky/listener_spec.rb'
- 'spec/uploaders/file_mover_spec.rb'
- 'spec/uploaders/gitlab_uploader_spec.rb'
- 'spec/uploaders/object_storage_spec.rb'
diff --git a/.rubocop_todo/rspec/missing_feature_category.yml b/.rubocop_todo/rspec/missing_feature_category.yml
index bc050a40a737f57362e2972d5680e6c864ea1564..806226149289daec857d71e424dc46f09889d123 100644
--- a/.rubocop_todo/rspec/missing_feature_category.yml
+++ b/.rubocop_todo/rspec/missing_feature_category.yml
@@ -5618,12 +5618,6 @@ RSpec/MissingFeatureCategory:
- 'spec/tooling/lib/tooling/test_map_generator_spec.rb'
- 'spec/tooling/lib/tooling/test_map_packer_spec.rb'
- 'spec/tooling/merge_request_spec.rb'
- - 'spec/tooling/rspec_flaky/config_spec.rb'
- - 'spec/tooling/rspec_flaky/example_spec.rb'
- - 'spec/tooling/rspec_flaky/flaky_example_spec.rb'
- - 'spec/tooling/rspec_flaky/flaky_examples_collection_spec.rb'
- - 'spec/tooling/rspec_flaky/listener_spec.rb'
- - 'spec/tooling/rspec_flaky/report_spec.rb'
- 'spec/uploaders/attachment_uploader_spec.rb'
- 'spec/uploaders/avatar_uploader_spec.rb'
- 'spec/uploaders/ci/pipeline_artifact_uploader_spec.rb'
diff --git a/.rubocop_todo/rspec/verified_doubles.yml b/.rubocop_todo/rspec/verified_doubles.yml
index 285dd769e251074185cf7dac264f384367d45565..0d66bf5604b5fd04d206f95e23f4e91e8b86c616 100644
--- a/.rubocop_todo/rspec/verified_doubles.yml
+++ b/.rubocop_todo/rspec/verified_doubles.yml
@@ -983,8 +983,6 @@ RSpec/VerifiedDoubles:
- 'spec/tooling/danger/project_helper_spec.rb'
- 'spec/tooling/lib/tooling/helm3_client_spec.rb'
- 'spec/tooling/lib/tooling/kubernetes_client_spec.rb'
- - 'spec/tooling/rspec_flaky/example_spec.rb'
- - 'spec/tooling/rspec_flaky/listener_spec.rb'
- 'spec/uploaders/file_uploader_spec.rb'
- 'spec/uploaders/object_storage_spec.rb'
- 'spec/uploaders/personal_file_uploader_spec.rb'
diff --git a/.rubocop_todo/style/guard_clause.yml b/.rubocop_todo/style/guard_clause.yml
index abd7fe7af98c1fb63058b9677686d338b3fa16c7..f00f5bed9d048c3abc00ef557dd9be9da4703b9a 100644
--- a/.rubocop_todo/style/guard_clause.yml
+++ b/.rubocop_todo/style/guard_clause.yml
@@ -640,5 +640,3 @@ Style/GuardClause:
- 'tooling/lib/tooling/helm3_client.rb'
- 'tooling/lib/tooling/image.rb'
- 'tooling/lib/tooling/kubernetes_client.rb'
- - 'tooling/rspec_flaky/flaky_example.rb'
- - 'tooling/rspec_flaky/listener.rb'
diff --git a/.rubocop_todo/style/if_unless_modifier.yml b/.rubocop_todo/style/if_unless_modifier.yml
index 09707e6864836e9dbcbc9aa526acf85307d32419..a1f516a8abec71448e36020fcacea60d61b5e60c 100644
--- a/.rubocop_todo/style/if_unless_modifier.yml
+++ b/.rubocop_todo/style/if_unless_modifier.yml
@@ -1074,5 +1074,3 @@ Style/IfUnlessModifier:
- 'tooling/lib/tooling/find_codeowners.rb'
- 'tooling/lib/tooling/image.rb'
- 'tooling/lib/tooling/test_map_packer.rb'
- - 'tooling/rspec_flaky/flaky_example.rb'
- - 'tooling/rspec_flaky/flaky_examples_collection.rb'
diff --git a/.rubocop_todo/style/redundant_interpolation.yml b/.rubocop_todo/style/redundant_interpolation.yml
index a618632f5525d0f838d40558ef2d3c504ec693ae..27cd954f506f748640f09169561f092bb6dbb8a7 100644
--- a/.rubocop_todo/style/redundant_interpolation.yml
+++ b/.rubocop_todo/style/redundant_interpolation.yml
@@ -35,4 +35,3 @@ Style/RedundantInterpolation:
- 'qa/qa/tools/generate_perf_testdata.rb'
- 'scripts/perf/gc/print_gc_stats.rb'
- 'scripts/qa/testcases-check'
- - 'tooling/rspec_flaky/flaky_example.rb'
diff --git a/.rubocop_todo/style/symbol_proc.yml b/.rubocop_todo/style/symbol_proc.yml
index 0e4602f9f317e18f156fd9470cb19f84f2c19170..67229a8b704ee766362b448bf1021ff90ea130a9 100644
--- a/.rubocop_todo/style/symbol_proc.yml
+++ b/.rubocop_todo/style/symbol_proc.yml
@@ -241,4 +241,3 @@ Style/SymbolProc:
- 'spec/views/layouts/_published_experiments.html.haml_spec.rb'
- 'spec/workers/snippets/schedule_bulk_repository_shard_moves_worker_spec.rb'
- 'tooling/lib/tooling/test_map_generator.rb'
- - 'tooling/rspec_flaky/flaky_examples_collection.rb'
diff --git a/Gemfile b/Gemfile
index 3a6ce1553f7fcf4cb2adce1e672b49d9e99831cd..125f8bf1fe3a820a97cf5ca2bff679c5b0c93cff 100644
--- a/Gemfile
+++ b/Gemfile
@@ -462,6 +462,7 @@ end
# Gems required in various pipelines
group :development, :test, :monorepo do
gem 'gitlab-rspec', path: 'gems/gitlab-rspec'
+ gem 'rspec_flaky', path: 'gems/rspec_flaky'
end
group :test do
diff --git a/Gemfile.lock b/Gemfile.lock
index adbca941778c13dab16b7465f55bdff01e6fc28d..acfce81552ae69141df5f056cbb4dedd257d3066 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -41,6 +41,13 @@ PATH
diffy (~> 3.4)
oj (~> 3.13.16)
+PATH
+ remote: gems/rspec_flaky
+ specs:
+ rspec_flaky (0.1.0)
+ activesupport (>= 6.1, < 7.1)
+ rspec (~> 3.0)
+
PATH
remote: vendor/gems/attr_encrypted
specs:
@@ -1961,6 +1968,7 @@ DEPENDENCIES
rspec-parameterized (~> 1.0)
rspec-rails (~> 6.0.1)
rspec-retry (~> 0.6.2)
+ rspec_flaky!
rspec_junit_formatter
rspec_profiling (~> 0.0.6)
rubocop
diff --git a/doc/development/gems.md b/doc/development/gems.md
index a10fb86e88338288e4b16bfae509afabf4e1dbf6..0779cc5daa38d6f8f9b2f250f619b49bc82be00b 100644
--- a/doc/development/gems.md
+++ b/doc/development/gems.md
@@ -80,20 +80,29 @@ You can see example adding a new gem: [!121676](https://gitlab.com/gitlab-org/gi
1. Edit `gems/<name-of-gem>/<name-of-gem>.gemspec` and fill the details about the Gem as in the following example:
```ruby
+ # frozen_string_literal: true
+
+ require_relative "lib/name/of/gem/version"
+
Gem::Specification.new do |spec|
spec.name = "<name-of-gem>"
- spec.version = Gitlab::NameOfGem::VERSION
+ spec.version = Name::Of::Gem::Version::VERSION
spec.authors = ["group::tenant-scale"]
spec.email = ["engineering@gitlab.com"]
- spec.summary = "GitLab's RSpec extensions"
- spec.description = "A set of useful helpers to configure RSpec with various stubs and CI configs."
+ spec.summary = "Gem summary"
+ spec.description = "A more descriptive text about what the gem is doing."
spec.homepage = "https://gitlab.com/gitlab-org/gitlab/-/tree/master/gems/<name-of-gem>"
- spec.required_ruby_version = ">= 2.7"
+ spec.license = "MIT"
+ spec.required_ruby_version = ">= 3.0"
+ spec.metadata["rubygems_mfa_required"] = "true"
+
+ spec.files = Dir['lib/**/*.rb']
+ spec.require_paths = ["lib"]
end
```
-1. Update `gems/<name-of-gem>/.rubocop` with:
+1. Update `gems/<name-of-gem>/.rubocop.yml` with:
```yaml
inherit_from:
diff --git a/doc/development/testing_guide/flaky_tests.md b/doc/development/testing_guide/flaky_tests.md
index d111021f4863aedaa9fb8856cbe722b9bc77e8d5..ad3ce0da88168e4f7b7dbdcf14231742cc14bfec 100644
--- a/doc/development/testing_guide/flaky_tests.md
+++ b/doc/development/testing_guide/flaky_tests.md
@@ -210,10 +210,10 @@ Once a test is in quarantine, there are 3 choices:
## Automatic retries and flaky tests detection
-On our CI, we use [RSpec::Retry](https://github.com/NoRedInk/rspec-retry) to automatically retry a failing example a few
+On our CI, we use [`RSpec::Retry`](https://github.com/NoRedInk/rspec-retry) to automatically retry a failing example a few
times (see [`spec/spec_helper.rb`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/spec/spec_helper.rb) for the precise retries count).
-We also use a custom [`RspecFlaky::Listener`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/tooling/rspec_flaky/listener.rb).
+We also use a custom [`RspecFlaky::Listener`](https://gitlab.com/gitlab-org/gitlab/-/blob/master/gems/rspec_flaky/lib/rspec_flaky/listener.rb).
This listener runs in the `update-tests-metadata` job in `maintenance` scheduled pipelines
on the `master` branch, and saves flaky examples to `rspec/flaky/report-suite.json`.
The report file is then retrieved by the `retrieve-tests-metadata` job in all pipelines.
diff --git a/gems/activerecord-gitlab/.rubocop.yml b/gems/activerecord-gitlab/.rubocop.yml
index 8c99d7c47d18b8ea2f188ad9c7deef0161e3acb4..8c670b439d3eb02d2ce5f3847696991962de7614 100644
--- a/gems/activerecord-gitlab/.rubocop.yml
+++ b/gems/activerecord-gitlab/.rubocop.yml
@@ -1,6 +1,2 @@
inherit_from:
- ../config/rubocop.yml
-
-# FIXME
-Gitlab/RSpec/AvoidSetup:
- Enabled: false
diff --git a/gems/config/rubocop.yml b/gems/config/rubocop.yml
index 5422348ebeff7815fb8bfa06d6fb22a5fcf160d1..f591cde534fa5f28ef0fa1a503614a90d8035bff 100644
--- a/gems/config/rubocop.yml
+++ b/gems/config/rubocop.yml
@@ -43,6 +43,10 @@ Gitlab/DocUrl:
Gitlab/NamespacedClass:
Enabled: false
+# This cop doesn't make sense in the context of gems
+Gitlab/RSpec/AvoidSetup:
+ Enabled: false
+
# This cop doesn't make sense in the context of gems
Graphql/AuthorizeTypes:
Enabled: false
diff --git a/gems/gitlab-rspec/.rubocop.yml b/gems/gitlab-rspec/.rubocop.yml
index a2bdc5735c4d9ae2a8aea956777aa1fdc8d95b4b..38c0c592dad64319078ce50b7cac174b2c8ecb36 100644
--- a/gems/gitlab-rspec/.rubocop.yml
+++ b/gems/gitlab-rspec/.rubocop.yml
@@ -8,7 +8,3 @@ RSpec/InstanceVariable:
Gitlab/ChangeTimezone:
Exclude:
- spec/gitlab/rspec/time_travel_spec.rb
-
-# FIXME
-Gitlab/RSpec/AvoidSetup:
- Enabled: false
diff --git a/gems/gitlab-utils/.rubocop.yml b/gems/gitlab-utils/.rubocop.yml
index caaa0787be2dd89aebf2c2e60088ed14381118b0..eeafd850c9bb25552f7c72c86c0e68393f2d36c3 100644
--- a/gems/gitlab-utils/.rubocop.yml
+++ b/gems/gitlab-utils/.rubocop.yml
@@ -14,10 +14,6 @@ Lint/BinaryOperatorWithIdenticalOperands:
Style/NumericPredicate:
EnforcedStyle: comparison
-# FIXME
-Gitlab/RSpec/AvoidSetup:
- Enabled: false
-
# FIXME: When enabled, there's a spec failure in ee/spec/requests/api/graphql/mutations/merge_requests/update_approval_rule_spec.rb:51,
# due to the `default_value` of `remove_hidden_groups` set to `[]`, most probably instead of `false`, in ee/app/graphql/mutations/merge_requests/update_approval_rule.rb.
# The problem is that `Object#=~` exists (even though it's deprecated), hence calling it on an `Array` doesn't blow up, but `Array#match?` doesn't exist.
diff --git a/gems/gitlab-utils/Gemfile.lock b/gems/gitlab-utils/Gemfile.lock
index 2a571c95517150b9d9787b76d73634838c40373e..971d90ce146d016eb4f539aab40ed541c17e9626 100644
--- a/gems/gitlab-utils/Gemfile.lock
+++ b/gems/gitlab-utils/Gemfile.lock
@@ -2,6 +2,7 @@ PATH
remote: ../gitlab-rspec
specs:
gitlab-rspec (0.1.0)
+ activesupport (>= 6.1, < 7.1)
rspec (~> 3.0)
PATH
diff --git a/gems/rspec_flaky/.gitignore b/gems/rspec_flaky/.gitignore
new file mode 100644
index 0000000000000000000000000000000000000000..b04a8c840df1a534cfd67449e31919721b410986
--- /dev/null
+++ b/gems/rspec_flaky/.gitignore
@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status
diff --git a/gems/rspec_flaky/.gitlab-ci.yml b/gems/rspec_flaky/.gitlab-ci.yml
new file mode 100644
index 0000000000000000000000000000000000000000..41fac86e7a52e0a93de7e1f8125e0991f77232cc
--- /dev/null
+++ b/gems/rspec_flaky/.gitlab-ci.yml
@@ -0,0 +1,4 @@
+include:
+ - local: gems/gem.gitlab-ci.yml
+ inputs:
+ gem_name: "rspec_flaky"
diff --git a/gems/rspec_flaky/.rspec b/gems/rspec_flaky/.rspec
new file mode 100644
index 0000000000000000000000000000000000000000..34c5164d9b56c7d528f061c97f2d2fe02c834bdd
--- /dev/null
+++ b/gems/rspec_flaky/.rspec
@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper
diff --git a/gems/rspec_flaky/.rubocop.yml b/gems/rspec_flaky/.rubocop.yml
new file mode 100644
index 0000000000000000000000000000000000000000..62cb8a982c5d4b52d763ab36118a64fa6f76412f
--- /dev/null
+++ b/gems/rspec_flaky/.rubocop.yml
@@ -0,0 +1,13 @@
+inherit_from:
+ - ../config/rubocop.yml
+
+# FIXME once Gitlab::Json is in a gem
+Gitlab/Json:
+ Enabled: false
+
+# FIXME
+RSpec/MultipleMemoizedHelpers:
+ Enabled: false
+
+RSpec/VerifiedDoubles:
+ Enabled: false
diff --git a/gems/rspec_flaky/Gemfile b/gems/rspec_flaky/Gemfile
new file mode 100644
index 0000000000000000000000000000000000000000..90bf29fb6478586484484d5ab46256b9c26228ee
--- /dev/null
+++ b/gems/rspec_flaky/Gemfile
@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in rspec_flaky.gemspec
+gemspec
+
+gem "gitlab-rspec", "~> 0.1", path: "../gitlab-rspec"
diff --git a/gems/rspec_flaky/Gemfile.lock b/gems/rspec_flaky/Gemfile.lock
new file mode 100644
index 0000000000000000000000000000000000000000..547dc24e3756a95105b73a3581c349799c7cf3b3
--- /dev/null
+++ b/gems/rspec_flaky/Gemfile.lock
@@ -0,0 +1,126 @@
+PATH
+ remote: ../gitlab-rspec
+ specs:
+ gitlab-rspec (0.1.0)
+ activesupport (>= 6.1, < 7.1)
+ rspec (~> 3.0)
+
+PATH
+ remote: .
+ specs:
+ rspec_flaky (0.1.0)
+ activesupport (>= 6.1, < 7.1)
+ rspec (~> 3.0)
+
+GEM
+ remote: https://rubygems.org/
+ specs:
+ activesupport (7.0.6)
+ concurrent-ruby (~> 1.0, >= 1.0.2)
+ i18n (>= 1.6, < 2)
+ minitest (>= 5.1)
+ tzinfo (~> 2.0)
+ ast (2.4.2)
+ binding_of_caller (1.0.0)
+ debug_inspector (>= 0.0.1)
+ coderay (1.1.3)
+ concurrent-ruby (1.2.2)
+ debug_inspector (1.1.0)
+ diff-lcs (1.5.0)
+ gitlab-styles (10.1.0)
+ rubocop (~> 1.50.2)
+ rubocop-graphql (~> 0.18)
+ rubocop-performance (~> 1.15)
+ rubocop-rails (~> 2.17)
+ rubocop-rspec (~> 2.22)
+ i18n (1.14.1)
+ concurrent-ruby (~> 1.0)
+ json (2.6.3)
+ minitest (5.18.1)
+ parallel (1.23.0)
+ parser (3.2.2.3)
+ ast (~> 2.4.1)
+ racc
+ proc_to_ast (0.1.0)
+ coderay
+ parser
+ unparser
+ racc (1.7.1)
+ rack (3.0.8)
+ rainbow (3.1.1)
+ regexp_parser (2.8.1)
+ rexml (3.2.5)
+ rspec (3.12.0)
+ rspec-core (~> 3.12.0)
+ rspec-expectations (~> 3.12.0)
+ rspec-mocks (~> 3.12.0)
+ rspec-core (3.12.2)
+ rspec-support (~> 3.12.0)
+ rspec-expectations (3.12.3)
+ diff-lcs (>= 1.2.0, < 2.0)
+ rspec-support (~> 3.12.0)
+ rspec-mocks (3.12.5)
+ diff-lcs (>= 1.2.0, < 2.0)
+ rspec-support (~> 3.12.0)
+ rspec-parameterized (1.0.0)
+ rspec-parameterized-core (< 2)
+ rspec-parameterized-table_syntax (< 2)
+ rspec-parameterized-core (1.0.0)
+ parser
+ proc_to_ast
+ rspec (>= 2.13, < 4)
+ unparser
+ rspec-parameterized-table_syntax (1.0.0)
+ binding_of_caller
+ rspec-parameterized-core (< 2)
+ rspec-support (3.12.1)
+ rubocop (1.50.2)
+ json (~> 2.3)
+ parallel (~> 1.10)
+ parser (>= 3.2.0.0)
+ rainbow (>= 2.2.2, < 4.0)
+ regexp_parser (>= 1.8, < 3.0)
+ rexml (>= 3.2.5, < 4.0)
+ rubocop-ast (>= 1.28.0, < 2.0)
+ ruby-progressbar (~> 1.7)
+ unicode-display_width (>= 2.4.0, < 3.0)
+ rubocop-ast (1.29.0)
+ parser (>= 3.2.1.0)
+ rubocop-capybara (2.18.0)
+ rubocop (~> 1.41)
+ rubocop-factory_bot (2.23.1)
+ rubocop (~> 1.33)
+ rubocop-graphql (0.19.0)
+ rubocop (>= 0.87, < 2)
+ rubocop-performance (1.18.0)
+ rubocop (>= 1.7.0, < 2.0)
+ rubocop-ast (>= 0.4.0)
+ rubocop-rails (2.20.2)
+ activesupport (>= 4.2.0)
+ rack (>= 1.1)
+ rubocop (>= 1.33.0, < 2.0)
+ rubocop-rspec (2.22.0)
+ rubocop (~> 1.33)
+ rubocop-capybara (~> 2.17)
+ rubocop-factory_bot (~> 2.22)
+ ruby-progressbar (1.13.0)
+ tzinfo (2.0.6)
+ concurrent-ruby (~> 1.0)
+ unicode-display_width (2.4.2)
+ unparser (0.6.8)
+ diff-lcs (~> 1.3)
+ parser (>= 3.2.0)
+
+PLATFORMS
+ ruby
+
+DEPENDENCIES
+ gitlab-rspec (~> 0.1)!
+ gitlab-styles (~> 10.1.0)
+ rspec-parameterized (~> 1.0)
+ rspec_flaky!
+ rubocop (~> 1.50)
+ rubocop-rspec (~> 2.22)
+
+BUNDLED WITH
+ 2.4.14
diff --git a/gems/rspec_flaky/lib/rspec_flaky.rb b/gems/rspec_flaky/lib/rspec_flaky.rb
new file mode 100644
index 0000000000000000000000000000000000000000..90fc6b1dc490bbda65a16114529a4cb0163e4ef5
--- /dev/null
+++ b/gems/rspec_flaky/lib/rspec_flaky.rb
@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+require "rspec"
+require_relative "rspec_flaky/config"
+require_relative "rspec_flaky/listener"
+require_relative "rspec_flaky/version"
diff --git a/gems/rspec_flaky/lib/rspec_flaky/config.rb b/gems/rspec_flaky/lib/rspec_flaky/config.rb
new file mode 100644
index 0000000000000000000000000000000000000000..ca57d1e08be3ea7d5ed233474fe596a8d044753e
--- /dev/null
+++ b/gems/rspec_flaky/lib/rspec_flaky/config.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module RspecFlaky
+ class Config
+ def self.generate_report?
+ !!(ENV['FLAKY_RSPEC_GENERATE_REPORT'] =~ /1|true/)
+ end
+
+ def self.suite_flaky_examples_report_path
+ ENV['FLAKY_RSPEC_SUITE_REPORT_PATH'] || "rspec/flaky/suite-report.json"
+ end
+
+ def self.flaky_examples_report_path
+ ENV['FLAKY_RSPEC_REPORT_PATH'] || "rspec/flaky/report.json"
+ end
+
+ def self.new_flaky_examples_report_path
+ ENV['NEW_FLAKY_RSPEC_REPORT_PATH'] || "rspec/flaky/new-report.json"
+ end
+ end
+end
diff --git a/tooling/rspec_flaky/example.rb b/gems/rspec_flaky/lib/rspec_flaky/example.rb
similarity index 92%
rename from tooling/rspec_flaky/example.rb
rename to gems/rspec_flaky/lib/rspec_flaky/example.rb
index 8f369c99c5b14d10dd9d828e4f43e51d85bf09a4..4a128a151dc3224ecb25f6b156057f03ac5d0f59 100644
--- a/tooling/rspec_flaky/example.rb
+++ b/gems/rspec_flaky/lib/rspec_flaky/example.rb
@@ -15,7 +15,7 @@ def initialize(rspec_example)
end
def uid
- @uid ||= Digest::MD5.hexdigest("#{description}-#{file}")
+ @uid ||= Digest::MD5.hexdigest("#{description}-#{file}") # rubocop:disable Fips/MD5
end
def example_id
diff --git a/tooling/rspec_flaky/flaky_example.rb b/gems/rspec_flaky/lib/rspec_flaky/flaky_example.rb
similarity index 79%
rename from tooling/rspec_flaky/flaky_example.rb
rename to gems/rspec_flaky/lib/rspec_flaky/flaky_example.rb
index 3ce48ce1cd3ffa22106f5d39a2c1536f77b1dc80..35d1f34d2a28185ae812460690d9bc267a4cc64f 100644
--- a/tooling/rspec_flaky/flaky_example.rb
+++ b/gems/rspec_flaky/lib/rspec_flaky/flaky_example.rb
@@ -3,21 +3,21 @@
require 'ostruct'
module RspecFlaky
- ALLOWED_ATTRIBUTES = %i[
- example_id
- file
- line
- description
- first_flaky_at
- last_flaky_at
- last_flaky_job
- last_attempts_count
- flaky_reports
- feature_category
- ].freeze
-
# This represents a flaky RSpec example and is mainly meant to be saved in a JSON file
class FlakyExample
+ ALLOWED_ATTRIBUTES = %i[
+ example_id
+ file
+ line
+ description
+ first_flaky_at
+ last_flaky_at
+ last_flaky_job
+ last_attempts_count
+ flaky_reports
+ feature_category
+ ].freeze
+
def initialize(example_hash)
@attributes = {
first_flaky_at: Time.now,
@@ -43,9 +43,9 @@ def update!(example_hash)
attributes[:feature_category] = example_hash[:feature_category]
attributes[:last_attempts_count] = example_hash[:last_attempts_count] if example_hash[:last_attempts_count]
- if ENV['CI_JOB_URL']
- attributes[:last_flaky_job] = "#{ENV['CI_JOB_URL']}"
- end
+ return unless ENV['CI_JOB_URL']
+
+ attributes[:last_flaky_job] = (ENV['CI_JOB_URL']).to_s
end
def to_h
diff --git a/tooling/rspec_flaky/flaky_examples_collection.rb b/gems/rspec_flaky/lib/rspec_flaky/flaky_examples_collection.rb
similarity index 57%
rename from tooling/rspec_flaky/flaky_examples_collection.rb
rename to gems/rspec_flaky/lib/rspec_flaky/flaky_examples_collection.rb
index 019ebf703da20ecc7e0d6ae1fddbeb63eed267b4..f03fe63d11b509efca48d9d96b6ef301dea29408 100644
--- a/tooling/rspec_flaky/flaky_examples_collection.rb
+++ b/gems/rspec_flaky/lib/rspec_flaky/flaky_examples_collection.rb
@@ -8,15 +8,13 @@
module RspecFlaky
class FlakyExamplesCollection < SimpleDelegator
def initialize(collection = {})
- unless collection.is_a?(Hash)
- raise ArgumentError, "`collection` must be a Hash, #{collection.class} given!"
- end
+ raise ArgumentError, "`collection` must be a Hash, #{collection.class} given!" unless collection.is_a?(Hash)
collection_of_flaky_examples =
collection.map do |uid, example|
[
uid,
- RspecFlaky::FlakyExample.new(example.to_h.symbolize_keys)
+ FlakyExample.new(example.to_h.symbolize_keys)
]
end
@@ -24,13 +22,11 @@ def initialize(collection = {})
end
def to_h
- transform_values { |example| example.to_h }.deep_symbolize_keys
+ transform_values(&:to_h).deep_symbolize_keys
end
def -(other)
- unless other.respond_to?(:key)
- raise ArgumentError, "`other` must respond to `#key?`, #{other.class} does not!"
- end
+ raise ArgumentError, "`other` must respond to `#key?`, #{other.class} does not!" unless other.respond_to?(:key)
self.class.new(reject { |uid, _| other.key?(uid) })
end
diff --git a/tooling/rspec_flaky/listener.rb b/gems/rspec_flaky/lib/rspec_flaky/listener.rb
similarity index 56%
rename from tooling/rspec_flaky/listener.rb
rename to gems/rspec_flaky/lib/rspec_flaky/listener.rb
index 3431d814a8a4e8b9a6f28c678a8c4fabdce8997a..c2deb1c327a64766e0cc62d36c3ad16427b85489 100644
--- a/tooling/rspec_flaky/listener.rb
+++ b/gems/rspec_flaky/lib/rspec_flaky/listener.rb
@@ -17,32 +17,32 @@ class Listener
attr_reader :suite_flaky_examples, :flaky_examples
def initialize(suite_flaky_examples_json = nil)
- @flaky_examples = RspecFlaky::FlakyExamplesCollection.new
+ @flaky_examples = FlakyExamplesCollection.new
@suite_flaky_examples = init_suite_flaky_examples(suite_flaky_examples_json)
end
def example_passed(notification)
- current_example = RspecFlaky::Example.new(notification.example)
+ current_example = Example.new(notification.example)
return unless current_example.attempts > 1
- flaky_example = suite_flaky_examples.fetch(current_example.uid) { RspecFlaky::FlakyExample.new(current_example.to_h) }
+ flaky_example = suite_flaky_examples.fetch(current_example.uid) do
+ FlakyExample.new(current_example.to_h)
+ end
flaky_example.update!(current_example.to_h)
flaky_examples[current_example.uid] = flaky_example
end
def dump_summary(_)
- RspecFlaky::Report.new(flaky_examples).write(RspecFlaky::Config.flaky_examples_report_path)
- # write_report_file(flaky_examples, RspecFlaky::Config.flaky_examples_report_path)
+ Report.new(flaky_examples).write(Config.flaky_examples_report_path)
- if new_flaky_examples.any?
- rails_logger_warn("\nNew flaky examples detected:\n")
- rails_logger_warn(JSON.pretty_generate(new_flaky_examples.to_h)) # rubocop:disable Gitlab/Json
+ return unless new_flaky_examples.any?
- RspecFlaky::Report.new(new_flaky_examples).write(RspecFlaky::Config.new_flaky_examples_report_path)
- # write_report_file(new_flaky_examples, RspecFlaky::Config.new_flaky_examples_report_path)
- end
+ rails_logger_warn("\nNew flaky examples detected:\n")
+ rails_logger_warn(JSON.pretty_generate(new_flaky_examples.to_h))
+
+ Report.new(new_flaky_examples).write(Config.new_flaky_examples_report_path)
end
private
@@ -53,11 +53,11 @@ def new_flaky_examples
def init_suite_flaky_examples(suite_flaky_examples_json = nil)
if suite_flaky_examples_json
- RspecFlaky::Report.load_json(suite_flaky_examples_json).flaky_examples
+ Report.load_json(suite_flaky_examples_json).flaky_examples
else
- return {} unless File.exist?(RspecFlaky::Config.suite_flaky_examples_report_path)
+ return {} unless File.exist?(Config.suite_flaky_examples_report_path)
- RspecFlaky::Report.load(RspecFlaky::Config.suite_flaky_examples_report_path).flaky_examples
+ Report.load(Config.suite_flaky_examples_report_path).flaky_examples
end
end
diff --git a/tooling/rspec_flaky/report.rb b/gems/rspec_flaky/lib/rspec_flaky/report.rb
similarity index 72%
rename from tooling/rspec_flaky/report.rb
rename to gems/rspec_flaky/lib/rspec_flaky/report.rb
index 17dbb277446a52ef7eddab253a4f38d074b1be0b..3c8e1c933e1199b2a422924e93c37fa44fe4edba 100644
--- a/tooling/rspec_flaky/report.rb
+++ b/gems/rspec_flaky/lib/rspec_flaky/report.rb
@@ -19,12 +19,13 @@ def self.load(file_path)
end
def self.load_json(json)
- new(RspecFlaky::FlakyExamplesCollection.new(JSON.parse(json)))
+ new(FlakyExamplesCollection.new(JSON.parse(json)))
end
def initialize(flaky_examples)
- unless flaky_examples.is_a?(RspecFlaky::FlakyExamplesCollection)
- raise ArgumentError, "`flaky_examples` must be a RspecFlaky::FlakyExamplesCollection, #{flaky_examples.class} given!"
+ unless flaky_examples.is_a?(FlakyExamplesCollection)
+ raise ArgumentError,
+ "`flaky_examples` must be a RspecFlaky::FlakyExamplesCollection, #{flaky_examples.class} given!"
end
@flaky_examples = flaky_examples
@@ -32,13 +33,13 @@ def initialize(flaky_examples)
end
def write(file_path)
- unless RspecFlaky::Config.generate_report?
+ unless Config.generate_report?
Kernel.warn "! Generating reports is disabled. To enable it, please set the `FLAKY_RSPEC_GENERATE_REPORT=1` !"
return
end
report_path_dir = File.dirname(file_path)
- FileUtils.mkdir_p(report_path_dir) unless Dir.exist?(report_path_dir)
+ FileUtils.mkdir_p(report_path_dir)
File.write(file_path, JSON.pretty_generate(flaky_examples.to_h))
end
@@ -51,7 +52,7 @@ def prune_outdated(days: OUTDATED_DAYS_THRESHOLD)
last_flaky_at && last_flaky_at.to_i < outdated_date_threshold.to_i
end
- self.class.new(RspecFlaky::FlakyExamplesCollection.new(recent_flaky_examples))
+ self.class.new(FlakyExamplesCollection.new(recent_flaky_examples))
end
end
end
diff --git a/gems/rspec_flaky/lib/rspec_flaky/version.rb b/gems/rspec_flaky/lib/rspec_flaky/version.rb
new file mode 100644
index 0000000000000000000000000000000000000000..ec507d734c873e78ca2342c820ac7b5a49a4a0fa
--- /dev/null
+++ b/gems/rspec_flaky/lib/rspec_flaky/version.rb
@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module RspecFlaky
+ module Version
+ VERSION = "0.1.0"
+ end
+end
diff --git a/gems/rspec_flaky/rspec_flaky.gemspec b/gems/rspec_flaky/rspec_flaky.gemspec
new file mode 100644
index 0000000000000000000000000000000000000000..5c0a434218f1d8f06cd49cef89f2135bd5431755
--- /dev/null
+++ b/gems/rspec_flaky/rspec_flaky.gemspec
@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require_relative "lib/rspec_flaky/version"
+
+Gem::Specification.new do |spec|
+ spec.name = "rspec_flaky"
+ spec.version = RspecFlaky::Version::VERSION
+ spec.authors = ["Engineering Productivity"]
+ spec.email = ["quality@gitlab.com"]
+
+ spec.summary = "GitLab's RSpec Flaky test detector"
+ spec.description =
+ "This gem provide an RSpec listener that allows to detect flaky examples. See " \
+ "https://docs.gitlab.com/ee/development/testing_guide/flaky_tests.html#automatic-retries-and-flaky-tests-detection."
+ spec.homepage = "https://gitlab.com/gitlab-org/gitlab/-/tree/master/gems/rspec_flaky"
+ spec.license = "MIT"
+ spec.required_ruby_version = ">= 3.0"
+ spec.metadata["rubygems_mfa_required"] = "true"
+
+ spec.files = Dir["lib/**/*.rb"]
+ spec.require_paths = ["lib"]
+
+ spec.add_runtime_dependency "activesupport", ">= 6.1", "< 7.1"
+ spec.add_runtime_dependency "rspec", "~> 3.0"
+
+ spec.add_development_dependency "gitlab-styles", "~> 10.1.0"
+ spec.add_development_dependency "rspec-parameterized", "~> 1.0"
+ spec.add_development_dependency "rubocop", "~> 1.50"
+ spec.add_development_dependency "rubocop-rspec", "~> 2.22"
+end
diff --git a/spec/tooling/rspec_flaky/config_spec.rb b/gems/rspec_flaky/spec/rspec_flaky/config_spec.rb
similarity index 88%
rename from spec/tooling/rspec_flaky/config_spec.rb
rename to gems/rspec_flaky/spec/rspec_flaky/config_spec.rb
index 8ba4dd1640e9071a44d84c62643f54123f8b4d02..827249efefa027488f25e2341e9242644e1dd609 100644
--- a/spec/tooling/rspec_flaky/config_spec.rb
+++ b/gems/rspec_flaky/spec/rspec_flaky/config_spec.rb
@@ -1,9 +1,6 @@
# frozen_string_literal: true
-require 'rspec-parameterized'
-require 'gitlab/rspec/all'
-
-require_relative '../../../tooling/rspec_flaky/config'
+require 'rspec_flaky/config'
RSpec.describe RspecFlaky::Config, :aggregate_failures do
include StubENV
@@ -14,10 +11,6 @@
stub_env('FLAKY_RSPEC_SUITE_REPORT_PATH', nil)
stub_env('FLAKY_RSPEC_REPORT_PATH', nil)
stub_env('NEW_FLAKY_RSPEC_REPORT_PATH', nil)
- # Ensure the behavior is the same locally and on CI (where Rails is defined since we run this test as part of the whole suite), i.e. Rails isn't defined
- allow(described_class).to receive(:rails_path).and_wrap_original do |method, path|
- path
- end
end
describe '.generate_report?' do
diff --git a/spec/tooling/rspec_flaky/example_spec.rb b/gems/rspec_flaky/spec/rspec_flaky/example_spec.rb
similarity index 96%
rename from spec/tooling/rspec_flaky/example_spec.rb
rename to gems/rspec_flaky/spec/rspec_flaky/example_spec.rb
index d001ed324440f233ca8067dfc34d4847bfd637e0..64d65c0e1705ca4d6bf55eac51ccd0ba374c2d10 100644
--- a/spec/tooling/rspec_flaky/example_spec.rb
+++ b/gems/rspec_flaky/spec/rspec_flaky/example_spec.rb
@@ -1,6 +1,6 @@
# frozen_string_literal: true
-require_relative '../../../tooling/rspec_flaky/example'
+require 'rspec_flaky/example'
RSpec.describe RspecFlaky::Example do
let(:example_attrs) do
@@ -45,7 +45,7 @@
describe '#uid' do
it 'returns a hash of the full description' do
- expect(subject.uid).to eq(Digest::MD5.hexdigest("#{subject.description}-#{subject.file}"))
+ expect(subject.uid).to eq(Digest::MD5.hexdigest("#{subject.description}-#{subject.file}")) # rubocop:disable Fips/MD5
end
end
diff --git a/spec/tooling/rspec_flaky/flaky_example_spec.rb b/gems/rspec_flaky/spec/rspec_flaky/flaky_example_spec.rb
similarity index 95%
rename from spec/tooling/rspec_flaky/flaky_example_spec.rb
rename to gems/rspec_flaky/spec/rspec_flaky/flaky_example_spec.rb
index c100756d7342665586ba11522840ccfd9b0adefe..9a26d11d30cf1c5e234625cc919b97e185833e63 100644
--- a/spec/tooling/rspec_flaky/flaky_example_spec.rb
+++ b/gems/rspec_flaky/spec/rspec_flaky/flaky_example_spec.rb
@@ -1,11 +1,8 @@
# frozen_string_literal: true
-require 'gitlab/rspec/all'
-
-require_relative '../../../tooling/rspec_flaky/flaky_example'
+require 'rspec_flaky/flaky_example'
RSpec.describe RspecFlaky::FlakyExample, :aggregate_failures do
- include ActiveSupport::Testing::TimeHelpers
include StubENV
let(:example_attrs) do
@@ -87,7 +84,7 @@
end
it 'updates the flaky_reports' do
- expected_flaky_reports = flaky_example.to_h[:first_flaky_at] ? flaky_example.to_h[:flaky_reports] + 1 : 1
+ expected_flaky_reports = flaky_example.to_h[:first_flaky_at] ? flaky_example.to_h[:flaky_reports] + 1 : 1 # rubocop:disable RSpec/AvoidConditionalStatements
expect { flaky_example.update!(example_attrs) }.to change { flaky_example.to_h[:flaky_reports] }.by(1)
expect(flaky_example.to_h[:flaky_reports]).to eq(expected_flaky_reports)
diff --git a/spec/tooling/rspec_flaky/flaky_examples_collection_spec.rb b/gems/rspec_flaky/spec/rspec_flaky/flaky_examples_collection_spec.rb
similarity index 85%
rename from spec/tooling/rspec_flaky/flaky_examples_collection_spec.rb
rename to gems/rspec_flaky/spec/rspec_flaky/flaky_examples_collection_spec.rb
index 8304bcb426ea7ef10ee306ecbc4c6482cf413daa..260ebc7219248608bd00a765efd89b4756d0b4d0 100644
--- a/spec/tooling/rspec_flaky/flaky_examples_collection_spec.rb
+++ b/gems/rspec_flaky/spec/rspec_flaky/flaky_examples_collection_spec.rb
@@ -1,8 +1,6 @@
# frozen_string_literal: true
-require 'gitlab/rspec/all'
-
-require_relative '../../../tooling/rspec_flaky/flaky_examples_collection'
+require 'rspec_flaky/flaky_examples_collection'
RSpec.describe RspecFlaky::FlakyExamplesCollection, :aggregate_failures, :freeze_time do
let(:collection_hash) do
@@ -45,7 +43,9 @@
end
it 'does not accept anything else' do
- expect { described_class.new([1, 2, 3]) }.to raise_error(ArgumentError, "`collection` must be a Hash, Array given!")
+ expect do
+ described_class.new([1, 2, 3])
+ end.to raise_error(ArgumentError, "`collection` must be a Hash, Array given!")
end
end
@@ -79,7 +79,9 @@
it 'fails if the given collection does not respond to `#key?`' do
collection = described_class.new(collection_hash)
- expect { collection - [1, 2, 3] }.to raise_error(ArgumentError, "`other` must respond to `#key?`, Array does not!")
+ expect do
+ collection - [1, 2, 3]
+ end.to raise_error(ArgumentError, "`other` must respond to `#key?`, Array does not!")
end
end
end
diff --git a/spec/tooling/rspec_flaky/listener_spec.rb b/gems/rspec_flaky/spec/rspec_flaky/listener_spec.rb
similarity index 90%
rename from spec/tooling/rspec_flaky/listener_spec.rb
rename to gems/rspec_flaky/spec/rspec_flaky/listener_spec.rb
index d1ed84dad76415e3a47cd72cea67178988af16ca..cbc5422a76329b5731cb0f156b2b712186373b55 100644
--- a/spec/tooling/rspec_flaky/listener_spec.rb
+++ b/gems/rspec_flaky/spec/rspec_flaky/listener_spec.rb
@@ -1,11 +1,8 @@
# frozen_string_literal: true
-require 'gitlab/rspec/all'
-
-require_relative '../../../tooling/rspec_flaky/listener'
+require 'rspec_flaky/listener'
RSpec.describe RspecFlaky::Listener, :aggregate_failures do
- include ActiveSupport::Testing::TimeHelpers
include StubENV
let(:already_flaky_example_uid) { '6e869794f4cfd2badd93eb68719371d1' }
@@ -37,7 +34,10 @@
}
end
- let(:already_flaky_example) { RspecFlaky::FlakyExample.new(suite_flaky_example_report[already_flaky_example_uid]) }
+ let(:already_flaky_example) do
+ RspecFlaky::FlakyExample.new(suite_flaky_example_report[already_flaky_example_uid])
+ end
+
let(:new_example_attrs) do
{
id: 'spec/foo/baz_spec.rb:3',
@@ -79,22 +79,23 @@
stub_env('FLAKY_RSPEC_SUITE_REPORT_PATH', report_file_path)
end
- context 'and report file exists' do
+ context 'when report file exists' do
before do
- expect(File).to receive(:exist?).with(report_file_path).and_return(true)
+ allow(File).to receive(:exist?).with(report_file_path).and_return(true)
end
it 'delegates the load to RspecFlaky::Report' do
- report = RspecFlaky::Report.new(RspecFlaky::FlakyExamplesCollection.new(suite_flaky_example_report))
+ report = RspecFlaky::Report
+ .new(RspecFlaky::FlakyExamplesCollection.new(suite_flaky_example_report))
expect(RspecFlaky::Report).to receive(:load).with(report_file_path).and_return(report)
expect(described_class.new.suite_flaky_examples.to_h).to eq(report.flaky_examples.to_h)
end
end
- context 'and report file does not exist' do
+ context 'when report file does not exist' do
before do
- expect(File).to receive(:exist?).with(report_file_path).and_return(false)
+ allow(File).to receive(:exist?).with(report_file_path).and_return(false)
end
it 'return an empty hash' do
@@ -217,7 +218,8 @@
expect(RspecFlaky::Report).to receive(:new).with(listener.flaky_examples).and_return(report1)
expect(report1).to receive(:write).with(RspecFlaky::Config.flaky_examples_report_path)
- expect(RspecFlaky::Report).to receive(:new).with(listener.__send__(:new_flaky_examples)).and_return(report2)
+ expect(RspecFlaky::Report)
+ .to receive(:new).with(listener.__send__(:new_flaky_examples)).and_return(report2)
expect(report2).to receive(:write).with(RspecFlaky::Config.new_flaky_examples_report_path)
listener.dump_summary(nil)
diff --git a/spec/tooling/rspec_flaky/report_spec.rb b/gems/rspec_flaky/spec/rspec_flaky/report_spec.rb
similarity index 84%
rename from spec/tooling/rspec_flaky/report_spec.rb
rename to gems/rspec_flaky/spec/rspec_flaky/report_spec.rb
index 543a69cd8ee1379f476eb0e4e93edfa4dd7b2ff9..e1e9bd6a7b1819efc25550035a1d338e088f92ae 100644
--- a/spec/tooling/rspec_flaky/report_spec.rb
+++ b/gems/rspec_flaky/spec/rspec_flaky/report_spec.rb
@@ -1,16 +1,16 @@
# frozen_string_literal: true
require 'tempfile'
-require 'gitlab/rspec/all'
-require_relative '../../../tooling/rspec_flaky/report'
+require 'rspec_flaky/report'
RSpec.describe RspecFlaky::Report, :aggregate_failures, :freeze_time do
let(:thirty_one_days) { 3600 * 24 * 31 }
let(:collection_hash) do
{
a: { example_id: 'spec/foo/bar_spec.rb:2' },
- b: { example_id: 'spec/foo/baz_spec.rb:3', first_flaky_at: (Time.now - thirty_one_days).to_s, last_flaky_at: (Time.now - thirty_one_days).to_s }
+ b: { example_id: 'spec/foo/baz_spec.rb:3', first_flaky_at: (Time.now - thirty_one_days).to_s,
+ last_flaky_at: (Time.now - thirty_one_days).to_s }
}
end
@@ -41,7 +41,7 @@
describe '.load' do
let!(:report_file) do
Tempfile.new(%w[rspec_flaky_report .json]).tap do |f|
- f.write(JSON.pretty_generate(suite_flaky_example_report)) # rubocop:disable Gitlab/Json
+ f.write(JSON.pretty_generate(suite_flaky_example_report))
f.rewind
end
end
@@ -58,7 +58,7 @@
describe '.load_json' do
let(:report_json) do
- JSON.pretty_generate(suite_flaky_example_report) # rubocop:disable Gitlab/Json
+ JSON.pretty_generate(suite_flaky_example_report)
end
it 'loads the report file' do
@@ -72,7 +72,11 @@
end
it 'does not accept anything else' do
- expect { described_class.new([1, 2, 3]) }.to raise_error(ArgumentError, "`flaky_examples` must be a RspecFlaky::FlakyExamplesCollection, Array given!")
+ expect do
+ described_class.new([1, 2,
+ 3])
+ end.to raise_error(ArgumentError,
+ "`flaky_examples` must be a RspecFlaky::FlakyExamplesCollection, Array given!")
end
end
@@ -84,11 +88,11 @@
let(:report_file_path) { File.join('tmp', 'rspec_flaky_report.json') }
before do
- FileUtils.rm(report_file_path) if File.exist?(report_file_path)
+ FileUtils.rm_f(report_file_path)
end
after do
- FileUtils.rm(report_file_path) if File.exist?(report_file_path)
+ FileUtils.rm_f(report_file_path)
end
context 'when RspecFlaky::Config.generate_report? is false' do
@@ -113,7 +117,7 @@
expect(File.exist?(report_file_path)).to be(true)
expect(File.read(report_file_path))
- .to eq(JSON.pretty_generate(report.flaky_examples.to_h)) # rubocop:disable Gitlab/Json
+ .to eq(JSON.pretty_generate(report.flaky_examples.to_h))
end
end
end
diff --git a/gems/rspec_flaky/spec/spec_helper.rb b/gems/rspec_flaky/spec/spec_helper.rb
new file mode 100644
index 0000000000000000000000000000000000000000..72d48ee6e6384c8469c806a0d402bbb9f665c482
--- /dev/null
+++ b/gems/rspec_flaky/spec/spec_helper.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require "rspec-parameterized"
+require "gitlab/rspec/all"
+require "rspec_flaky"
+
+RSpec.configure do |config|
+ # Enable flags like --only-failures and --next-failure
+ config.example_status_persistence_file_path = ".rspec_status"
+
+ # Disable RSpec exposing methods globally on `Module` and `main`
+ config.disable_monkey_patching!
+
+ config.expect_with :rspec do |c|
+ c.syntax = :expect
+ end
+end
diff --git a/scripts/flaky_examples/prune-old-flaky-examples b/scripts/flaky_examples/prune-old-flaky-examples
index a5b50a7e8eac22d3415289d9f830155fd1f07834..24964cbf4dfaf524fbc74d3136a0785e9ba54189 100755
--- a/scripts/flaky_examples/prune-old-flaky-examples
+++ b/scripts/flaky_examples/prune-old-flaky-examples
@@ -1,15 +1,15 @@
#!/usr/bin/env ruby
# frozen_string_literal: true
-# tooling/rspec_flaky/flaky_examples_collection.rb is requiring
-# `active_support/hash_with_indifferent_access`, and we install the `activesupport`
-# gem manually on the CI
-require 'rubygems'
-require_relative '../../tooling/rspec_flaky/report'
+require 'bundler/inline'
+
+gemfile do
+ gem 'rspec_flaky', path: 'gems/rspec_flaky'
+end
report_file = ARGV.shift
unless report_file
- puts 'usage: prune-old-flaky-specs <report-file> <new-report-file>'
+ puts "usage: #{__FILE__} <report-file> <new-report-file>"
exit 1
end
@@ -20,5 +20,6 @@ puts "Current report has #{report.size} entries."
new_report = report.prune_outdated
-puts "New report has #{new_report.size} entries: #{report.size - new_report.size} entries older than #{RspecFlaky::Report::OUTDATED_DAYS_THRESHOLD} days were removed."
+puts "New report has #{new_report.size} entries: #{report.size - new_report.size} entries older than " \
+ "#{RspecFlaky::Report::OUTDATED_DAYS_THRESHOLD} days were removed."
puts "Saved #{new_report_file}." if new_report.write(new_report_file)
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index 18dba7e68ad382cade8f5f37d41c3f14b05b4e4d..eb7796cb55abe916acb3227ca3513d53912dc536 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -38,6 +38,8 @@
require 'parslet/rig/rspec'
require 'axe-rspec'
+require 'rspec_flaky'
+
rspec_profiling_is_configured =
ENV['RSPEC_PROFILING_POSTGRES_URL'].present? ||
ENV['RSPEC_PROFILING']
@@ -228,11 +230,7 @@
config.exceptions_to_hard_fail = [DeprecationToolkitEnv::DeprecationBehaviors::SelectiveRaise::RaiseDisallowedDeprecation]
end
- require_relative '../tooling/rspec_flaky/config'
-
if RspecFlaky::Config.generate_report?
- require_relative '../tooling/rspec_flaky/listener'
-
config.reporter.register_listener(
RspecFlaky::Listener.new,
:example_passed,
diff --git a/spec/support/rspec_order_todo.yml b/spec/support/rspec_order_todo.yml
index 0f91ff5f12cf903699559853a8c08401e9080718..d45595d5fc86a851770dc529f3947050357ca790 100644
--- a/spec/support/rspec_order_todo.yml
+++ b/spec/support/rspec_order_todo.yml
@@ -9748,12 +9748,6 @@
- './spec/tooling/lib/tooling/test_map_packer_spec.rb'
- './spec/tooling/merge_request_spec.rb'
- './spec/tooling/quality/test_level_spec.rb'
-- './spec/tooling/rspec_flaky/config_spec.rb'
-- './spec/tooling/rspec_flaky/example_spec.rb'
-- './spec/tooling/rspec_flaky/flaky_examples_collection_spec.rb'
-- './spec/tooling/rspec_flaky/flaky_example_spec.rb'
-- './spec/tooling/rspec_flaky/listener_spec.rb'
-- './spec/tooling/rspec_flaky/report_spec.rb'
- './spec/uploaders/attachment_uploader_spec.rb'
- './spec/uploaders/avatar_uploader_spec.rb'
- './spec/uploaders/ci/pipeline_artifact_uploader_spec.rb'
diff --git a/tooling/rspec_flaky/config.rb b/tooling/rspec_flaky/config.rb
deleted file mode 100644
index 0e36e985aad59ebb9b6907642b6092c8134c7ede..0000000000000000000000000000000000000000
--- a/tooling/rspec_flaky/config.rb
+++ /dev/null
@@ -1,27 +0,0 @@
-# frozen_string_literal: true
-
-module RspecFlaky
- class Config
- def self.generate_report?
- !!(ENV['FLAKY_RSPEC_GENERATE_REPORT'] =~ /1|true/)
- end
-
- def self.suite_flaky_examples_report_path
- ENV['FLAKY_RSPEC_SUITE_REPORT_PATH'] || rails_path("rspec/flaky/suite-report.json")
- end
-
- def self.flaky_examples_report_path
- ENV['FLAKY_RSPEC_REPORT_PATH'] || rails_path("rspec/flaky/report.json")
- end
-
- def self.new_flaky_examples_report_path
- ENV['NEW_FLAKY_RSPEC_REPORT_PATH'] || rails_path("rspec/flaky/new-report.json")
- end
-
- def self.rails_path(path)
- return path unless defined?(Rails)
-
- Rails.root.join(path)
- end
- end
-end