diff --git a/qa/.gitignore b/qa/.gitignore
index 537798a0bb6ad3e2847e869b1065e5ebddef3657..97fb08534795ae600b66cff90d17349c1e110cb9 100644
--- a/qa/.gitignore
+++ b/qa/.gitignore
@@ -1,5 +1,4 @@
reports/
-no_of_examples/
.ruby-version
.tool-versions
.ruby-gemset
diff --git a/qa/.rspec_parallel b/qa/.rspec_parallel
deleted file mode 100644
index e5927927eaa76d0203c8983f16a37be743d53fd0..0000000000000000000000000000000000000000
--- a/qa/.rspec_parallel
+++ /dev/null
@@ -1,5 +0,0 @@
---color
---format documentation
---format ParallelTests::RSpec::SummaryLogger --out tmp/spec_summary.log
---format ParallelTests::RSpec::RuntimeLogger --out tmp/parallel_runtime_rspec.log
---require spec_helper
diff --git a/qa/qa/runtime/env.rb b/qa/qa/runtime/env.rb
index bba5cfcdac593565fc4a7433eca179638e365cce..84f5f56a5830bd3fde9fef76f1bd8267eb365654 100644
--- a/qa/qa/runtime/env.rb
+++ b/qa/qa/runtime/env.rb
@@ -2,6 +2,7 @@
require 'active_support/deprecation'
require 'uri'
+require 'etc'
module QA
module Runtime
@@ -734,6 +735,16 @@ def rspec_retried?
enabled?(ENV['QA_RSPEC_RETRIED'], default: false)
end
+ def parallel_processes
+ ENV.fetch('QA_PARALLEL_PROCESSES') do
+ [Etc.nprocessors / 2, 1].max
+ end.to_i
+ end
+
+ def parallel_run?
+ enabled?(ENV["QA_PARALLEL_RUN"], default: false)
+ end
+
private
def remote_grid_credentials
diff --git a/qa/qa/runtime/logger.rb b/qa/qa/runtime/logger.rb
index 7e78ba470d8287903faf751d685118f9a4fe95b3..8728dd2010ab035a3db6a171520ae6411e84eec5 100644
--- a/qa/qa/runtime/logger.rb
+++ b/qa/qa/runtime/logger.rb
@@ -10,12 +10,26 @@ class << self
def logger
@logger ||= Gitlab::QA::TestLogger.logger(
level: Gitlab::QA::Runtime::Env.log_level,
- source: 'QA Tests',
- path: File.expand_path('../../tmp', __dir__)
+ source: logger_source,
+ path: log_path
)
end
delegate :debug, :info, :warn, :error, :fatal, :unknown, to: :logger
+
+ private
+
+ def logger_source
+ if ENV['TEST_ENV_NUMBER']
+ "QA Tests ENV-#{ENV['TEST_ENV_NUMBER']}"
+ else
+ "QA Tests"
+ end
+ end
+
+ def log_path
+ File.expand_path('../../tmp', __dir__)
+ end
end
end
end
diff --git a/qa/qa/specs/knapsack_runner.rb b/qa/qa/specs/knapsack_runner.rb
index 0c4f938ee28b16df68e774463b2888103aeeeaad..b8914ca3846ae3137bb49656e3bc1a8f5c64ec5f 100644
--- a/qa/qa/specs/knapsack_runner.rb
+++ b/qa/qa/specs/knapsack_runner.rb
@@ -3,7 +3,7 @@
module QA
module Specs
class KnapsackRunner
- def self.run(args)
+ def self.run(args, parallel: false)
QA::Support::KnapsackReport.configure!
allocator = Knapsack::AllocatorBuilder.new(Knapsack::Adapters::RSpecAdapter).allocator
@@ -14,6 +14,12 @@ def self.run(args)
Knapsack.logger.info 'Leftover specs:'
Knapsack.logger.info allocator.leftover_node_tests
+ if parallel
+ rspec_args = args.reject { |arg| arg == "--" || arg.start_with?("qa/specs/features") }
+ run_args = [*rspec_args, '--', *allocator.node_tests]
+ return ParallelRunner.run(run_args)
+ end
+
status = RSpec::Core::Runner.run([*args, '--', *allocator.node_tests])
yield status if block_given?
status
diff --git a/qa/qa/specs/parallel_runner.rb b/qa/qa/specs/parallel_runner.rb
index b92fdb610b6951762034e05c72001d552ef1b376..b3349b985b599096a7964c30db78554299331ebc 100644
--- a/qa/qa/specs/parallel_runner.rb
+++ b/qa/qa/specs/parallel_runner.rb
@@ -1,31 +1,52 @@
# frozen_string_literal: true
-require 'open3'
+require "parallel_tests"
+require "etc"
module QA
module Specs
- module ParallelRunner
- module_function
+ class ParallelRunner
+ class << self
+ def run(rspec_args)
+ used_processes = Runtime::Env.parallel_processes
- def run(args)
- unless args.include?('--')
- index = args.index { |opt| opt.include?('features') }
+ args = [
+ "--type", "rspec",
+ "-n", used_processes.to_s,
+ "--serialize-stdout",
+ '--first-is-1',
+ "--combine-stderr"
+ ]
- args.insert(index, '--') if index
+ unless rspec_args.include?('--')
+ index = rspec_args.index { |opt| opt.include?("qa/specs/features") }
+
+ rspec_args.insert(index, '--') if index
+ end
+
+ args.push("--", *rspec_args) unless rspec_args.empty?
+
+ set_environment!
+ perform_global_setup!
+
+ ParallelTests::CLI.new.run(args)
end
- env = {}
- Runtime::Env::ENV_VARIABLES.each_key do |key|
- env[key] = ENV[key] if ENV[key]
+ private
+
+ def perform_global_setup!
+ Runtime::Browser.configure!
+ Runtime::Release.perform_before_hooks
end
- env['QA_RUNTIME_SCENARIO_ATTRIBUTES'] = Runtime::Scenario.attributes.to_json
- env['GITLAB_QA_ACCESS_TOKEN'] = Runtime::API::Client.new(:gitlab).personal_access_token unless env['GITLAB_QA_ACCESS_TOKEN']
- cmd = "bundle exec parallel_test -t rspec --combine-stderr --serialize-stdout -- #{args.flatten.join(' ')}"
- ::Open3.popen2e(env, cmd) do |_, out, wait|
- out.each { |line| puts line }
+ def set_environment!
+ ENV.store("NO_KNAPSACK", "true")
+ ENV.store("QA_PARALLEL_RUN", "true")
+
+ return if ENV["QA_GITLAB_URL"].present?
- exit wait.value.exitstatus
+ Support::GitlabAddress.define_gitlab_address_attribute!
+ ENV.store("QA_GITLAB_URL", Support::GitlabAddress.address_with_port(with_default_port: false))
end
end
end
diff --git a/qa/qa/specs/runner.rb b/qa/qa/specs/runner.rb
index 5650587f573b3b09a377aa105605dd5676cb02cf..431a7e3fc10a8639a299e90ad70f57c7ae1e8833 100644
--- a/qa/qa/specs/runner.rb
+++ b/qa/qa/specs/runner.rb
@@ -2,6 +2,7 @@
require 'rspec/core'
require 'rspec/expectations'
+require 'tempfile'
module QA
module Specs
@@ -47,51 +48,57 @@ def rspec_tags
end
def perform
- args = []
- args.push('--tty') if tty
- args.push(rspec_tags)
- args.push(options)
+ args = build_initial_args
+ # use options from default .rspec file for metadata only runs
+ configure_default_formatters!(args) unless metadata_run?
+ args.push(DEFAULT_TEST_PATH_ARGS) unless custom_test_paths?
- unless Runtime::Env.knapsack? || options.any? { |opt| opt.include?('features') }
- args.push(DEFAULT_TEST_PATH_ARGS)
+ run_rspec(args)
+ end
+
+ private
+
+ delegate :rspec_retried?, :parallel_run?, to: Runtime::Env
+
+ def build_initial_args
+ [].tap do |args|
+ args.push('--tty') if tty
+ args.push(rspec_tags)
+ args.push(options)
end
+ end
- if Runtime::Env.knapsack?
- KnapsackRunner.run(args.flatten) { |status| abort if status.nonzero? }
- elsif Runtime::Scenario.attributes[:parallel]
- ParallelRunner.run(args.flatten)
- elsif Runtime::Scenario.attributes[:loop]
- LoopRunner.run(args.flatten)
- elsif Runtime::Scenario.attributes[:count_examples_only]
+ def run_rspec(args)
+ if Runtime::Scenario.attributes[:count_examples_only]
count_examples_only(args)
elsif Runtime::Scenario.attributes[:test_metadata_only]
test_metadata_only(args)
+ elsif Runtime::Env.knapsack?
+ KnapsackRunner.run(args.flatten, parallel: parallel_run?) { |status| abort if status.nonzero? }
+ elsif !rspec_retried? && parallel_run?
+ ParallelRunner.run(args.flatten)
+ elsif Runtime::Scenario.attributes[:loop]
+ LoopRunner.run(args.flatten)
else
RSpec::Core::Runner.run(args.flatten, *DEFAULT_STD_ARGS).tap { |status| abort if status.nonzero? }
end
end
- private
-
def count_examples_only(args)
args.unshift('--dry-run')
out = StringIO.new
+ err = StringIO.new
- RSpec::Core::Runner.run(args.flatten, $stderr, out).tap do |status|
- abort if status.nonzero?
- end
+ total_examples = Tempfile.open('test-metadata.json') do |file|
+ RSpec.configure { |config| config.add_formatter(QA::Support::JsonFormatter, file.path) }
+ RSpec::Core::Runner.run(args.flatten, err, out).tap { |status| abort if status.nonzero? }
- begin
- total_examples = out.string.match(/(\d+) examples?,/)[1]
- rescue StandardError
- raise RegexMismatchError, 'Rspec output did not match regex'
+ JSON.load_file(file, symbolize_names: true).dig(:summary, :example_count)
end
- filename = build_filename
-
- File.open(filename, 'w') { |f| f.write(total_examples) } if total_examples.to_i > 0
-
- $stdout.puts total_examples
+ puts total_examples
+ rescue StandardError => e
+ raise e, "Failed to detect example count, error: '#{e}'.\nOut: '#{out.string}'\nErr: #{err.string}"
end
def test_metadata_only(args)
@@ -111,23 +118,28 @@ def test_metadata_only(args)
$stdout.puts "Saved to file: #{output_file}"
end
- def build_filename
- filename = Runtime::Scenario.klass.split('::').last(3).join('_').downcase
+ def configure_default_formatters!(args)
+ default_formatter_file_name = "tmp/rspec-#{ENV['CI_JOB_ID'] || 'local'}-retried-#{rspec_retried?}"
+ filename = if parallel_run?
+ rspec_retried? ? default_formatter_file_name : "#{default_formatter_file_name}-$TEST_ENV_NUMBER"
+ else
+ default_formatter_file_name
+ end
- tags = []
- tag_opts = %w[--tag -t]
- options.reduce do |before, after|
- tags << after if tag_opts.include?(before)
- after
- end
- tags = tags.compact.join('_')
+ args.push("--format", "documentation") unless args.flatten.include?("documentation")
+ { "QA::Support::JsonFormatter" => "json", "RspecJunitFormatter" => "xml" }.each do |formatter, extension|
+ next if args.flatten.include?(formatter)
- filename.concat("_#{tags}") unless tags.empty?
+ args.push("--format", formatter, "--out", "#{filename}.#{extension}")
+ end
+ end
- filename.concat('.txt')
+ def custom_test_paths?
+ Runtime::Env.knapsack? || options.any? { |opt| opt.include?('features') }
+ end
- FileUtils.mkdir_p('no_of_examples')
- File.join('no_of_examples', filename)
+ def metadata_run?
+ Runtime::Scenario.attributes[:count_examples_only] || Runtime::Scenario.attributes[:test_metadata_only]
end
end
end
diff --git a/qa/qa/specs/spec_helper.rb b/qa/qa/specs/spec_helper.rb
index 915d7acff19950189ebbdd259e278d38ace4f30e..1eb3eaaa48d1196c7489a4b85fa5ddd3ec9348ae 100644
--- a/qa/qa/specs/spec_helper.rb
+++ b/qa/qa/specs/spec_helper.rb
@@ -42,7 +42,12 @@
config.example_status_persistence_file_path = ENV.fetch('RSPEC_LAST_RUN_RESULTS_FILE', 'tmp/examples.txt')
config.prepend_before do |example|
- QA::Runtime::Logger.info("Starting test: #{Rainbow(example.full_description).bright}")
+ if QA::Runtime::Env.parallel_run?
+ QA::Runtime::Logger.info("Starting test - PID #{Process.pid}: #{Rainbow(example.full_description).bright}")
+ else
+ QA::Runtime::Logger.info("Starting test: #{Rainbow(example.full_description).bright}")
+ end
+
QA::Runtime::Example.current = example
visit(QA::Runtime::Scenario.gitlab_address) if QA::Runtime::Env.mobile_layout?
@@ -53,8 +58,8 @@
end
config.prepend_before(:suite) do
- # Perform before hooks at the very start of the test run
- QA::Runtime::Release.perform_before_hooks unless QA::Runtime::Env.dry_run
+ # Perform before hooks at the very start of the test run, perform once for parallel runs
+ QA::Runtime::Release.perform_before_hooks unless QA::Runtime::Env.dry_run || QA::Runtime::Env.parallel_run?
end
config.before(:suite) do
diff --git a/qa/qa/support/formatters/test_metrics_formatter.rb b/qa/qa/support/formatters/test_metrics_formatter.rb
index 4bb0a4f4343c5a199d27a11925847aeb1b0e0c55..5418d48a0a54252f90e13e670eb8d38444d2dcc4 100644
--- a/qa/qa/support/formatters/test_metrics_formatter.rb
+++ b/qa/qa/support/formatters/test_metrics_formatter.rb
@@ -97,9 +97,12 @@ def push_test_metrics_to_influxdb
def push_test_metrics_to_gcs
init_gcs_client! # init client and exit early if mandatory configuration is missing
retry_on_exception(sleep_interval: 30, message: 'Failed to push test metrics to GCS') do
- gcs_client.put_object(gcs_bucket, metrics_file_name(prefix: 'test',
- postfix: "-#{env('CI_PIPELINE_ID') || 'local'}"), execution_data.to_json,
- force: true, content_type: 'application/json')
+ gcs_client.put_object(
+ gcs_bucket,
+ metrics_file_name(prefix: 'test', postfix: metrics_filename_postfix),
+ execution_data.to_json,
+ force: true, content_type: 'application/json'
+ )
log(:info, "Pushed #{execution_data.length} test execution entries to GCS")
end
@@ -140,10 +143,12 @@ def push_code_runtime_metrics
def push_fabrication_metrics_gcs(data)
init_gcs_client! # init client and exit early if mandatory configuration is missing
retry_on_exception(sleep_interval: 30, message: 'Failed to push resource fabrication metrics to GCS') do
- gcs_client.put_object(gcs_bucket,
- metrics_file_name(prefix: 'fabrication',
- postfix: "-#{env('CI_PIPELINE_ID') || 'local'}"),
- data.to_json, force: true, content_type: 'application/json')
+ gcs_client.put_object(
+ gcs_bucket,
+ metrics_file_name(prefix: 'fabrication', postfix: metrics_filename_postfix),
+ data.to_json, force: true,
+ content_type: 'application/json'
+ )
log(:info, "Pushed #{data.length} resource fabrication entries to GCS")
end
@@ -186,6 +191,17 @@ def metrics_file_name(prefix:, postfix: '')
"#{retry_failed_specs? ? "-retry-#{rspec_retried?}" : ''}#{postfix}.json"
end
+ # Postfix for metrics filenames
+ #
+ # @return [String]
+ def metrics_filename_postfix
+ @metrics_filename_postfix ||= if QA::Runtime::Env.parallel_run?
+ "-#{Process.pid}-#{env('CI_PIPELINE_ID') || 'local'}"
+ else
+ "-#{env('CI_PIPELINE_ID') || 'local'}"
+ end
+ end
+
# Transform example to influxdb compatible metrics data
# https://github.com/influxdata/influxdb-client-ruby#data-format
#
diff --git a/qa/spec/specs/parallel_runner_spec.rb b/qa/spec/specs/parallel_runner_spec.rb
index d77b50fbe094a2ece1e41a2ff661df37ac3f4e6b..1622d0b210a1336ac34c0f4c1506c20165b1452c 100644
--- a/qa/spec/specs/parallel_runner_spec.rb
+++ b/qa/spec/specs/parallel_runner_spec.rb
@@ -1,58 +1,93 @@
# frozen_string_literal: true
+require 'etc'
+
RSpec.describe QA::Specs::ParallelRunner do
include QA::Support::Helpers::StubEnv
- before do
- allow(QA::Runtime::Scenario).to receive(:attributes).and_return(parallel: true)
- stub_env('GITLAB_QA_ACCESS_TOKEN', 'skip_token_creation')
- end
+ subject(:runner) { described_class }
- it 'passes args to parallel_tests' do
- expect_cli_arguments(['--tag', '~orchestrated', *QA::Specs::Runner::DEFAULT_TEST_PATH_ARGS])
+ let(:parallel_tests) { instance_double(ParallelTests::CLI, run: nil) }
- subject.run(['--tag', '~orchestrated', *QA::Specs::Runner::DEFAULT_TEST_PATH_ARGS])
- end
+ before do
+ allow(ParallelTests::CLI).to receive(:new).and_return(parallel_tests)
+ allow(Etc).to receive(:nprocessors).and_return(8)
+ allow(ENV).to receive(:store)
- it 'passes a given test path to parallel_tests and adds a separator' do
- expect_cli_arguments(%w[-- qa/specs/features/foo])
+ allow(QA::Runtime::Browser).to receive(:configure!)
+ allow(QA::Runtime::Release).to receive(:perform_before_hooks)
- subject.run(%w[qa/specs/features/foo])
+ stub_env("QA_GITLAB_URL", "http://127.0.0.1:3000")
+ stub_env("QA_PARALLEL_PROCESSES", "8")
end
- it 'passes tags and test paths to parallel_tests and adds a separator' do
- expect_cli_arguments(%w[--tag smoke -- qa/specs/features/foo qa/specs/features/bar])
+ it "runs cli without additional rspec args" do
+ runner.run([])
- subject.run(%w[--tag smoke qa/specs/features/foo qa/specs/features/bar])
+ expect(parallel_tests).to have_received(:run).with([
+ "--type", "rspec",
+ "-n", "8",
+ "--serialize-stdout",
+ "--first-is-1",
+ "--combine-stderr"
+ ])
end
- it 'passes tags and test paths with separators to parallel_tests' do
- expect_cli_arguments(%w[-- --tag smoke -- qa/specs/features/foo qa/specs/features/bar])
-
- subject.run(%w[-- --tag smoke -- qa/specs/features/foo qa/specs/features/bar])
+ it "runs cli with additional rspec args" do
+ runner.run(["--force-color", "qa/specs/features/api"])
+
+ expect(parallel_tests).to have_received(:run).with([
+ "--type", "rspec",
+ "-n", "8",
+ "--serialize-stdout",
+ "--first-is-1",
+ "--combine-stderr",
+ "--", "--force-color",
+ "--", "qa/specs/features/api"
+ ])
end
- it 'passes supported environment variables' do
- # Test only env vars starting with GITLAB because some of the others
- # affect how the runner behaves, and we're not concerned with those
- # behaviors in this test
- gitlab_env_vars = QA::Runtime::Env::ENV_VARIABLES.reject { |v| !v.start_with?('GITLAB') }
+ context "with QA_GITLAB_URL not set" do
+ before do
+ stub_env("QA_GITLAB_URL", nil)
- gitlab_env_vars.each do |k, v|
- stub_env(k, v)
+ QA::Support::GitlabAddress.instance_variable_set(:@initialized, nil)
end
- gitlab_env_vars['QA_RUNTIME_SCENARIO_ATTRIBUTES'] = '{"parallel":true}'
+ after do
+ QA::Support::GitlabAddress.instance_variable_set(:@initialized, nil)
+ end
- expect_cli_arguments([], gitlab_env_vars)
+ it "sets QA_GITLAB_URL variable for subprocess" do
+ runner.run([])
- subject.run([])
+ expect(ENV).to have_received(:store).with("QA_GITLAB_URL", "http://127.0.0.1:3000")
+ end
end
- def expect_cli_arguments(arguments, env = { 'QA_RUNTIME_SCENARIO_ATTRIBUTES' => '{"parallel":true}' })
- cmd = "bundle exec parallel_test -t rspec --combine-stderr --serialize-stdout -- #{arguments.join(' ')}"
- expect(Open3).to receive(:popen2e)
- .with(hash_including(env), cmd)
- .and_return(0)
+ context "with QA_PARALLEL_PROCESSES not set" do
+ before do
+ stub_env("QA_PARALLEL_PROCESSES", nil)
+ allow(Etc).to receive(:nprocessors).and_return(8)
+ end
+
+ it "sets number of processes to half of available processors" do
+ allow(QA::Runtime::Env).to receive(:parallel_processes).and_call_original
+
+ runner.run([])
+
+ expect(QA::Runtime::Env).to have_received(:parallel_processes)
+ actual_processes = QA::Runtime::Env.parallel_processes
+
+ expect(parallel_tests).to have_received(:run) do |args|
+ expect(args).to eq([
+ "--type", "rspec",
+ "-n", actual_processes.to_s,
+ "--serialize-stdout",
+ "--first-is-1",
+ "--combine-stderr"
+ ])
+ end
+ end
end
end
diff --git a/qa/spec/specs/runner_spec.rb b/qa/spec/specs/runner_spec.rb
index c5a6f455f2dca201d3fc9b434fba0dd49a4e97f8..51cb80d728aa9b7ece3f6480dfbf6581e86752e2 100644
--- a/qa/spec/specs/runner_spec.rb
+++ b/qa/spec/specs/runner_spec.rb
@@ -3,7 +3,10 @@
RSpec.describe QA::Specs::Runner do
shared_examples 'excludes default skipped, and geo' do
it 'excludes the default skipped and geo tags, and includes default args' do
- expect_rspec_runner_arguments(DEFAULT_SKIPPED_TAGS + ['--tag', '~geo', *described_class::DEFAULT_TEST_PATH_ARGS])
+ expect_rspec_runner_arguments(DEFAULT_SKIPPED_TAGS + ['--tag', '~geo',
+ '--format', 'documentation', '--format', 'QA::Support::JsonFormatter',
+ '--out', "tmp/rspec-#{ENV['CI_JOB_ID'] || 'local'}-retried-false.json",
+ '--format', 'RspecJunitFormatter', '--out', "tmp/rspec-#{ENV['CI_JOB_ID'] || 'local'}-retried-false.xml", *described_class::DEFAULT_TEST_PATH_ARGS])
subject.perform
end
@@ -28,7 +31,10 @@
it 'sets the `--tty` flag' do
expect_rspec_runner_arguments(
- ['--tty'] + DEFAULT_SKIPPED_TAGS + ['--tag', '~geo', *described_class::DEFAULT_TEST_PATH_ARGS]
+ ['--tty'] + DEFAULT_SKIPPED_TAGS + ['--tag', '~geo',
+ '--format', 'documentation', '--format', 'QA::Support::JsonFormatter',
+ '--out', "tmp/rspec-#{ENV['CI_JOB_ID'] || 'local'}-retried-false.json",
+ '--format', 'RspecJunitFormatter', '--out', "tmp/rspec-#{ENV['CI_JOB_ID'] || 'local'}-retried-false.xml", *described_class::DEFAULT_TEST_PATH_ARGS]
)
subject.perform
@@ -37,63 +43,37 @@
context 'when count_examples_only is set as an option' do
let(:out) { StringIO.new }
+ let(:err) { StringIO.new }
before do
QA::Runtime::Scenario.define(:count_examples_only, true)
- out.string = '22 examples,'
- allow(StringIO).to receive(:new).and_return(out)
+ allow(StringIO).to receive(:new).and_return(out, err)
+ allow(RSpec::Core::Runner).to receive(:run).and_return(0)
end
after do
QA::Runtime::Scenario.attributes.delete(:count_examples_only)
end
- it 'sets the `--dry-run` flag' do
+ it 'sets the `--dry-run` flag and minimal arguments' do
expect_rspec_runner_arguments(
- ['--dry-run'] + DEFAULT_SKIPPED_TAGS + ['--tag', '~geo', *described_class::DEFAULT_TEST_PATH_ARGS],
- [$stderr, anything]
+ ['--dry-run', '--tag', '~orchestrated', '--tag', '~transient', '--tag', '~geo', *described_class::DEFAULT_TEST_PATH_ARGS],
+ [err, out]
)
- subject.perform
- end
-
- it 'writes to file when examples are more than zero' do
- allow(RSpec::Core::Runner).to receive(:run).and_return(0)
-
- expect(File).to receive(:open).with('no_of_examples/test_instance_all.txt', 'w') { '22' }
-
- subject.perform
- end
-
- it 'does not write to file when zero examples' do
- out.string = '0 examples,'
- allow(RSpec::Core::Runner).to receive(:run).and_return(0)
-
- expect(File).not_to receive(:open)
-
- subject.perform
- end
-
- it 'raises error when Rspec output does not match regex' do
- out.string = '0'
- allow(RSpec::Core::Runner).to receive(:run).and_return(0)
-
- expect { subject.perform }
- .to raise_error(QA::Specs::Runner::RegexMismatchError, 'Rspec output did not match regex')
+ expect { subject.perform }.to raise_error(JSON::ParserError, /Failed to detect example count/)
end
context 'when --tag is specified as an option' do
subject { described_class.new.tap { |runner| runner.options = %w[--tag actioncable] } }
- it 'includes the option value in the file name' do
+ it 'includes the tag in the RSpec arguments' do
expect_rspec_runner_arguments(
['--dry-run', '--tag', '~geo', '--tag', 'actioncable', *described_class::DEFAULT_TEST_PATH_ARGS],
- [$stderr, anything]
+ [err, out]
)
- expect(File).to receive(:open).with('no_of_examples/test_instance_all_actioncable.txt', 'w') { '22' }
-
- subject.perform
+ expect { subject.perform }.to raise_error(JSON::ParserError, /Failed to detect example count/)
end
end
end
@@ -116,24 +96,11 @@
it 'sets the `--dry-run` flag' do
expect_rspec_runner_arguments(
['--dry-run', *described_class::DEFAULT_TEST_PATH_ARGS],
- [$stderr, anything]
+ [$stderr, $stdout]
)
subject.perform
end
-
- it 'configures json formatted output to file' do
- allow(QA::Runtime::Path).to receive(:qa_root).and_return('/root')
-
- expect(rspec_config).to receive(:add_formatter)
- .with(QA::Support::JsonFormatter, output_file)
- expect(rspec_config).to receive(:fail_if_no_examples=)
- .with(true)
-
- allow(RSpec::Core::Runner).to receive(:run).and_return(0)
-
- subject.perform
- end
end
context 'when tags are set' do
@@ -141,7 +108,10 @@
it 'focuses on the given tags' do
expect_rspec_runner_arguments(
- ['--tag', 'orchestrated', '--tag', 'github', '--tag', '~geo', *described_class::DEFAULT_TEST_PATH_ARGS]
+ ['--tag', 'orchestrated', '--tag', 'github', '--tag', '~geo',
+ '--format', 'documentation', '--format', 'QA::Support::JsonFormatter',
+ '--out', "tmp/rspec-#{ENV['CI_JOB_ID'] || 'local'}-retried-false.json",
+ '--format', 'RspecJunitFormatter', '--out', "tmp/rspec-#{ENV['CI_JOB_ID'] || 'local'}-retried-false.xml", *described_class::DEFAULT_TEST_PATH_ARGS]
)
subject.perform
@@ -152,7 +122,10 @@
subject { described_class.new.tap { |runner| runner.options = %w[--tag smoke] } }
it 'focuses on the given tag without excluded tags' do
- expect_rspec_runner_arguments(['--tag', '~geo', '--tag', 'smoke', *described_class::DEFAULT_TEST_PATH_ARGS])
+ expect_rspec_runner_arguments(['--tag', '~geo', '--tag', 'smoke',
+ '--format', 'documentation', '--format', 'QA::Support::JsonFormatter',
+ '--out', "tmp/rspec-#{ENV['CI_JOB_ID'] || 'local'}-retried-false.json",
+ '--format', 'RspecJunitFormatter', '--out', "tmp/rspec-#{ENV['CI_JOB_ID'] || 'local'}-retried-false.xml", *described_class::DEFAULT_TEST_PATH_ARGS])
subject.perform
end
@@ -162,7 +135,13 @@
subject { described_class.new.tap { |runner| runner.options = %w[qa/specs/features/foo] } }
it 'passes the given tests path and excludes the default skipped, and geo tags' do
- expect_rspec_runner_arguments(DEFAULT_SKIPPED_TAGS + ['--tag', '~geo', 'qa/specs/features/foo'])
+ expect_rspec_runner_arguments(
+ ['--tag', '~orchestrated', '--tag', '~transient', '--tag', '~geo',
+ 'qa/specs/features/foo',
+ '--format', 'documentation', '--format', 'QA::Support::JsonFormatter',
+ '--out', "tmp/rspec-#{ENV['CI_JOB_ID'] || 'local'}-retried-false.json",
+ '--format', 'RspecJunitFormatter', '--out', "tmp/rspec-#{ENV['CI_JOB_ID'] || 'local'}-retried-false.xml"]
+ )
subject.perform
end
@@ -172,7 +151,13 @@
subject { described_class.new.tap { |runner| runner.options = %w[--tag smoke qa/specs/features/foo] } }
it 'focuses on the given tag and includes the path without excluding the orchestrated or transient tags' do
- expect_rspec_runner_arguments(['--tag', '~geo', '--tag', 'smoke', 'qa/specs/features/foo'])
+ expect_rspec_runner_arguments(
+ ['--tag', '~geo', '--tag', 'smoke',
+ 'qa/specs/features/foo',
+ '--format', 'documentation', '--format', 'QA::Support::JsonFormatter',
+ '--out', "tmp/rspec-#{ENV['CI_JOB_ID'] || 'local'}-retried-false.json",
+ '--format', 'RspecJunitFormatter', '--out', "tmp/rspec-#{ENV['CI_JOB_ID'] || 'local'}-retried-false.xml"]
+ )
subject.perform
end
@@ -184,7 +169,10 @@
end
it 'includes default args and excludes the skip_signup_disabled tag' do
- expect_rspec_runner_arguments(DEFAULT_SKIPPED_TAGS + ['--tag', '~geo', '--tag', '~skip_signup_disabled', *described_class::DEFAULT_TEST_PATH_ARGS])
+ expect_rspec_runner_arguments(DEFAULT_SKIPPED_TAGS + ['--tag', '~geo', '--tag', '~skip_signup_disabled',
+ '--format', 'documentation', '--format', 'QA::Support::JsonFormatter',
+ '--out', "tmp/rspec-#{ENV['CI_JOB_ID'] || 'local'}-retried-false.json",
+ '--format', 'RspecJunitFormatter', '--out', "tmp/rspec-#{ENV['CI_JOB_ID'] || 'local'}-retried-false.xml", *described_class::DEFAULT_TEST_PATH_ARGS])
subject.perform
end
@@ -196,7 +184,10 @@
end
it 'includes default args and excludes the skip_live_env tag' do
- expect_rspec_runner_arguments(DEFAULT_SKIPPED_TAGS + ['--tag', '~geo', '--tag', '~skip_live_env', *described_class::DEFAULT_TEST_PATH_ARGS])
+ expect_rspec_runner_arguments(DEFAULT_SKIPPED_TAGS + ['--tag', '~geo', '--tag', '~skip_live_env',
+ '--format', 'documentation', '--format', 'QA::Support::JsonFormatter',
+ '--out', "tmp/rspec-#{ENV['CI_JOB_ID'] || 'local'}-retried-false.json",
+ '--format', 'RspecJunitFormatter', '--out', "tmp/rspec-#{ENV['CI_JOB_ID'] || 'local'}-retried-false.xml", *described_class::DEFAULT_TEST_PATH_ARGS])
subject.perform
end
end
@@ -213,7 +204,10 @@
subject { described_class.new.tap { |runner| runner.tags = %i[geo] } }
it 'includes the geo tag' do
- expect_rspec_runner_arguments(['--tag', 'geo', *described_class::DEFAULT_TEST_PATH_ARGS])
+ expect_rspec_runner_arguments(['--tag', 'geo',
+ '--format', 'documentation', '--format', 'QA::Support::JsonFormatter',
+ '--out', "tmp/rspec-#{ENV['CI_JOB_ID'] || 'local'}-retried-false.json",
+ '--format', 'RspecJunitFormatter', '--out', "tmp/rspec-#{ENV['CI_JOB_ID'] || 'local'}-retried-false.xml", *described_class::DEFAULT_TEST_PATH_ARGS])
subject.perform
end
end
@@ -231,7 +225,9 @@
it 'includes default args and excludes all unsupported tags' do
expect_rspec_runner_arguments(
DEFAULT_SKIPPED_TAGS + ['--tag', '~geo', *excluded_feature_tags_except(feature),
- *described_class::DEFAULT_TEST_PATH_ARGS]
+ '--format', 'documentation', '--format', 'QA::Support::JsonFormatter',
+ '--out', "tmp/rspec-#{ENV['CI_JOB_ID'] || 'local'}-retried-false.json",
+ '--format', 'RspecJunitFormatter', '--out', "tmp/rspec-#{ENV['CI_JOB_ID'] || 'local'}-retried-false.xml", *described_class::DEFAULT_TEST_PATH_ARGS]
)
subject.perform