diff --git a/qa/qa/specs/features/sanity/feature_flags_spec.rb b/qa/qa/specs/features/sanity/feature_flags_spec.rb
index acb9528fe6a4c72f42b57007922471f5ea83fe14..36069558701a7806319d2b1f5900ec5773c1fe1e 100644
--- a/qa/qa/specs/features/sanity/feature_flags_spec.rb
+++ b/qa/qa/specs/features/sanity/feature_flags_spec.rb
@@ -1,86 +1,88 @@
# frozen_string_literal: true
module QA
- RSpec.describe 'Feature flag handler sanity checks', :sanity_feature_flags do
- context 'with an existing feature flag definition file' do
- let(:definition) do
- path = Pathname.new('../config/feature_flags')
- .expand_path(Runtime::Path.qa_root)
- .glob('**/*.yml')
- .first
- YAML.safe_load(File.read(path))
- end
+ RSpec.describe 'Framework sanity', :sanity_feature_flags do
+ describe 'Feature flag handler checks' do
+ context 'with an existing feature flag definition file' do
+ let(:definition) do
+ path = Pathname.new('../config/feature_flags')
+ .expand_path(Runtime::Path.qa_root)
+ .glob('**/*.yml')
+ .first
+ YAML.safe_load(File.read(path))
+ end
- it 'reads the correct default enabled state' do
- # This test will fail if we ever remove all the feature flags, but that's very unlikely given how many there
- # are and how much we rely on them.
- expect(QA::Runtime::Feature.enabled?(definition['name'])).to be definition['default_enabled']
+ it 'reads the correct default enabled state' do
+ # This test will fail if we ever remove all the feature flags, but that's very unlikely given how many there
+ # are and how much we rely on them.
+ expect(QA::Runtime::Feature.enabled?(definition['name'])).to be definition['default_enabled']
+ end
end
- end
- describe 'feature flag definition files' do
- let(:file) do
- path = Pathname.new("#{root}/config/feature_flags/development").expand_path(Runtime::Path.qa_root)
- path.mkpath
- Tempfile.new(%w[ff-test .yml], path)
- end
+ describe 'feature flag definition files' do
+ let(:file) do
+ path = Pathname.new("#{root}/config/feature_flags/development").expand_path(Runtime::Path.qa_root)
+ path.mkpath
+ Tempfile.new(%w[ff-test .yml], path)
+ end
- let(:flag) { Pathname.new(file.path).basename('.yml').to_s }
- let(:root) { '..' }
+ let(:flag) { Pathname.new(file.path).basename('.yml').to_s }
+ let(:root) { '..' }
- before do
- definition = <<~YAML
+ before do
+ definition = <<~YAML
name: #{flag}
type: development
default_enabled: #{flag_enabled}
- YAML
- File.write(file, definition)
- end
+ YAML
+ File.write(file, definition)
+ end
- after do
- file.close!
- end
+ after do
+ file.close!
+ end
- shared_examples 'gets flag value' do
- context 'with a default disabled feature flag' do
- let(:flag_enabled) { 'false' }
+ shared_examples 'gets flag value' do
+ context 'with a default disabled feature flag' do
+ let(:flag_enabled) { 'false' }
- it 'reads the flag as disabled' do
- expect(QA::Runtime::Feature.enabled?(flag)).to be false
- end
+ it 'reads the flag as disabled' do
+ expect(QA::Runtime::Feature.enabled?(flag)).to be false
+ end
- it 'reads as enabled after the flag is enabled' do
- QA::Runtime::Feature.enable(flag)
+ it 'reads as enabled after the flag is enabled' do
+ QA::Runtime::Feature.enable(flag)
- expect { QA::Runtime::Feature.enabled?(flag) }.to eventually_be_truthy
- .within(max_duration: 60, sleep_interval: 5)
+ expect { QA::Runtime::Feature.enabled?(flag) }.to eventually_be_truthy
+ .within(max_duration: 60, sleep_interval: 5)
+ end
end
- end
- context 'with a default enabled feature flag' do
- let(:flag_enabled) { 'true' }
+ context 'with a default enabled feature flag' do
+ let(:flag_enabled) { 'true' }
- it 'reads the flag as enabled' do
- expect(QA::Runtime::Feature.enabled?(flag)).to be true
- end
+ it 'reads the flag as enabled' do
+ expect(QA::Runtime::Feature.enabled?(flag)).to be true
+ end
- it 'reads as disabled after the flag is disabled' do
- QA::Runtime::Feature.disable(flag)
+ it 'reads as disabled after the flag is disabled' do
+ QA::Runtime::Feature.disable(flag)
- expect { QA::Runtime::Feature.enabled?(flag) }.to eventually_be_falsey
- .within(max_duration: 60, sleep_interval: 5)
+ expect { QA::Runtime::Feature.enabled?(flag) }.to eventually_be_falsey
+ .within(max_duration: 60, sleep_interval: 5)
+ end
end
end
- end
- context 'with a CE feature flag' do
- include_examples 'gets flag value'
- end
+ context 'with a CE feature flag' do
+ include_examples 'gets flag value'
+ end
- context 'with an EE feature flag' do
- let(:root) { '../ee' }
+ context 'with an EE feature flag' do
+ let(:root) { '../ee' }
- include_examples 'gets flag value'
+ include_examples 'gets flag value'
+ end
end
end
end
diff --git a/qa/qa/specs/features/sanity/framework_spec.rb b/qa/qa/specs/features/sanity/framework_spec.rb
index fa34f525a857b038081064055c11f9cd3b2d292f..5c80afe338e67f3ae5dc73d7f289769f3ca1a2fa 100644
--- a/qa/qa/specs/features/sanity/framework_spec.rb
+++ b/qa/qa/specs/features/sanity/framework_spec.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module QA
- RSpec.describe 'Framework sanity checks', :orchestrated, :framework do
+ RSpec.describe 'Framework sanity', :orchestrated, :framework do
describe 'Passing orchestrated example' do
it 'succeeds' do
Runtime::Browser.visit(:gitlab, Page::Main::Login)
diff --git a/qa/qa/specs/features/sanity/interception_spec.rb b/qa/qa/specs/features/sanity/interception_spec.rb
index f8930db3aa55cbf6dd2b9d82bfc2e94282ebea38..67be832055df03202468ff928efbb23aede8e0a2 100644
--- a/qa/qa/specs/features/sanity/interception_spec.rb
+++ b/qa/qa/specs/features/sanity/interception_spec.rb
@@ -1,39 +1,41 @@
# frozen_string_literal: true
module QA
- RSpec.describe 'Browser request interception', :orchestrated, :framework do
- before(:context) do
- skip 'Only can test for chrome' unless QA::Runtime::Env.can_intercept?
- end
+ RSpec.describe 'Framework sanity', :orchestrated, :framework do
+ describe 'Browser request interception' do
+ before(:context) do
+ skip 'Only can test for chrome' unless QA::Runtime::Env.can_intercept?
+ end
- before do
- Runtime::Browser.visit(:gitlab, Page::Main::Login)
- end
+ before do
+ Runtime::Browser.visit(:gitlab, Page::Main::Login)
+ end
- let(:page) { Capybara.current_session }
- let(:logger) { class_double('QA::Runtime::Logger') }
+ let(:page) { Capybara.current_session }
+ let(:logger) { class_double('QA::Runtime::Logger') }
- it 'intercepts failed graphql calls' do
- page.execute_script <<~JS
+ it 'intercepts failed graphql calls' do
+ page.execute_script <<~JS
fetch('/api/graphql', {
method: 'POST',
body: JSON.stringify({ query: 'query {}'}),
headers: { 'Content-Type': 'application/json' }
})
- JS
+ JS
- Support::Waiter.wait_until do
- !get_cached_error.nil?
+ Support::Waiter.wait_until do
+ !get_cached_error.nil?
+ end
+ expect(**get_cached_error).to include({ 'method' => 'POST', 'status' => 200, 'url' => '/api/graphql' })
end
- expect(**get_cached_error).to include({ 'method' => 'POST', 'status' => 200, 'url' => '/api/graphql' })
- end
- def get_cached_error
- cache = page.execute_script <<~JS
+ def get_cached_error
+ cache = page.execute_script <<~JS
return Interceptor.getCache()
- JS
+ JS
- cache['errors']&.first
+ cache['errors']&.first
+ end
end
end
end
diff --git a/qa/qa/specs/features/sanity/version_spec.rb b/qa/qa/specs/features/sanity/version_spec.rb
index e93a8a6fea1a6777ddac87c50a273ac8adb3c271..deefe830c3605772cb6896716f02c7f23fb970e0 100644
--- a/qa/qa/specs/features/sanity/version_spec.rb
+++ b/qa/qa/specs/features/sanity/version_spec.rb
@@ -7,31 +7,33 @@ module QA
# environment variable is the version actually running.
#
# See https://gitlab.com/gitlab-com/gl-infra/delivery/-/issues/1179
- RSpec.describe 'Version sanity check', :smoke, only: { pipeline: [:pre, :release] } do
- let(:api_client) { Runtime::API::Client.new(:gitlab) }
- let(:request) { Runtime::API::Request.new(api_client, '/version') }
+ RSpec.describe 'Framework sanity', :smoke, only: { pipeline: [:pre, :release] } do
+ describe 'Version check' do
+ let(:api_client) { Runtime::API::Client.new(:gitlab) }
+ let(:request) { Runtime::API::Request.new(api_client, '/version') }
- it 'is the specified version' do
- # The `DEPLOY_VERSION` variable will only be provided for deploys to the
- # `pre` and `release` environments, which only receive packaged releases.
- #
- # For these releases, `deploy_version` will be a package string (e.g.,
- # `13.1.3-ee.0`), and the reported version will be something like
- # `13.1.3-ee`, so we only compare the leading SemVer string.
- #
- # | Package | Version |
- # | ---------------- | -------------- |
- # | 13.3.5-ee.0 | 13.3.5-ee |
- # | 13.3.0-rc42.ee.0 | 13.3.0-rc42-ee |
- deploy = Runtime::Env.deploy_version&.gsub(/\A(\d+\.\d+\.\d+).*\z/, '\1')
+ it 'is the specified version' do
+ # The `DEPLOY_VERSION` variable will only be provided for deploys to the
+ # `pre` and `release` environments, which only receive packaged releases.
+ #
+ # For these releases, `deploy_version` will be a package string (e.g.,
+ # `13.1.3-ee.0`), and the reported version will be something like
+ # `13.1.3-ee`, so we only compare the leading SemVer string.
+ #
+ # | Package | Version |
+ # | ---------------- | -------------- |
+ # | 13.3.5-ee.0 | 13.3.5-ee |
+ # | 13.3.0-rc42.ee.0 | 13.3.0-rc42-ee |
+ deploy = Runtime::Env.deploy_version&.gsub(/\A(\d+\.\d+\.\d+).*\z/, '\1')
- skip('No deploy version provided') if deploy.nil? || deploy.empty?
+ skip('No deploy version provided') if deploy.nil? || deploy.empty?
- get request.url
+ get request.url
- expect_status(200)
- expect(json_body).to have_key(:version)
- expect(json_body[:version]).to start_with(deploy)
+ expect_status(200)
+ expect(json_body).to have_key(:version)
+ expect(json_body[:version]).to start_with(deploy)
+ end
end
end
end
diff --git a/qa/qa/support/formatters/allure_metadata_formatter.rb b/qa/qa/support/formatters/allure_metadata_formatter.rb
index c8ddbeb45368c3143751fc8fb36423b2606472ec..eac74a3b961947c589bf6506d757e13cf4fe8a33 100644
--- a/qa/qa/support/formatters/allure_metadata_formatter.rb
+++ b/qa/qa/support/formatters/allure_metadata_formatter.rb
@@ -3,6 +3,14 @@
module QA
module Support
module Formatters
+ # RSpec formatter to enhance metadata present in allure report
+ # Following additional data is added:
+ # * quarantine issue links
+ # * failure issues search link
+ # * ci job link
+ # * flaky status and test pass rate
+ # * devops stage and group as epic and feature behaviour tags
+ #
class AllureMetadataFormatter < ::RSpec::Core::Formatters::BaseFormatter
include Support::InfluxdbTools
@@ -18,8 +26,6 @@ class AllureMetadataFormatter < ::RSpec::Core::Formatters::BaseFormatter
# @param [RSpec::Core::Notifications::StartNotification] _start_notification
# @return [void]
def start(_start_notification)
- return unless merge_request_iid # on main runs allure native history has pass rate already
-
save_flaky_specs
log(:debug, "Fetched #{flaky_specs.length} flaky testcases!")
rescue StandardError => e
@@ -63,11 +69,11 @@ def add_quarantine_issue_link(example)
# @param [RSpec::Core::Example] example
# @return [void]
def add_failure_issues_link(example)
- spec_file = example.file_path.split('/').last
- example.issue(
- 'Failure issues',
- "https://gitlab.com/gitlab-org/gitlab/-/issues?scope=all&state=opened&search=#{spec_file}"
- )
+ return unless example.execution_result.status == :failed
+
+ search_query = ERB::Util.url_encode("Failure in #{example.file_path.gsub('./qa/specs/features/', '')}")
+ search_url = "https://gitlab.com/gitlab-org/gitlab/-/issues?scope=all&state=opened&search=#{search_query}"
+ example.issue('Failure issues', search_url)
rescue StandardError => e
log(:error, "Failed to add failure issue link for example '#{example.description}', error: #{e}")
end
@@ -89,10 +95,10 @@ def add_ci_job_link(example)
# @param [RSpec::Core::Example] example
# @return [void]
def set_flaky_status(example)
- return unless merge_request_iid && flaky_specs.key?(example.metadata[:testcase])
+ return unless flaky_specs.key?(example.metadata[:testcase]) && example.execution_result.status != :pending
example.set_flaky
- example.parameter("pass_rate", "#{flaky_specs[example.metadata[:testcase]].round(1)}%")
+ example.parameter("pass_rate", "#{flaky_specs[example.metadata[:testcase]].round(0)}%")
log(:debug, "Setting spec as flaky because it's pass rate is below 98%")
rescue StandardError => e
log(:error, "Failed to add spec pass rate data for example '#{example.description}', error: #{e}")
diff --git a/qa/spec/support/formatters/allure_metadata_formatter_spec.rb b/qa/spec/support/formatters/allure_metadata_formatter_spec.rb
index d84e190fd56b9c00e26ab36d1f562c80670639bd..ab3b753c3b0e55f44e25406bc2aadde62349a2a1 100644
--- a/qa/spec/support/formatters/allure_metadata_formatter_spec.rb
+++ b/qa/spec/support/formatters/allure_metadata_formatter_spec.rb
@@ -5,25 +5,33 @@
let(:formatter) { described_class.new(StringIO.new) }
- let(:rspec_example_notification) { double('RSpec::Core::Notifications::ExampleNotification', example: rspec_example) }
+ let(:rspec_example_notification) do
+ instance_double(RSpec::Core::Notifications::ExampleNotification, example: rspec_example)
+ end
+
+ # rubocop:disable RSpec/VerifiedDoubles
let(:rspec_example) do
double(
- 'RSpec::Core::Example',
+ RSpec::Core::Example,
tms: nil,
issue: nil,
add_link: nil,
+ set_flaky: nil,
+ parameter: nil,
attempts: 0,
- file_path: 'file/path/spec.rb',
- execution_result: instance_double("RSpec::Core::Example::ExecutionResult", status: :passed),
+ file_path: 'spec.rb',
+ execution_result: instance_double(RSpec::Core::Example::ExecutionResult, status: status),
metadata: {
testcase: 'testcase',
quarantine: { issue: 'issue' }
}
)
end
+ # rubocop:enable RSpec/VerifiedDoubles
let(:ci_job) { 'ee:relative 5' }
let(:ci_job_url) { 'url' }
+ let(:status) { :failed }
before do
stub_env('CI', 'true')
@@ -31,16 +39,62 @@
stub_env('CI_JOB_URL', ci_job_url)
end
- it "adds additional data to report" do
- formatter.example_finished(rspec_example_notification)
+ context 'with links' do
+ it 'adds quarantine, failure issue and ci job links', :aggregate_failures do
+ formatter.example_finished(rspec_example_notification)
- aggregate_failures do
expect(rspec_example).to have_received(:issue).with('Quarantine issue', 'issue')
expect(rspec_example).to have_received(:add_link).with(name: "Job(#{ci_job})", url: ci_job_url)
expect(rspec_example).to have_received(:issue).with(
'Failure issues',
- 'https://gitlab.com/gitlab-org/gitlab/-/issues?scope=all&state=opened&search=spec.rb'
+ 'https://gitlab.com/gitlab-org/gitlab/-/issues?scope=all&state=opened&search=Failure%20in%20spec.rb'
)
end
end
+
+ context 'with flaky test data', :aggregate_failures do
+ let(:influx_client) { instance_double(InfluxDB2::Client, create_query_api: influx_query_api) }
+ let(:influx_query_api) { instance_double(InfluxDB2::QueryApi, query: data) }
+ let(:data) do
+ [
+ instance_double(
+ InfluxDB2::FluxTable,
+ records: [
+ instance_double(InfluxDB2::FluxRecord, values: { 'status' => 'failed', 'testcase' => 'testcase' }),
+ instance_double(InfluxDB2::FluxRecord, values: { 'status' => 'passed', 'testcase' => 'testcase' })
+ ]
+ )
+ ]
+ end
+
+ before do
+ stub_env('QA_RUN_TYPE', 'package-and-test')
+ stub_env('QA_INFLUXDB_URL', 'url')
+ stub_env('QA_INFLUXDB_TOKEN', 'token')
+
+ allow(InfluxDB2::Client).to receive(:new) { influx_client }
+ end
+
+ context 'with non skipped spec' do
+ it 'adds flaky test data' do
+ formatter.start(nil)
+ formatter.example_finished(rspec_example_notification)
+
+ expect(rspec_example).to have_received(:set_flaky)
+ expect(rspec_example).to have_received(:parameter).with('pass_rate', '50%')
+ end
+ end
+
+ context 'with skipped spec' do
+ let(:status) { :pending }
+
+ it 'skips adding flaky test data' do
+ formatter.start(nil)
+ formatter.example_finished(rspec_example_notification)
+
+ expect(rspec_example).not_to have_received(:set_flaky)
+ expect(rspec_example).not_to have_received(:parameter)
+ end
+ end
+ end
end