diff --git a/qa/qa/support/formatters/test_metrics_formatter.rb b/qa/qa/support/formatters/test_metrics_formatter.rb
index 1e7d4f10f0d7dedae8d98fa95a7e6e860856d69f..9bc530fedf579070f494049625686221ddf07aa2 100644
--- a/qa/qa/support/formatters/test_metrics_formatter.rb
+++ b/qa/qa/support/formatters/test_metrics_formatter.rb
@@ -74,6 +74,7 @@ def push_test_metrics_to_influxdb
#
# @return [void]
def push_test_metrics_to_gcs
+ init_gcs_client! # init client and exit early if mandatory configuration is missing
retry_on_exception(sleep_interval: 30, message: 'Failed to push test metrics to GCS') do
gcs_client.put_object(gcs_bucket, metrics_file_name(prefix: 'test',
postfix: "-#{env('CI_PIPELINE_ID') || 'local'}"), execution_data.to_json,
@@ -81,6 +82,8 @@ def push_test_metrics_to_gcs
log(:info, "Pushed #{execution_data.length} test execution entries to GCS")
end
+ rescue StandardError => e
+ log(:error, "Failed to push test execution metrics to gcs, error: #{e}")
end
# Push resource fabrication metrics to influxdb
@@ -104,6 +107,7 @@ def push_fabrication_metrics
# @param [Hash] data fabrication data hash
# @return [void]
def push_fabrication_metrics_gcs(data)
+ init_gcs_client! # init client and exit early if mandatory configuration is missing
retry_on_exception(sleep_interval: 30, message: 'Failed to push resource fabrication metrics to GCS') do
gcs_client.put_object(gcs_bucket,
metrics_file_name(prefix: 'fabrication',
@@ -112,6 +116,8 @@ def push_fabrication_metrics_gcs(data)
log(:info, "Pushed #{data.length} resource fabrication entries to GCS")
end
+ rescue StandardError => e
+ log(:error, "Failed to push test fabrication metrics to gcs, error: #{e}")
end
# Push resource fabrication metrics to InfluxDB
@@ -125,6 +131,13 @@ def push_fabrication_metrics_influxdb(data)
log(:error, "Failed to push fabrication metrics to influxdb, error: #{e}")
end
+ # Init client raising error if configuration variables are missing
+ #
+ # @return [void]
+ def init_gcs_client!
+ gcs_client || gcs_bucket
+ end
+
# Get GCS Bucket Name or raise error if missing
#
# @return [String]
@@ -220,8 +233,9 @@ def fields(example)
pipeline_id: env('CI_PIPELINE_ID'),
job_id: env('CI_JOB_ID'),
merge_request_iid: merge_request_iid,
- failure_issue: example.metadata[:quarantine] ? example.metadata[:quarantine][:issue] : nil,
failure_exception: example.execution_result.exception.to_s.delete("\n"),
+ location: example_location(example),
+ failure_issue: example.metadata.dig(:quarantine, :issue),
**custom_metrics_fields(example.metadata)
}.compact
end
@@ -351,6 +365,23 @@ def devops_stage(file_path)
def log(level, message)
QA::Runtime::Logger.public_send(level, "[influxdb exporter]: #{message}")
end
+
+ # Example location
+ #
+ # @param [RSpec::Core::Example] example
+ # @return [String]
+ def example_location(example)
+ # ensures that location will be correct even in case of shared examples
+ file = example
+ .metadata
+ .fetch(:shared_group_inclusion_backtrace)
+ .last
+ &.formatted_inclusion_location
+
+ return example.location unless file
+
+ file
+ end
end
end
end
diff --git a/qa/spec/support/formatters/test_metrics_formatter_spec.rb b/qa/spec/support/formatters/test_metrics_formatter_spec.rb
index ecba65a3711fe90f337af535405bc22a2d9f84b9..fee781b9a22740a8b9a97d08bff167eb31bbc732 100644
--- a/qa/spec/support/formatters/test_metrics_formatter_spec.rb
+++ b/qa/spec/support/formatters/test_metrics_formatter_spec.rb
@@ -2,6 +2,7 @@
require 'rspec/core/sandbox'
require 'active_support/testing/time_helpers'
+require 'pathname'
# rubocop:disable RSpec/MultipleMemoizedHelpers, Lint/EmptyBlock -- false positives for empty blocks and memoized helpers help with testing different data hash parameters
describe QA::Support::Formatters::TestMetricsFormatter do
@@ -75,7 +76,8 @@
pipeline_url: ci_pipeline_url,
pipeline_id: ci_pipeline_id,
job_id: ci_job_id,
- failure_exception: ''
+ failure_exception: '',
+ location: %r{./#{Pathname.new(__FILE__).relative_path_from(Dir.pwd)}:\d+}
}
}
end
@@ -111,6 +113,8 @@ def run_spec(passed: true, &spec)
.and_return(gcs_client)
allow(QA::Tools::TestResourceDataProcessor).to receive(:resources) { fabrication_resources }
allow_any_instance_of(RSpec::Core::Example::ExecutionResult).to receive(:run_time).and_return(0) # rubocop:disable RSpec/AnyInstanceOf -- simplifies mocking runtime
+
+ stub_env('QA_RUN_TYPE', run_type)
end
context 'without influxdb variables configured' do
@@ -159,16 +163,14 @@ def run_spec(passed: true, &spec)
end
end
- context 'with influxdb and GCS variables configured' do
+ context 'with influxdb variables configured' do
let(:spec_name) { 'exports data' }
let(:run_type) { ci_job_name.gsub(%r{ \d{1,2}/\d{1,2}}, '') }
before do
+ stub_env('QA_METRICS_GCS_CREDS', nil)
stub_env('QA_INFLUXDB_URL', url)
stub_env('QA_INFLUXDB_TOKEN', token)
- stub_env('QA_METRICS_GCS_PROJECT_ID', metrics_gcs_project_id)
- stub_env('QA_METRICS_GCS_CREDS', metrics_gcs_creds)
- stub_env('QA_METRICS_GCS_BUCKET_NAME', metrics_gcs_bucket_name)
stub_env('CI_PIPELINE_CREATED_AT', ci_timestamp)
stub_env('CI_JOB_URL', ci_job_url)
stub_env('CI_JOB_NAME', ci_job_name)
@@ -177,7 +179,6 @@ def run_spec(passed: true, &spec)
stub_env('CI_JOB_ID', ci_job_id)
stub_env('CI_MERGE_REQUEST_IID', nil)
stub_env('TOP_UPSTREAM_MERGE_REQUEST_IID', nil)
- stub_env('QA_RUN_TYPE', run_type)
stub_env('QA_EXPORT_TEST_METRICS', "true")
stub_env('QA_RSPEC_RETRIED', "false")
stub_env('QA_INFLUXDB_TIMEOUT', "10")
@@ -193,14 +194,11 @@ def run_spec(passed: true, &spec)
expect(influx_write_api).to have_received(:write).once
expect(influx_write_api).to have_received(:write).with(data: [data])
-
- expect(gcs_client).to have_received(:put_object).with(metrics_gcs_bucket_name,
- anything, [data].to_json, **gcs_client_options)
end
end
context 'with product group tag' do
- let(:expected_data) { [data.tap { |d| d[:tags][:product_group] = :import }] }
+ let(:expected_data) { data.tap { |d| d[:tags][:product_group] = :import } }
it 'exports data with correct product group tag' do
run_spec do
@@ -208,9 +206,7 @@ def run_spec(passed: true, &spec)
end
expect(influx_write_api).to have_received(:write).once
- expect(influx_write_api).to have_received(:write).with(
- data: expected_data
- )
+ expect(influx_write_api).to have_received(:write).with(data: [expected_data])
end
end
@@ -224,32 +220,13 @@ def run_spec(passed: true, &spec)
expect(influx_write_api).to have_received(:write).once
expect(influx_write_api).to have_received(:write).with(data: [data])
-
- expect(gcs_client).to have_received(:put_object).with(metrics_gcs_bucket_name,
- anything, [data].to_json, **gcs_client_options)
end
end
context 'with quarantined spec' do
let(:quarantined) { 'true' }
let(:status) { :pending }
- let(:expected_data) do
- data.tap do |d|
- d[:fields] = {
- id: './spec/support/formatters/test_metrics_formatter_spec.rb[1:1]',
- run_time: 0,
- api_fabrication: api_fabrication * 1000,
- ui_fabrication: ui_fabrication * 1000,
- total_fabrication: (api_fabrication + ui_fabrication) * 1000,
- job_url: ci_job_url,
- pipeline_url: ci_pipeline_url,
- pipeline_id: ci_pipeline_id,
- job_id: ci_job_id,
- failure_issue: 'https://example.com/issue/1234',
- failure_exception: ''
- }
- end
- end
+ let(:expected_data) { data.tap { |d| d[:fields][:failure_issue] = 'https://example.com/issue/1234' } }
it 'exports data with correct quarantine tag', :aggregate_failures do
run_spec do
@@ -266,31 +243,12 @@ def run_spec(passed: true, &spec)
expect(influx_write_api).to have_received(:write).once
expect(influx_write_api).to have_received(:write).with(data: [expected_data])
-
- expect(gcs_client).to have_received(:put_object).with(metrics_gcs_bucket_name,
- anything, [expected_data].to_json, **gcs_client_options)
end
end
context 'with context quarantined spec' do
let(:quarantined) { 'false' }
- let(:expected_data) do
- data.tap do |d|
- d[:fields] = {
- id: './spec/support/formatters/test_metrics_formatter_spec.rb[1:1]',
- run_time: 0,
- api_fabrication: api_fabrication * 1000,
- ui_fabrication: ui_fabrication * 1000,
- total_fabrication: (api_fabrication + ui_fabrication) * 1000,
- job_url: ci_job_url,
- pipeline_url: ci_pipeline_url,
- pipeline_id: ci_pipeline_id,
- job_id: ci_job_id,
- failure_issue: 'https://example.com/issue/1234',
- failure_exception: ''
- }
- end
- end
+ let(:expected_data) { data.tap { |d| d[:fields][:failure_issue] = 'https://example.com/issue/1234' } }
it 'exports data with correct quarantine tag', :aggregate_failures do
run_spec do
@@ -303,9 +261,6 @@ def run_spec(passed: true, &spec)
expect(influx_write_api).to have_received(:write).once
expect(influx_write_api).to have_received(:write).with(data: [expected_data])
-
- expect(gcs_client).to have_received(:put_object).with(metrics_gcs_bucket_name,
- anything, [expected_data].to_json, **gcs_client_options)
end
end
@@ -322,9 +277,6 @@ def run_spec(passed: true, &spec)
end
expect(influx_write_api).to have_received(:write).with(data: [data])
-
- expect(gcs_client).to have_received(:put_object).with(metrics_gcs_bucket_name,
- anything, [data].to_json, **gcs_client_options)
end
end
@@ -337,12 +289,7 @@ def run_spec(passed: true, &spec)
it('spec', testcase: 'https://gitlab.com/gitlab-org/gitlab/-/quality/test_cases/1234') { raise }
end
- expect(influx_write_api).to have_received(:write).with(
- data: [expected_data]
- )
-
- expect(gcs_client).to have_received(:put_object).with(metrics_gcs_bucket_name,
- anything, [expected_data].to_json, **gcs_client_options)
+ expect(influx_write_api).to have_received(:write).with(data: [expected_data])
end
end
@@ -356,8 +303,6 @@ def run_spec(passed: true, &spec)
end
expect(influx_write_api).to have_received(:write).with(data: [])
- expect(gcs_client).to have_received(:put_object).with(metrics_gcs_bucket_name,
- anything, [].to_json, **gcs_client_options)
end
end
@@ -374,8 +319,6 @@ def run_spec(passed: true, &spec)
end
expect(influx_write_api).to have_received(:write).with(data: [data])
- expect(gcs_client).to have_received(:put_object).with(metrics_gcs_bucket_name,
- anything, [data].to_json, **gcs_client_options)
end
end
end
@@ -393,9 +336,6 @@ def run_spec(passed: true, &spec)
expect(influx_write_api).to have_received(:write).once
expect(influx_write_api).to have_received(:write).with(data: [data])
-
- expect(gcs_client).to have_received(:put_object).with(metrics_gcs_bucket_name,
- anything, [data].to_json, **gcs_client_options)
end
end
@@ -410,16 +350,12 @@ def run_spec(passed: true, &spec)
end
custom_data = data.merge({
- **data,
tags: data[:tags].merge({ custom_tag: "tag" }),
fields: data[:fields].merge({ custom_field: 1 })
})
expect(influx_write_api).to have_received(:write).once
expect(influx_write_api).to have_received(:write).with(data: [custom_data])
-
- expect(gcs_client).to have_received(:put_object).with(metrics_gcs_bucket_name,
- anything, [custom_data].to_json, **gcs_client_options)
end
end
@@ -443,25 +379,23 @@ def run_spec(passed: true, &spec)
expect(influx_write_api).to have_received(:write).once
expect(influx_write_api).to have_received(:write).with(data: [data])
-
- expect(gcs_client).to have_received(:put_object).with(metrics_gcs_bucket_name,
- anything, [data].to_json, **gcs_client_options)
end
end
context 'with a shared example' do
let(:file_path) { './qa/specs/features/shared_examples/merge_with_code_owner_shared_examples.rb' }
let(:rerun_file_path) { './qa/specs/features/3_create/subfolder/another_spec.rb' }
+ let(:expected_data) do
+ data.tap do |d|
+ d[:tags][:file_path] = '/3_create/subfolder/another_spec.rb'
+ d[:tags][:stage] = 'create'
+ end
+ end
it 'exports data to influxdb with correct filename', :aggregate_failures do
run_spec
- data[:tags][:file_path] = '/3_create/subfolder/another_spec.rb'
- data[:tags][:stage] = 'create'
- expect(influx_write_api).to have_received(:write).with(data: [data])
-
- expect(gcs_client).to have_received(:put_object).with(metrics_gcs_bucket_name,
- anything, [data].to_json, **gcs_client_options)
+ expect(influx_write_api).to have_received(:write).with(data: [expected_data])
end
end
@@ -503,17 +437,18 @@ def run_spec(passed: true, &spec)
freeze_time { example.run }
end
- it 'exports fabrication stats data to influxdb and GCS', :aggregate_failures do
+ it 'exports fabrication data to influxdb and GCS', :aggregate_failures do
run_spec
expect(influx_write_api).to have_received(:write).with(data: [fabrication_data])
-
- expect(gcs_client).to have_received(:put_object).with(metrics_gcs_bucket_name,
- anything, [fabrication_data].to_json, **gcs_client_options)
end
end
context 'with persisting metrics' do
+ let(:expected_data) do
+ data.tap { |d| d[:fields][:location] = "./#{Pathname.new(__FILE__).relative_path_from(Dir.pwd)}:86" }
+ end
+
before do
stub_env('QA_EXPORT_TEST_METRICS', "false")
stub_env('QA_SAVE_TEST_METRICS', "true")
@@ -528,7 +463,7 @@ def run_spec(passed: true, &spec)
it 'saves test metrics as json files' do
run_spec
- expect(File).to have_received(:write).with(file, [data].to_json)
+ expect(File).to have_received(:write).with(file, [expected_data].to_json)
end
end
@@ -539,10 +474,76 @@ def run_spec(passed: true, &spec)
it 'saves test metrics as json files' do
run_spec
- expect(File).to have_received(:write).with(file, [data].to_json)
+ expect(File).to have_received(:write).with(file, [expected_data].to_json)
end
end
end
+
+ context "with metrics upload to gcs" do
+ let(:fabrication_resources) do
+ {
+ 'QA::Resource::Project' => [{
+ info: "with id '1'",
+ api_path: '/project',
+ fabrication_method: :api,
+ fabrication_time: 1,
+ http_method: :post,
+ timestamp: Time.now.to_s
+ }]
+ }
+ end
+
+ let(:test_data) do
+ data.tap { |d| d[:fields][:location] = "./#{Pathname.new(__FILE__).relative_path_from(Dir.pwd)}:86" }
+ end
+
+ let(:fabrication_data) do
+ {
+ name: 'fabrication-stats',
+ time: DateTime.strptime(ci_timestamp).to_time,
+ tags: {
+ resource: 'QA::Resource::Project',
+ fabrication_method: :api,
+ http_method: :post,
+ run_type: run_type,
+ merge_request: "false"
+ },
+ fields: {
+ fabrication_time: 1,
+ info: "with id '1'",
+ job_url: ci_job_url,
+ timestamp: Time.now.to_s
+ }
+ }
+ end
+
+ before do
+ stub_env('QA_METRICS_GCS_PROJECT_ID', metrics_gcs_project_id)
+ stub_env('QA_METRICS_GCS_CREDS', metrics_gcs_creds)
+ stub_env('QA_METRICS_GCS_BUCKET_NAME', metrics_gcs_bucket_name)
+ end
+
+ around do |example|
+ freeze_time { example.run }
+ end
+
+ it "creates correct json files and uploads metrics to gcs" do
+ run_spec
+
+ expect(gcs_client).to have_received(:put_object).with(
+ metrics_gcs_bucket_name,
+ /test-metrics-\S+\.json/,
+ [test_data].to_json,
+ **gcs_client_options
+ )
+ expect(gcs_client).to have_received(:put_object).with(
+ metrics_gcs_bucket_name,
+ /fabrication-metrics-\S+\.json/,
+ [fabrication_data].to_json,
+ **gcs_client_options
+ )
+ end
+ end
end
end
# rubocop:enable RSpec/MultipleMemoizedHelpers, Lint/EmptyBlock