From 01ed92c377ef8b2b117776decc97f307c9c3a160 Mon Sep 17 00:00:00 2001
From: Jay McCure <jmccure@gitlab.com>
Date: Tue, 14 Nov 2023 18:55:55 +0000
Subject: [PATCH] E2E test: group reliable report by product group
---
qa/qa/tools/reliable_report.rb | 160 +++++++++++++++-----------
qa/spec/tools/reliable_report_spec.rb | 94 +++++++++++----
2 files changed, 161 insertions(+), 93 deletions(-)
diff --git a/qa/qa/tools/reliable_report.rb b/qa/qa/tools/reliable_report.rb
index a0933d07166d..de8fbf7f044e 100644
--- a/qa/qa/tools/reliable_report.rb
+++ b/qa/qa/tools/reliable_report.rb
@@ -59,14 +59,14 @@ def self.run(range: 14, report_in_issue_and_slack: "false")
#
# @return [void]
def print_report
- puts "#{stable_summary_table}\n\n"
+ puts "#{summary_table(stable: true)}\n\n"
puts "Total amount: #{stable_test_runs.sum { |_k, v| v.count }}\n\n"
- stable_results_tables.each { |stage, table| puts "#{table}\n\n" }
+ print_results(stable_results_tables)
return puts("No unstable reliable tests present!".colorize(:yellow)) if unstable_reliable_test_runs.empty?
- puts "#{unstable_summary_table}\n\n"
+ puts "#{summary_table(stable: false)}\n\n"
puts "Total amount: #{unstable_reliable_test_runs.sum { |_k, v| v.count }}\n\n"
- unstable_reliable_results_tables.each { |stage, table| puts "#{table}\n\n" }
+ print_results(unstable_reliable_results_tables)
end
# Create report issue
@@ -89,8 +89,8 @@ def report_in_issue_and_slack
notifier.post(
icon_emoji: ":tanuki-protect:",
text: <<~TEXT
- ```#{stable_summary_table}```
- ```#{unstable_summary_table}```
+ ```#{summary_table(stable: true)}```
+ ```#{summary_table(stable: false)}```
#{web_url}
TEXT
@@ -173,39 +173,30 @@ def report_issue_body
issue = []
issue << "[[_TOC_]]"
issue << "# Candidates for promotion to reliable #{execution_interval}"
- issue << "Total amount: **#{stable_test_runs.sum { |_k, v| v.count }}**"
- issue << stable_summary_table(markdown: true).to_s
+ issue << "Total amount: **#{test_count(stable_test_runs)}**"
+ issue << summary_table(markdown: true, stable: true).to_s
issue << results_markdown(:stable)
return issue.join("\n\n") if unstable_reliable_test_runs.empty?
issue << "# Reliable specs with failures #{execution_interval}"
- issue << "Total amount: **#{unstable_reliable_test_runs.sum { |_k, v| v.count }}**"
- issue << unstable_summary_table(markdown: true).to_s
+ issue << "Total amount: **#{test_count(unstable_reliable_test_runs)}**"
+ issue << summary_table(markdown: true, stable: false).to_s
issue << results_markdown(:unstable)
issue.join("\n\n")
end
- # Stable spec summary table
+ # Spec summary table
#
# @param [Boolean] markdown
+ # @param [Boolean] stable
# @return [Terminal::Table]
- def stable_summary_table(markdown: false)
+ def summary_table(markdown: false, stable: true)
+ test_runs = stable ? stable_test_runs : unstable_reliable_test_runs
terminal_table(
- rows: stable_test_runs.map { |stage, specs| [stage, specs.length] },
- title: "Stable spec summary for past #{range} days".ljust(50),
- headings: %w[STAGE COUNT],
- markdown: markdown
- )
- end
-
- # Unstable reliable summary table
- #
- # @param [Boolean] markdown
- # @return [Terminal::Table]
- def unstable_summary_table(markdown: false)
- terminal_table(
- rows: unstable_reliable_test_runs.map { |stage, specs| [stage, specs.length] },
- title: "Unstable spec summary for past #{range} days".ljust(50),
+ rows: test_runs.map do |stage, stage_specs|
+ [stage, stage_specs.sum { |_k, group_specs| group_specs.length }]
+ end,
+ title: "#{stable ? 'Stable' : 'Unstable'} spec summary for past #{range} days".ljust(50),
headings: %w[STAGE COUNT],
markdown: markdown
)
@@ -233,41 +224,51 @@ def unstable_reliable_results_tables(markdown: false)
# @return [String]
def results_markdown(type)
runs = type == :stable ? stable_test_runs : unstable_reliable_test_runs
- results_tables(type, markdown: true).map do |stage, table|
- <<~STAGE.strip
- ## #{stage} (#{runs[stage].count})
+ results_tables(type, markdown: true).map do |stage, group_tables|
+ markdown = "## #{stage.capitalize} (#{runs[stage].sum { |_k, group_runs| group_runs.count }})\n\n"
- <details>
- <summary>Executions table</summary>
+ markdown << group_tables.map { |product_group, table| group_results_markdown(product_group, table) }.join
+ end.join("\n\n")
+ end
- #{table}
+ # Markdown formatted group results table
+ #
+ # @param [String] product_group
+ # @param [Terminal::Table] table
+ # @return [String]
+ def group_results_markdown(product_group, table)
+ <<~MARKDOWN.chomp
+ <details>
+ <summary>Executions table ~"group::#{product_group.tr('_', ' ')}" (#{table.rows.size})</summary>
- </details>
- STAGE
- end.join("\n\n")
+ #{table}
+
+ </details>
+ MARKDOWN
end
# Results table
#
# @param [Symbol] type result type - :stable, :unstable
# @param [Boolean] markdown
- # @return [Hash<Symbol, Terminal::Table>]
+ # @return [Hash<String, Hash<String, Terminal::Table>>] grouped by stage and product_group
def results_tables(type, markdown: false)
(type == :stable ? stable_test_runs : unstable_reliable_test_runs).to_h do |stage, specs|
- headings = ["name", "runs", "failures", "failure rate"]
-
- [stage, terminal_table(
- title: "Top #{type} specs in '#{stage}' stage for past #{range} days",
- headings: headings.map(&:upcase),
- markdown: markdown,
- rows: specs.map do |k, v|
- [
- name_column(name: k, file: v[:file], link: v[:link],
- exceptions_and_job_urls: v[:exceptions_and_job_urls], markdown: markdown),
- *table_params(v.values)
- ]
- end
- )]
+ headings = ['NAME', 'RUNS', 'FAILURES', 'FAILURE RATE'].freeze
+ [stage, specs.transform_values do |group_specs|
+ terminal_table(
+ title: "Top #{type} specs in '#{stage}::#{specs.key(group_specs)}' group for past #{range} days",
+ headings: headings,
+ markdown: markdown,
+ rows: group_specs.map do |name, result|
+ [
+ name_column(name: name, file: result[:file], link: result[:link],
+ exceptions_and_job_urls: result[:exceptions_and_job_urls], markdown: markdown),
+ *table_params(result.values)
+ ]
+ end
+ )
+ end]
end
end
@@ -276,14 +277,14 @@ def results_tables(type, markdown: false)
# @return [Hash]
def stable_test_runs
@top_stable ||= begin
- stable_specs = test_runs(reliable: false).transform_values do |specs|
- specs
- .reject { |k, v| v[:failure_rate] != 0 }
- .sort_by { |k, v| -v[:runs] }
- .to_h
+ stable_specs = test_runs(reliable: false).each do |stage, stage_specs|
+ stage_specs.transform_values! do |group_specs|
+ group_specs.reject { |k, v| v[:failure_rate] != 0 }
+ .sort_by { |k, v| -v[:runs] }
+ .to_h
+ end
end
-
- stable_specs.reject { |k, v| v.empty? }
+ stable_specs.transform_values { |v| v.reject { |_, v| v.empty? } }.reject { |_, v| v.empty? }
end
end
@@ -292,14 +293,26 @@ def stable_test_runs
# @return [Hash]
def unstable_reliable_test_runs
@top_unstable_reliable ||= begin
- unstable = test_runs(reliable: true).transform_values do |specs|
- specs
- .reject { |k, v| v[:failure_rate] == 0 }
- .sort_by { |k, v| -v[:failure_rate] }
- .to_h
+ unstable = test_runs(reliable: true).each do |_stage, stage_specs|
+ stage_specs.transform_values! do |group_specs|
+ group_specs.reject { |_, v| v[:failure_rate] == 0 }
+ .sort_by { |_, v| -v[:failure_rate] }
+ .to_h
+ end
end
+ unstable.transform_values { |v| v.reject { |_, v| v.empty? } }.reject { |_, v| v.empty? }
+ end
+ end
- unstable.reject { |k, v| v.empty? }
+ def print_results(results)
+ results.each do |_stage, stage_results|
+ stage_results.each_value { |group_results_table| puts "#{group_results_table}\n\n" }
+ end
+ end
+
+ def test_count(test_runs)
+ test_runs.sum do |_stage, stage_results|
+ stage_results.sum { |_product_group, group_results| group_results.count }
end
end
@@ -368,16 +381,14 @@ def test_runs(reliable:)
all_runs.each_with_object(Hash.new { |hsh, key| hsh[key] = {} }) do |table, result|
records = table.records.sort_by { |record| record.values["_time"] }
- # skip specs that executed less time than defined by range or stopped executing before report date
- # offset 1 day due to how schedulers are configured and first run can be 1 day later
- next if (Date.today - Date.parse(records.first.values["_time"])).to_i < (range - 1)
- next if (Date.today - Date.parse(records.last.values["_time"])).to_i > 1
+ next if within_execution_range(records.first.values["_time"], records.last.values["_time"])
last_record = records.last.values
name = last_record["name"]
file = last_record["file_path"].split("/").last
link = FEATURES_DIR + last_record["file_path"]
stage = last_record["stage"] || "unknown"
+ product_group = last_record["product_group"] || "unknown"
runs = records.count
@@ -394,7 +405,8 @@ def test_runs(reliable:)
[r.values["failure_exception"], r.values["job_url"]]
end
- result[stage][name] = {
+ result[stage][product_group] ||= {}
+ result[stage][product_group][name] = {
file: file,
link: link,
runs: runs,
@@ -415,6 +427,16 @@ def allowed_failure?(failure_exception)
ALLOWED_EXCEPTION_PATTERNS.any? { |pattern| pattern.match?(failure_exception) }
end
+ # Returns true if first_time is before our range, or if last_time is before report date
+ # offset 1 day due to how schedulers are configured and first run can be 1 day later
+ #
+ # @param [String] first_time
+ # @param [String] last_time
+ # @return [Boolean]
+ def within_execution_range(first_time, last_time)
+ (Date.today - Date.parse(first_time)).to_i < (range - 1) || (Date.today - Date.parse(last_time)).to_i > 1
+ end
+
# Flux query
#
# @param [Boolean] reliable
diff --git a/qa/spec/tools/reliable_report_spec.rb b/qa/spec/tools/reliable_report_spec.rb
index cf5c9dea7943..eca3baffdf00 100644
--- a/qa/spec/tools/reliable_report_spec.rb
+++ b/qa/spec/tools/reliable_report_spec.rb
@@ -18,10 +18,19 @@
let(:runs) do
values = {
- "name" => "stable spec",
+ "name" => "stable spec1",
+ "status" => "passed",
+ "file_path" => "some/spec.rb",
+ "stage" => "create",
+ "product_group" => "code_review",
+ "_time" => time
+ }
+ more_values = {
+ "name" => "stable spec2",
"status" => "passed",
"file_path" => "some/spec.rb",
"stage" => "manage",
+ "product_group" => "import_and_integrate",
"_time" => time
}
[
@@ -32,6 +41,14 @@
instance_double("InfluxDB2::FluxRecord", values: values),
instance_double("InfluxDB2::FluxRecord", values: values.merge({ "_time" => Time.now.to_s }))
]
+ ),
+ instance_double(
+ "InfluxDB2::FluxTable",
+ records: [
+ instance_double("InfluxDB2::FluxRecord", values: more_values),
+ instance_double("InfluxDB2::FluxRecord", values: more_values),
+ instance_double("InfluxDB2::FluxRecord", values: more_values.merge({ "_time" => Time.now.to_s }))
+ ]
)
]
end
@@ -42,6 +59,17 @@
"status" => "failed",
"file_path" => "some/spec.rb",
"stage" => "create",
+ "product_group" => "code_review",
+ "failure_exception" => failure_message,
+ "job_url" => "https://job/url",
+ "_time" => time
+ }
+ more_values = {
+ "name" => "unstable spec",
+ "status" => "failed",
+ "file_path" => "some/spec.rb",
+ "stage" => "manage",
+ "product_group" => "import_and_integrate",
"failure_exception" => failure_message,
"job_url" => "https://job/url",
"_time" => time
@@ -54,6 +82,14 @@
instance_double("InfluxDB2::FluxRecord", values: values),
instance_double("InfluxDB2::FluxRecord", values: values.merge({ "_time" => Time.now.to_s }))
]
+ ),
+ instance_double(
+ "InfluxDB2::FluxTable",
+ records: [
+ instance_double("InfluxDB2::FluxRecord", values: { **more_values, "status" => "passed" }),
+ instance_double("InfluxDB2::FluxRecord", values: more_values),
+ instance_double("InfluxDB2::FluxRecord", values: more_values.merge({ "_time" => Time.now.to_s }))
+ ]
)
]
end
@@ -89,14 +125,12 @@ def flux_query(reliable:)
QUERY
end
- def markdown_section(summary, result, stage, type)
+ def expected_stage_markdown(result, stage, product_group, type)
<<~SECTION.strip
- #{summary_table(summary, type, true)}
-
- ## #{stage} (1)
+ ## #{stage.capitalize} (1)
<details>
- <summary>Executions table</summary>
+ <summary>Executions table ~\"group::#{product_group}\" (1)</summary>
#{table(result, ['NAME', 'RUNS', 'FAILURES', 'FAILURE RATE'], "Top #{type} specs in '#{stage}' stage for past #{range} days", true)}
@@ -104,7 +138,7 @@ def markdown_section(summary, result, stage, type)
SECTION
end
- def summary_table(summary, type, markdown = false)
+ def expected_summary_table(summary, type, markdown = false)
table(summary, %w[STAGE COUNT], "#{type.capitalize} spec summary for past #{range} days".ljust(50), markdown)
end
@@ -241,28 +275,36 @@ def exceptions_markdown(exceptions_and_job_urls)
let(:expected_issue_body) do
<<~TXT.strip
- [[_TOC_]]
+ [[_TOC_]]
+
+ # Candidates for promotion to reliable (#{Date.today - range} - #{Date.today})
+
+ Total amount: **2**
+
+ #{expected_summary_table([['create', 1], ['manage', 1]], :stable, true)}
- # Candidates for promotion to reliable (#{Date.today - range} - #{Date.today})
+ #{expected_stage_markdown([[name_column('stable spec1'), 3, 0, '0%']], 'create', 'code review', :stable)}
- Total amount: **1**
+ #{expected_stage_markdown([[name_column('stable spec2'), 3, 0, '0%']], 'manage', 'import and integrate', :stable)}
- #{markdown_section([['manage', 1]], [[name_column('stable spec'), 3, 0, '0%']], 'manage', 'stable')}
+ # Reliable specs with failures (#{Date.today - range} - #{Date.today})
- # Reliable specs with failures (#{Date.today - range} - #{Date.today})
+ Total amount: **2**
- Total amount: **1**
+ #{expected_summary_table([['create', 1], ['manage', 1]], :unstable, true)}
- #{markdown_section([['create', 1]], [[name_column('unstable spec', { failure_message => 'https://job/url' }), 3, 2, '66.67%']], 'create', 'unstable')}
+ #{expected_stage_markdown([[name_column('unstable spec', { failure_message => 'https://job/url' }), 3, 2, '66.67%']], 'create', 'code review', :unstable)}
+
+ #{expected_stage_markdown([[name_column('unstable spec', { failure_message => 'https://job/url' }), 3, 2, '66.67%']], 'manage', 'import and integrate', :unstable)}
TXT
end
let(:expected_slack_text) do
<<~TEXT
- ```#{summary_table([['manage', 1]], 'stable')}```
- ```#{summary_table([['create', 1]], 'unstable')}```
+ ```#{expected_summary_table([['create', 1], ['manage', 1]], :stable)}```
+ ```#{expected_summary_table([['create', 1], ['manage', 1]], :unstable)}```
- #{issue_url}
+ #{issue_url}
TEXT
end
@@ -274,22 +316,26 @@ def exceptions_markdown(exceptions_and_job_urls)
let(:expected_issue_body) do
<<~TXT.strip
- [[_TOC_]]
+ [[_TOC_]]
+
+ # Candidates for promotion to reliable (#{Date.today - range} - #{Date.today})
+
+ Total amount: **2**
- # Candidates for promotion to reliable (#{Date.today - range} - #{Date.today})
+ #{expected_summary_table([['create', 1], ['manage', 1]], :stable, true)}
- Total amount: **1**
+ #{expected_stage_markdown([[name_column('stable spec1'), 3, 0, '0%']], 'create', 'code review', :stable)}
- #{markdown_section([['manage', 1]], [[name_column('stable spec'), 3, 0, '0%']], 'manage', 'stable')}
+ #{expected_stage_markdown([[name_column('stable spec2'), 3, 0, '0%']], 'manage', 'import and integrate', :stable)}
TXT
end
let(:expected_slack_text) do
<<~TEXT
- ```#{summary_table([['manage', 1]], 'stable')}```
- ```#{summary_table([], 'unstable')}```
+ ```#{expected_summary_table([['create', 1], ['manage', 1]], :stable)}```
+ ```#{expected_summary_table([], :unstable)}```
- #{issue_url}
+ #{issue_url}
TEXT
end
--
GitLab