E2E test: group reliable report by product group

qa/qa/tools/reliable_report.rb

@@ -59,14 +59,14 @@ def self.run(range: 14, report_in_issue_and_slack: "false")
       #
       # @return [void]
       def print_report
-        puts "#{stable_summary_table}\n\n"
+        puts "#{summary_table(stable: true)}\n\n"
         puts "Total amount: #{stable_test_runs.sum { |_k, v| v.count }}\n\n"
-        stable_results_tables.each { |stage, table| puts "#{table}\n\n" }
+        print_results(stable_results_tables)
         return puts("No unstable reliable tests present!".colorize(:yellow)) if unstable_reliable_test_runs.empty?
 
-        puts "#{unstable_summary_table}\n\n"
+        puts "#{summary_table(stable: false)}\n\n"
         puts "Total amount: #{unstable_reliable_test_runs.sum { |_k, v| v.count }}\n\n"
-        unstable_reliable_results_tables.each { |stage, table| puts "#{table}\n\n" }
+        print_results(unstable_reliable_results_tables)
       end
 
       # Create report issue
@@ -89,8 +89,8 @@ def report_in_issue_and_slack
         notifier.post(
           icon_emoji: ":tanuki-protect:",
           text: <<~TEXT
-            ```#{stable_summary_table}```
-            ```#{unstable_summary_table}```
+            ```#{summary_table(stable: true)}```
+            ```#{summary_table(stable: false)}```
 
             #{web_url}
           TEXT
@@ -173,39 +173,30 @@ def report_issue_body
         issue = []
         issue << "[[_TOC_]]"
         issue << "# Candidates for promotion to reliable #{execution_interval}"
-        issue << "Total amount: **#{stable_test_runs.sum { |_k, v| v.count }}**"
-        issue << stable_summary_table(markdown: true).to_s
+        issue << "Total amount: **#{test_count(stable_test_runs)}**"
+        issue << summary_table(markdown: true, stable: true).to_s
         issue << results_markdown(:stable)
         return issue.join("\n\n") if unstable_reliable_test_runs.empty?
 
         issue << "# Reliable specs with failures #{execution_interval}"
-        issue << "Total amount: **#{unstable_reliable_test_runs.sum { |_k, v| v.count }}**"
-        issue << unstable_summary_table(markdown: true).to_s
+        issue << "Total amount: **#{test_count(unstable_reliable_test_runs)}**"
+        issue << summary_table(markdown: true, stable: false).to_s
         issue << results_markdown(:unstable)
         issue.join("\n\n")
       end
 
-      # Stable spec summary table
+      # Spec summary table
       #
       # @param [Boolean] markdown
+      # @param [Boolean] stable
       # @return [Terminal::Table]
-      def stable_summary_table(markdown: false)
+      def summary_table(markdown: false, stable: true)
+        test_runs = stable ? stable_test_runs : unstable_reliable_test_runs
         terminal_table(
-          rows: stable_test_runs.map { |stage, specs| [stage, specs.length] },
-          title: "Stable spec summary for past #{range} days".ljust(50),
-          headings: %w[STAGE COUNT],
-          markdown: markdown
-        )
-      end
-
-      # Unstable reliable summary table
-      #
-      # @param [Boolean] markdown
-      # @return [Terminal::Table]
-      def unstable_summary_table(markdown: false)
-        terminal_table(
-          rows: unstable_reliable_test_runs.map { |stage, specs| [stage, specs.length] },
-          title: "Unstable spec summary for past #{range} days".ljust(50),
+          rows: test_runs.map do |stage, stage_specs|
+            [stage, stage_specs.sum { |_k, group_specs| group_specs.length }]
+          end,
+          title: "#{stable ? 'Stable' : 'Unstable'} spec summary for past #{range} days".ljust(50),
           headings: %w[STAGE COUNT],
           markdown: markdown
         )
@@ -233,41 +224,51 @@ def unstable_reliable_results_tables(markdown: false)
       # @return [String]
       def results_markdown(type)
         runs = type == :stable ? stable_test_runs : unstable_reliable_test_runs
-        results_tables(type, markdown: true).map do |stage, table|
-          <<~STAGE.strip
-            ## #{stage} (#{runs[stage].count})
+        results_tables(type, markdown: true).map do |stage, group_tables|
+          markdown = "## #{stage.capitalize} (#{runs[stage].sum { |_k, group_runs| group_runs.count }})\n\n"
 
-            <details>
-            <summary>Executions table</summary>
+          markdown << group_tables.map { |product_group, table| group_results_markdown(product_group, table) }.join
+        end.join("\n\n")
+      end
 
-            #{table}
+      # Markdown formatted group results table
+      #
+      # @param [String] product_group
+      # @param [Terminal::Table] table
+      # @return [String]
+      def group_results_markdown(product_group, table)
+        <<~MARKDOWN.chomp
+          <details>
+          <summary>Executions table ~"group::#{product_group.tr('_', ' ')}" (#{table.rows.size})</summary>
 
-            </details>
-          STAGE
-        end.join("\n\n")
+          #{table}
+
+          </details>
+        MARKDOWN
       end
 
       # Results table
       #
       # @param [Symbol] type result type - :stable, :unstable
       # @param [Boolean] markdown
-      # @return [Hash<Symbol, Terminal::Table>]
+      # @return [Hash<String, Hash<String, Terminal::Table>>] grouped by stage and product_group
       def results_tables(type, markdown: false)
         (type == :stable ? stable_test_runs : unstable_reliable_test_runs).to_h do |stage, specs|
-          headings = ["name", "runs", "failures", "failure rate"]
-
-          [stage, terminal_table(
-            title: "Top #{type} specs in '#{stage}' stage for past #{range} days",
-            headings: headings.map(&:upcase),
-            markdown: markdown,
-            rows: specs.map do |k, v|
-              [
-                name_column(name: k, file: v[:file], link: v[:link],
-                  exceptions_and_job_urls: v[:exceptions_and_job_urls], markdown: markdown),
-                *table_params(v.values)
-              ]
-            end
-          )]
+          headings = ['NAME', 'RUNS', 'FAILURES', 'FAILURE RATE'].freeze
+          [stage, specs.transform_values do |group_specs|
+            terminal_table(
+              title: "Top #{type} specs in '#{stage}::#{specs.key(group_specs)}' group for past #{range} days",
+              headings: headings,
+              markdown: markdown,
+              rows: group_specs.map do |name, result|
+                [
+                  name_column(name: name, file: result[:file], link: result[:link],
+                    exceptions_and_job_urls: result[:exceptions_and_job_urls], markdown: markdown),
+                  *table_params(result.values)
+                ]
+              end
+            )
+          end]
         end
       end
 
@@ -276,14 +277,14 @@ def results_tables(type, markdown: false)
       # @return [Hash]
       def stable_test_runs
         @top_stable ||= begin
-          stable_specs = test_runs(reliable: false).transform_values do |specs|
-            specs
-              .reject { |k, v| v[:failure_rate] != 0 }
-              .sort_by { |k, v| -v[:runs] }
-              .to_h
+          stable_specs = test_runs(reliable: false).each do |stage, stage_specs|
+            stage_specs.transform_values! do |group_specs|
+              group_specs.reject { |k, v| v[:failure_rate] != 0 }
+                         .sort_by { |k, v| -v[:runs] }
+                         .to_h
+            end
           end
-
-          stable_specs.reject { |k, v| v.empty? }
+          stable_specs.transform_values { |v| v.reject { |_, v| v.empty? } }.reject { |_, v| v.empty? }
         end
       end
 
@@ -292,14 +293,26 @@ def stable_test_runs
       # @return [Hash]
       def unstable_reliable_test_runs
         @top_unstable_reliable ||= begin
-          unstable = test_runs(reliable: true).transform_values do |specs|
-            specs
-              .reject { |k, v| v[:failure_rate] == 0 }
-              .sort_by { |k, v| -v[:failure_rate] }
-              .to_h
+          unstable = test_runs(reliable: true).each do |_stage, stage_specs|
+            stage_specs.transform_values! do |group_specs|
+              group_specs.reject { |_, v| v[:failure_rate] == 0 }
+                         .sort_by { |_, v| -v[:failure_rate] }
+                         .to_h
+            end
           end
+          unstable.transform_values { |v| v.reject { |_, v| v.empty? } }.reject { |_, v| v.empty? }
+        end
+      end
 
-          unstable.reject { |k, v| v.empty? }
+      def print_results(results)
+        results.each do |_stage, stage_results|
+          stage_results.each_value { |group_results_table| puts "#{group_results_table}\n\n" }
+        end
+      end
+
+      def test_count(test_runs)
+        test_runs.sum do |_stage, stage_results|
+          stage_results.sum { |_product_group, group_results| group_results.count }
         end
       end
 
@@ -368,16 +381,14 @@ def test_runs(reliable:)
         all_runs.each_with_object(Hash.new { |hsh, key| hsh[key] = {} }) do |table, result|
           records = table.records.sort_by { |record| record.values["_time"] }
 
-          # skip specs that executed less time than defined by range or stopped executing before report date
-          # offset 1 day due to how schedulers are configured and first run can be 1 day later
-          next if (Date.today - Date.parse(records.first.values["_time"])).to_i < (range - 1)
-          next if (Date.today - Date.parse(records.last.values["_time"])).to_i > 1
+          next if within_execution_range(records.first.values["_time"], records.last.values["_time"])
 
           last_record = records.last.values
           name = last_record["name"]
           file = last_record["file_path"].split("/").last
           link = FEATURES_DIR + last_record["file_path"]
           stage = last_record["stage"] || "unknown"
+          product_group = last_record["product_group"] || "unknown"
 
           runs = records.count
 
@@ -394,7 +405,8 @@ def test_runs(reliable:)
             [r.values["failure_exception"], r.values["job_url"]]
           end
 
-          result[stage][name] = {
+          result[stage][product_group] ||= {}
+          result[stage][product_group][name] = {
             file: file,
             link: link,
             runs: runs,
@@ -415,6 +427,16 @@ def allowed_failure?(failure_exception)
         ALLOWED_EXCEPTION_PATTERNS.any? { |pattern| pattern.match?(failure_exception) }
       end
 
+      # Returns true if first_time is before our range, or if last_time is before report date
+      # offset 1 day due to how schedulers are configured and first run can be 1 day later
+      #
+      # @param [String] first_time
+      # @param [String] last_time
+      # @return [Boolean]
+      def within_execution_range(first_time, last_time)
+        (Date.today - Date.parse(first_time)).to_i < (range - 1) || (Date.today - Date.parse(last_time)).to_i > 1
+      end
+
       # Flux query
       #
       # @param [Boolean] reliable

qa/spec/tools/reliable_report_spec.rb

@@ -18,10 +18,19 @@
 
     let(:runs) do
       values = {
-        "name" => "stable spec",
+        "name" => "stable spec1",
+        "status" => "passed",
+        "file_path" => "some/spec.rb",
+        "stage" => "create",
+        "product_group" => "code_review",
+        "_time" => time
+      }
+      more_values = {
+        "name" => "stable spec2",
         "status" => "passed",
         "file_path" => "some/spec.rb",
         "stage" => "manage",
+        "product_group" => "import_and_integrate",
         "_time" => time
       }
       [
@@ -32,6 +41,14 @@
             instance_double("InfluxDB2::FluxRecord", values: values),
             instance_double("InfluxDB2::FluxRecord", values: values.merge({ "_time" => Time.now.to_s }))
           ]
+        ),
+        instance_double(
+          "InfluxDB2::FluxTable",
+          records: [
+            instance_double("InfluxDB2::FluxRecord", values: more_values),
+            instance_double("InfluxDB2::FluxRecord", values: more_values),
+            instance_double("InfluxDB2::FluxRecord", values: more_values.merge({ "_time" => Time.now.to_s }))
+          ]
         )
       ]
     end
@@ -42,6 +59,17 @@
         "status" => "failed",
         "file_path" => "some/spec.rb",
         "stage" => "create",
+        "product_group" => "code_review",
+        "failure_exception" => failure_message,
+        "job_url" => "https://job/url",
+        "_time" => time
+      }
+      more_values = {
+        "name" => "unstable spec",
+        "status" => "failed",
+        "file_path" => "some/spec.rb",
+        "stage" => "manage",
+        "product_group" => "import_and_integrate",
         "failure_exception" => failure_message,
         "job_url" => "https://job/url",
         "_time" => time
@@ -54,6 +82,14 @@
             instance_double("InfluxDB2::FluxRecord", values: values),
             instance_double("InfluxDB2::FluxRecord", values: values.merge({ "_time" => Time.now.to_s }))
           ]
+        ),
+        instance_double(
+          "InfluxDB2::FluxTable",
+          records: [
+            instance_double("InfluxDB2::FluxRecord", values: { **more_values, "status" => "passed" }),
+            instance_double("InfluxDB2::FluxRecord", values: more_values),
+            instance_double("InfluxDB2::FluxRecord", values: more_values.merge({ "_time" => Time.now.to_s }))
+          ]
         )
       ]
     end
@@ -89,14 +125,12 @@ def flux_query(reliable:)
       QUERY
     end
 
-    def markdown_section(summary, result, stage, type)
+    def expected_stage_markdown(result, stage, product_group, type)
       <<~SECTION.strip
-        #{summary_table(summary, type, true)}
-
-        ## #{stage} (1)
+        ## #{stage.capitalize} (1)
 
         <details>
-        <summary>Executions table</summary>
+        <summary>Executions table ~\"group::#{product_group}\" (1)</summary>
 
         #{table(result, ['NAME', 'RUNS', 'FAILURES', 'FAILURE RATE'], "Top #{type} specs in '#{stage}' stage for past #{range} days", true)}
 
@@ -104,7 +138,7 @@ def markdown_section(summary, result, stage, type)
       SECTION
     end
 
-    def summary_table(summary, type, markdown = false)
+    def expected_summary_table(summary, type, markdown = false)
       table(summary, %w[STAGE COUNT], "#{type.capitalize} spec summary for past #{range} days".ljust(50), markdown)
     end
 
@@ -241,28 +275,36 @@ def exceptions_markdown(exceptions_and_job_urls)
 
         let(:expected_issue_body) do
           <<~TXT.strip
-          [[_TOC_]]
+            [[_TOC_]]
+
+            # Candidates for promotion to reliable (#{Date.today - range} - #{Date.today})
+
+            Total amount: **2**
+
+            #{expected_summary_table([['create', 1], ['manage', 1]], :stable, true)}
 
-          # Candidates for promotion to reliable (#{Date.today - range} - #{Date.today})
+            #{expected_stage_markdown([[name_column('stable spec1'), 3, 0, '0%']], 'create', 'code review', :stable)}
 
-          Total amount: **1**
+            #{expected_stage_markdown([[name_column('stable spec2'), 3, 0, '0%']], 'manage', 'import and integrate', :stable)}
 
-          #{markdown_section([['manage', 1]], [[name_column('stable spec'), 3, 0, '0%']], 'manage', 'stable')}
+            # Reliable specs with failures (#{Date.today - range} - #{Date.today})
 
-          # Reliable specs with failures (#{Date.today - range} - #{Date.today})
+            Total amount: **2**
 
-          Total amount: **1**
+            #{expected_summary_table([['create', 1], ['manage', 1]], :unstable, true)}
 
-          #{markdown_section([['create', 1]], [[name_column('unstable spec', { failure_message => 'https://job/url' }), 3, 2, '66.67%']], 'create', 'unstable')}
+            #{expected_stage_markdown([[name_column('unstable spec', { failure_message => 'https://job/url' }), 3, 2, '66.67%']], 'create', 'code review', :unstable)}
+
+            #{expected_stage_markdown([[name_column('unstable spec', { failure_message => 'https://job/url' }), 3, 2, '66.67%']], 'manage', 'import and integrate', :unstable)}
           TXT
         end
 
         let(:expected_slack_text) do
           <<~TEXT
-              ```#{summary_table([['manage', 1]], 'stable')}```
-              ```#{summary_table([['create', 1]], 'unstable')}```
+            ```#{expected_summary_table([['create', 1], ['manage', 1]], :stable)}```
+            ```#{expected_summary_table([['create', 1], ['manage', 1]], :unstable)}```
 
-              #{issue_url}
+            #{issue_url}
           TEXT
         end
 
@@ -274,22 +316,26 @@ def exceptions_markdown(exceptions_and_job_urls)
 
         let(:expected_issue_body) do
           <<~TXT.strip
-          [[_TOC_]]
+            [[_TOC_]]
+
+            # Candidates for promotion to reliable (#{Date.today - range} - #{Date.today})
+
+            Total amount: **2**
 
-          # Candidates for promotion to reliable (#{Date.today - range} - #{Date.today})
+            #{expected_summary_table([['create', 1], ['manage', 1]], :stable, true)}
 
-          Total amount: **1**
+            #{expected_stage_markdown([[name_column('stable spec1'), 3, 0, '0%']], 'create', 'code review', :stable)}
 
-          #{markdown_section([['manage', 1]], [[name_column('stable spec'), 3, 0, '0%']], 'manage', 'stable')}
+            #{expected_stage_markdown([[name_column('stable spec2'), 3, 0, '0%']], 'manage', 'import and integrate', :stable)}
           TXT
         end
 
         let(:expected_slack_text) do
           <<~TEXT
-              ```#{summary_table([['manage', 1]], 'stable')}```
-              ```#{summary_table([], 'unstable')}```
+            ```#{expected_summary_table([['create', 1], ['manage', 1]], :stable)}```
+            ```#{expected_summary_table([], :unstable)}```
 
-              #{issue_url}
+            #{issue_url}
           TEXT
         end