ci: Notify master failures in Slack

Signed-off-by: Rémy Coutable <remy@rymai.me>

.gitlab-ci.yml

@@ -49,6 +49,7 @@ workflow:
     - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH && $CI_PIPELINE_SOURCE == "schedule" && $SCHEDULE_TYPE == "maintenance"'
       variables:
         CRYSTALBALL: "true"
+        NOTIFY_PIPELINE_FAILURE_CHANNEL: "master-broken"
     # Run pipelines for ruby3 branch
     - if: '$CI_COMMIT_BRANCH == "ruby3"'
       variables:
@@ -63,6 +64,8 @@ workflow:
         GITLAB_DEPENDENCY_PROXY_ADDRESS: ""
     # For `$CI_DEFAULT_BRANCH` branch, create a pipeline (this includes on schedules, pushes, merges, etc.).
     - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+      variables:
+        NOTIFY_PIPELINE_FAILURE_CHANNEL: "master-broken"
     # For tags, create a pipeline.
     - if: '$CI_COMMIT_TAG'
     # If `$GITLAB_INTERNAL` isn't set, don't create a pipeline.

.gitlab/ci/notify.gitlab-ci.yml

 .notify-slack:
-  image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}alpine
+  image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}alpine/curl
   stage: notify
   dependencies: []
   cache: {}
   variables:
     MERGE_REQUEST_URL: ${CI_MERGE_REQUEST_PROJECT_URL}/-/merge_requests/${CI_MERGE_REQUEST_IID}
   before_script:
-    - apk update && apk add git curl bash
+    - apk update && apk add git bash
     - echo "NOTIFY_CHANNEL is ${NOTIFY_CHANNEL}"
     - echo "CI_PIPELINE_URL is ${CI_PIPELINE_URL}"
 
@@ -34,13 +34,28 @@ notify-security-pipeline:
     - scripts/slack ${NOTIFY_CHANNEL} "<!subteam^S0127FU8PDE> ☠️  Pipeline for merged result failed! ☠️ See ${CI_PIPELINE_URL} (triggered from ${MERGE_REQUEST_URL})" ci_failing "GitLab Release Tools Bot"
 
 notify-pipeline-failure:
-  extends:
-    - .notify-slack
+  extends: .notify-slack
+  image: ${GITLAB_DEPENDENCY_PROXY_ADDRESS}ruby:${RUBY_VERSION}
   rules:
-    - if: '$NOTIFY_PIPELINE_FAILURE_CHANNEL'
+    # Don't report child pipeline failures
+    - if: '$CI_PIPELINE_SOURCE == "parent_pipeline"'
+      when: never
+    - if: '$CI_SLACK_WEBHOOK_URL && $NOTIFY_PIPELINE_FAILURE_CHANNEL'
       when: on_failure
       allow_failure: true
   variables:
-    NOTIFY_CHANNEL: "${NOTIFY_PIPELINE_FAILURE_CHANNEL}"
+    SLACK_CHANNEL: "${NOTIFY_PIPELINE_FAILURE_CHANNEL}"
+    FAILED_PIPELINE_REPORT_FILE: "failed_pipeline_report.json"
+  before_script:
+    - source scripts/utils.sh
+    - apt-get update && apt-get install -y jq
+    - install_gitlab_gem
   script:
-    - scripts/slack ${NOTIFY_CHANNEL} "❌ \`${CI_COMMIT_REF_NAME}\` pipeline failed! See ${CI_PIPELINE_URL}" ci_failing "notify-pipeline-failure"
+    - scripts/generate-failed-pipeline-slack-message.rb
+    - |
+      curl -X POST -H 'Content-Type: application/json' --data @${FAILED_PIPELINE_REPORT_FILE} "$CI_SLACK_WEBHOOK_URL"
+  artifacts:
+    paths:
+      - ${FAILED_PIPELINE_REPORT_FILE}
+    when: always
+    expire_in: 2 days

scripts/api/pipeline_failed_jobs.rb

+# frozen_string_literal: true
+
+require 'gitlab'
+require 'optparse'
+require_relative 'default_options'
+
+class PipelineFailedJobs
+  def initialize(options)
+    @project = options.delete(:project)
+    @pipeline_id = options.delete(:pipeline_id)
+    @exclude_allowed_to_fail_jobs = options.delete(:exclude_allowed_to_fail_jobs)
+
+    # Force the token to be a string so that if api_token is nil, it's set to '',
+    # allowing unauthenticated requests (for forks).
+    api_token = options.delete(:api_token).to_s
+
+    warn "No API token given." if api_token.empty?
+
+    @client = Gitlab.client(
+      endpoint: options.delete(:endpoint) || API::DEFAULT_OPTIONS[:endpoint],
+      private_token: api_token
+    )
+  end
+
+  def execute
+    failed_jobs = []
+
+    client.pipeline_jobs(project, pipeline_id, scope: 'failed', per_page: 100).auto_paginate do |job|
+      next if exclude_allowed_to_fail_jobs && job.allow_failure
+
+      failed_jobs << job
+    end
+
+    failed_jobs
+  end
+
+  private
+
+  attr_reader :project, :pipeline_id, :exclude_allowed_to_fail_jobs, :client
+end

scripts/generate-failed-pipeline-slack-message.rb

+#!/usr/bin/env ruby
+
+# frozen_string_literal: true
+
+require_relative 'api/pipeline_failed_jobs'
+
+finder_options = API::DEFAULT_OPTIONS.dup.merge(exclude_allowed_to_fail_jobs: true)
+failed_jobs = PipelineFailedJobs.new(finder_options).execute
+
+class SlackReporter
+  def initialize(failed_jobs)
+    @failed_jobs = failed_jobs
+  end
+
+  def report
+    payload = {
+      channel: ENV['SLACK_CHANNEL'],
+      username: "Failed pipeline reporter",
+      icon_emoji: ":boom:",
+      text: "*#{title}*",
+      blocks: [
+        {
+          type: "section",
+          text: {
+            type: "mrkdwn",
+            text: "*#{title}*"
+          }
+        },
+        {
+          type: "section",
+          fields: [
+            {
+              type: "mrkdwn",
+              text: "*Commit*\n#{commit_link}"
+            },
+            {
+              type: "mrkdwn",
+              text: "*Triggered by*\n#{triggered_by_link}"
+            }
+          ]
+        },
+        {
+          type: "section",
+          fields: [
+            {
+              type: "mrkdwn",
+              text: "*Source*\n#{source}"
+            },
+            {
+              type: "mrkdwn",
+              text: "*Duration*\n#{pipeline_duration} minutes"
+            }
+          ]
+        },
+        {
+          type: "section",
+          text: {
+            type: "mrkdwn",
+            text: "*Failed jobs (#{failed_jobs.size}):* #{failed_jobs_list}"
+          }
+        }
+      ]
+    }
+
+    File.write(ENV['FAILED_PIPELINE_REPORT_FILE'], JSON.pretty_generate(payload))
+  end
+
+  private
+
+  attr_reader :failed_jobs
+
+  def title
+    "Pipeline #{pipeline_link} for #{branch_link} failed"
+  end
+
+  def pipeline_link
+    "<#{ENV['CI_PIPELINE_URL']}|##{ENV['CI_PIPELINE_ID']}>"
+  end
+
+  def branch_link
+    "<#{ENV['CI_PROJECT_URL']}/-/commits/#{ENV['CI_COMMIT_REF_NAME']}|`#{ENV['CI_COMMIT_REF_NAME']}`>"
+  end
+
+  def pipeline_duration
+    ((Time.now - Time.parse(ENV['CI_PIPELINE_CREATED_AT'])) / 60.to_f).round(2)
+  end
+
+  def commit_link
+    "<#{ENV['CI_PROJECT_URL']}/-/commit/#{ENV['CI_COMMIT_SHA']}|#{ENV['CI_COMMIT_TITLE']}>"
+  end
+
+  def source
+    "`#{ENV['CI_PIPELINE_SOURCE']}`"
+  end
+
+  def triggered_by_link
+    "<#{ENV['CI_SERVER_URL']}/#{ENV['GITLAB_USER_LOGIN']}|#{ENV['GITLAB_USER_NAME']}>"
+  end
+
+  def failed_jobs_list
+    failed_jobs.map { |job| "<#{job.web_url}|#{job.name}>" }.join(', ')
+  end
+end
+
+SlackReporter.new(failed_jobs).report