#!/usr/bin/env ruby # frozen_string_literal: true require 'optparse' require 'json' require_relative 'api/pipeline_failed_jobs' require_relative 'api/create_issue' class CreatePipelineFailureIncident DEFAULT_OPTIONS = { project: nil, incident_json_file: 'incident.json' }.freeze DEFAULT_LABELS = ['Engineering Productivity'].freeze def initialize(options) @project = options.delete(:project) @api_token = options.delete(:api_token) end def execute payload = { issue_type: 'incident', title: title, description: description, labels: incident_labels } CreateIssue.new(project: project, api_token: api_token).execute(payload) end private attr_reader :project, :api_token def failed_jobs @failed_jobs ||= PipelineFailedJobs.new(API::DEFAULT_OPTIONS.dup.merge(exclude_allowed_to_fail_jobs: true)).execute end def now @now ||= Time.now.utc end def title "#{now.strftime('%A %F %R UTC')} - `#{ENV['CI_PROJECT_PATH']}` broken `#{ENV['CI_COMMIT_REF_NAME']}` " \ "with #{failed_jobs.size} failed jobs" end def description <<~MARKDOWN ## #{project_link} pipeline #{pipeline_link} failed **Branch: #{branch_link}** **Commit: #{commit_link}** **Triggered by** #{triggered_by_link} • **Source:** #{source} • **Duration:** #{pipeline_duration} minutes **Failed jobs (#{failed_jobs.size}):** #{failed_jobs_list} ### General guidelines Follow the [Broken `master` handbook guide](https://about.gitlab.com/handbook/engineering/workflow/#broken-master). ### Investigation **Be sure to fill the `Timeline` for this incident.** 1. If the failure is new, and looks like a potential flaky failure, you can retry the failing job. Make sure to mention the retry in the `Timeline` and leave a link to the retried job. 1. If the failure looks like a broken `master`, communicate the broken `master` in Slack using the "Broadcast Master Broken" workflow: - Click the Shortcut lightning bolt icon in the `#master-broken` channel and select "Broadcast Master Broken". - Click "Continue the broadcast" after the automated message in `#master-broken`. ### Pre-resolution If you believe that there's an easy resolution by either: - Reverting a particular merge request. Make sure to add the ~"pipeline:revert" label in that case, to speed up the revert pipeline. - Making a quick fix (for example, one line or a few similar simple changes in a few lines). You can create a merge request, assign to any available maintainer, and ping people that were involved/related to the introduction of the failure. Additionally, a message can be posted in `#backend_maintainers` or `#frontend_maintainers` to get a maintainer take a look at the fix ASAP. ### Resolution Follow [the Resolution steps from the handbook](https://about.gitlab.com/handbook/engineering/workflow/#responsibilities-of-the-resolution-dri). MARKDOWN end def incident_labels master_broken_label = if ENV['CI_PROJECT_NAME'] == 'gitlab-foss' 'master:foss-broken' else 'master:broken' end DEFAULT_LABELS.dup << master_broken_label end def pipeline_link "[##{ENV['CI_PIPELINE_ID']}](#{ENV['CI_PIPELINE_URL']})" end def branch_link "[`#{ENV['CI_COMMIT_REF_NAME']}`](#{ENV['CI_PROJECT_URL']}/-/commits/#{ENV['CI_COMMIT_REF_NAME']})" end def pipeline_duration ((Time.now - Time.parse(ENV['CI_PIPELINE_CREATED_AT'])) / 60.to_f).round(2) end def commit_link "[#{ENV['CI_COMMIT_TITLE']}](#{ENV['CI_PROJECT_URL']}/-/commit/#{ENV['CI_COMMIT_SHA']})" end def source "`#{ENV['CI_PIPELINE_SOURCE']}`" end def project_link "[#{ENV['CI_PROJECT_PATH']}](#{ENV['CI_PROJECT_URL']})" end def triggered_by_link "[#{ENV['GITLAB_USER_NAME']}](#{ENV['CI_SERVER_URL']}/#{ENV['GITLAB_USER_LOGIN']})" end def failed_jobs_list_for_title failed_jobs.map(&:name).join(', ') end def failed_jobs_list failed_jobs.map { |job| "- [#{job.name}](#{job.web_url})" }.join("\n") end end if $PROGRAM_NAME == __FILE__ options = CreatePipelineFailureIncident::DEFAULT_OPTIONS.dup OptionParser.new do |opts| opts.on("-p", "--project PROJECT", String, "Project where to create the incident (defaults to "\ "`#{CreatePipelineFailureIncident::DEFAULT_OPTIONS[:project]}`)") do |value| options[:project] = value end opts.on("-f", "--incident-json-file file_path", String, "Path to a file where to save the incident JSON data "\ "(defaults to `#{CreatePipelineFailureIncident::DEFAULT_OPTIONS[:incident_json_file]}`)") do |value| options[:incident_json_file] = value end opts.on("-t", "--api-token API_TOKEN", String, "A valid Project token with the `Reporter` role and `api` scope "\ "to create the incident") do |value| options[:api_token] = value end opts.on("-h", "--help", "Prints this help") do puts opts exit end end.parse! incident_json_file = options.delete(:incident_json_file) CreatePipelineFailureIncident.new(options).execute.tap do |incident| File.write(incident_json_file, JSON.pretty_generate(incident.to_h)) if incident_json_file end end