471 lines
15 KiB
Ruby
Executable file
471 lines
15 KiB
Ruby
Executable file
#!/usr/bin/env ruby
|
|
# frozen_string_literal: true
|
|
|
|
require 'gitlab'
|
|
|
|
module Trigger
|
|
def self.ee?
|
|
# Support former project name for `dev`
|
|
%w[gitlab gitlab-ee].include?(ENV['CI_PROJECT_NAME'])
|
|
end
|
|
|
|
def self.security?
|
|
%r{\Agitlab-org/security(\z|/)}.match?(ENV['CI_PROJECT_NAMESPACE'])
|
|
end
|
|
|
|
def self.non_empty_variable_value(variable)
|
|
variable_value = ENV[variable]
|
|
|
|
return if variable_value.nil? || variable_value.empty?
|
|
|
|
variable_value
|
|
end
|
|
|
|
class Base
|
|
# Can be overridden
|
|
def self.access_token
|
|
ENV['GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN']
|
|
end
|
|
|
|
def invoke!(post_comment: false, downstream_job_name: nil)
|
|
pipeline_variables = variables
|
|
|
|
puts "Triggering downstream pipeline on #{downstream_project_path}"
|
|
puts "with variables #{pipeline_variables}"
|
|
|
|
pipeline = gitlab_client(:downstream).run_trigger(
|
|
downstream_project_path,
|
|
trigger_token,
|
|
ref,
|
|
pipeline_variables)
|
|
|
|
puts "Triggered downstream pipeline: #{pipeline.web_url}\n"
|
|
puts "Waiting for downstream pipeline status"
|
|
|
|
Trigger::CommitComment.post!(pipeline, gitlab_client(:upstream)) if post_comment
|
|
downstream_job =
|
|
if downstream_job_name
|
|
gitlab_client(:downstream).pipeline_jobs(downstream_project_path, pipeline.id).auto_paginate.find do |potential_job|
|
|
potential_job.name == downstream_job_name
|
|
end
|
|
end
|
|
|
|
if downstream_job
|
|
Trigger::Job.new(downstream_project_path, downstream_job.id, gitlab_client(:downstream))
|
|
else
|
|
Trigger::Pipeline.new(downstream_project_path, pipeline.id, gitlab_client(:downstream))
|
|
end
|
|
end
|
|
|
|
private
|
|
|
|
# Override to trigger and work with pipeline on different GitLab instance
|
|
# type: :downstream -> downstream build and pipeline status
|
|
# type: :upstream -> this project, e.g. for posting comments
|
|
def gitlab_client(type)
|
|
# By default, always use the same client
|
|
@gitlab_client ||= Gitlab.client(
|
|
endpoint: 'https://gitlab.com/api/v4',
|
|
private_token: self.class.access_token
|
|
)
|
|
end
|
|
|
|
# Must be overridden
|
|
def downstream_project_path
|
|
raise NotImplementedError
|
|
end
|
|
|
|
# Must be overridden
|
|
def ref
|
|
raise NotImplementedError
|
|
end
|
|
|
|
# Can be overridden
|
|
def trigger_token
|
|
ENV['CI_JOB_TOKEN']
|
|
end
|
|
|
|
# Can be overridden
|
|
def extra_variables
|
|
{}
|
|
end
|
|
|
|
# Can be overridden
|
|
def version_param_value(version_file)
|
|
ENV[version_file]&.strip || File.read(version_file).strip
|
|
end
|
|
|
|
def variables
|
|
base_variables.merge(extra_variables).merge(version_file_variables)
|
|
end
|
|
|
|
def base_variables
|
|
# Use CI_MERGE_REQUEST_SOURCE_BRANCH_SHA for omnibus checkouts due to pipeline for merged results,
|
|
# and fallback to CI_COMMIT_SHA for the `detached` pipelines.
|
|
{
|
|
'GITLAB_REF_SLUG' => ENV['CI_COMMIT_TAG'] ? ENV['CI_COMMIT_REF_NAME'] : ENV['CI_COMMIT_REF_SLUG'],
|
|
'TRIGGERED_USER' => ENV['TRIGGERED_USER'] || ENV['GITLAB_USER_NAME'],
|
|
'TRIGGER_SOURCE' => ENV['CI_JOB_URL'],
|
|
'TOP_UPSTREAM_SOURCE_PROJECT' => ENV['CI_PROJECT_PATH'],
|
|
'TOP_UPSTREAM_SOURCE_JOB' => ENV['CI_JOB_URL'],
|
|
'TOP_UPSTREAM_SOURCE_SHA' => Trigger.non_empty_variable_value('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA') || ENV['CI_COMMIT_SHA'],
|
|
'TOP_UPSTREAM_SOURCE_REF' => ENV['CI_COMMIT_REF_NAME'],
|
|
'TOP_UPSTREAM_MERGE_REQUEST_PROJECT_ID' => ENV['CI_MERGE_REQUEST_PROJECT_ID'],
|
|
'TOP_UPSTREAM_MERGE_REQUEST_IID' => ENV['CI_MERGE_REQUEST_IID']
|
|
}
|
|
end
|
|
|
|
# Read version files from all components
|
|
def version_file_variables
|
|
Dir.glob("*_VERSION").each_with_object({}) do |version_file, params|
|
|
params[version_file] = version_param_value(version_file)
|
|
end
|
|
end
|
|
end
|
|
|
|
class Omnibus < Base
|
|
def self.access_token
|
|
# Default to "Multi-pipeline (from 'gitlab-org/gitlab' 'package-and-qa' job)" at https://gitlab.com/gitlab-org/build/omnibus-gitlab-mirror/-/settings/access_tokens
|
|
ENV['OMNIBUS_GITLAB_PROJECT_ACCESS_TOKEN'] || super
|
|
end
|
|
|
|
private
|
|
|
|
def downstream_project_path
|
|
ENV.fetch('OMNIBUS_PROJECT_PATH', 'gitlab-org/build/omnibus-gitlab-mirror')
|
|
end
|
|
|
|
def ref
|
|
ENV.fetch('OMNIBUS_BRANCH', 'master')
|
|
end
|
|
|
|
def extra_variables
|
|
# Use CI_MERGE_REQUEST_SOURCE_BRANCH_SHA (MR HEAD commit) so that the image is in sync with the assets and QA images.
|
|
# See https://docs.gitlab.com/ee/development/testing_guide/end_to_end/index.html#with-pipeline-for-merged-results.
|
|
# We also set IMAGE_TAG so the GitLab Docker image is tagged with that SHA.
|
|
source_sha = Trigger.non_empty_variable_value('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA') || ENV['CI_COMMIT_SHA']
|
|
|
|
{
|
|
'GITLAB_VERSION' => source_sha,
|
|
'IMAGE_TAG' => source_sha,
|
|
'QA_IMAGE' => ENV['QA_IMAGE'],
|
|
'SKIP_QA_DOCKER' => 'true',
|
|
'ALTERNATIVE_SOURCES' => 'true',
|
|
'SECURITY_SOURCES' => Trigger.security? ? 'true' : 'false',
|
|
'ee' => Trigger.ee? ? 'true' : 'false',
|
|
'QA_BRANCH' => ENV['QA_BRANCH'] || 'master',
|
|
'CACHE_UPDATE' => ENV['OMNIBUS_GITLAB_CACHE_UPDATE'],
|
|
'GITLAB_QA_OPTIONS' => ENV['GITLAB_QA_OPTIONS'],
|
|
'QA_TESTS' => ENV['QA_TESTS']
|
|
}
|
|
end
|
|
end
|
|
|
|
class CNG < Base
|
|
def self.access_token
|
|
# Default to "Multi-pipeline (from 'gitlab-org/gitlab' 'cloud-native-image' job)" at https://gitlab.com/gitlab-org/build/CNG/-/settings/access_tokens
|
|
ENV['CNG_PROJECT_ACCESS_TOKEN'] || super
|
|
end
|
|
|
|
private
|
|
|
|
def downstream_project_path
|
|
ENV.fetch('CNG_PROJECT_PATH', 'gitlab-org/build/CNG')
|
|
end
|
|
|
|
def ref
|
|
return ENV['CI_COMMIT_REF_NAME'] if ENV['CI_COMMIT_REF_NAME'] =~ /^[\d-]+-stable(-ee)?$/
|
|
|
|
ENV.fetch('CNG_BRANCH', 'master')
|
|
end
|
|
|
|
def extra_variables
|
|
edition = Trigger.ee? ? 'EE' : 'CE'
|
|
# Use CI_MERGE_REQUEST_SOURCE_BRANCH_SHA (MR HEAD commit) so that the image is in sync with the assets and QA images.
|
|
source_sha = Trigger.non_empty_variable_value('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA') || ENV['CI_COMMIT_SHA']
|
|
|
|
{
|
|
"ee" => Trigger.ee? ? "true" : "false",
|
|
"GITLAB_VERSION" => source_sha,
|
|
"GITLAB_TAG" => ENV['CI_COMMIT_TAG'],
|
|
"GITLAB_ASSETS_TAG" => ENV['CI_COMMIT_TAG'] ? ENV['CI_COMMIT_REF_NAME'] : source_sha,
|
|
"FORCE_RAILS_IMAGE_BUILDS" => 'true',
|
|
"#{edition}_PIPELINE" => 'true'
|
|
}
|
|
end
|
|
|
|
def version_param_value(_version_file)
|
|
raw_version = super
|
|
|
|
# if the version matches semver format, treat it as a tag and prepend `v`
|
|
if raw_version =~ Regexp.compile(/^\d+\.\d+\.\d+(-rc\d+)?(-ee)?$/)
|
|
"v#{raw_version}"
|
|
else
|
|
raw_version
|
|
end
|
|
end
|
|
end
|
|
|
|
class Docs < Base
|
|
def self.access_token
|
|
# Default to "DOCS_PROJECT_API_TOKEN" at https://gitlab.com/gitlab-org/gitlab-docs/-/settings/access_tokens
|
|
ENV['DOCS_PROJECT_API_TOKEN'] || super
|
|
end
|
|
|
|
SUCCESS_MESSAGE = <<~MSG
|
|
=> You should now be able to preview your changes under the following URL:
|
|
|
|
%<app_url>s
|
|
|
|
=> For more information, see the documentation
|
|
=> https://docs.gitlab.com/ee/development/documentation/index.html#previewing-the-changes-live
|
|
|
|
=> If something doesn't work, drop a line in the #docs chat channel.
|
|
MSG
|
|
|
|
def deploy!
|
|
invoke!.wait!
|
|
display_success_message
|
|
end
|
|
|
|
#
|
|
# Remove a remote branch in gitlab-docs.
|
|
#
|
|
def cleanup!
|
|
environment = gitlab_client(:downstream).environments(downstream_project_path, name: downstream_environment).first
|
|
return unless environment
|
|
|
|
environment = gitlab_client(:downstream).stop_environment(downstream_project_path, environment.id)
|
|
if environment.state == 'stopped'
|
|
puts "=> Downstream environment '#{downstream_environment}' stopped"
|
|
else
|
|
puts "=> Downstream environment '#{downstream_environment}' failed to stop."
|
|
end
|
|
end
|
|
|
|
private
|
|
|
|
def downstream_environment
|
|
"review/#{ref}#{review_slug}"
|
|
end
|
|
|
|
# We prepend the `-` here because we cannot use variable substitution in `environment.name`/`environment.url`
|
|
# Some projects (e.g. `omnibus-gitlab`) use this script for branch pipelines, so we fallback to using `CI_COMMIT_REF_SLUG` for those cases.
|
|
def review_slug
|
|
identifier = ENV['CI_MERGE_REQUEST_IID'] || ENV['CI_COMMIT_REF_SLUG']
|
|
|
|
"-#{project_slug}-#{identifier}"
|
|
end
|
|
|
|
def downstream_project_path
|
|
ENV.fetch('DOCS_PROJECT_PATH', 'gitlab-org/gitlab-docs')
|
|
end
|
|
|
|
def ref
|
|
ENV.fetch('DOCS_BRANCH', 'main')
|
|
end
|
|
|
|
# `gitlab-org/gitlab-docs` pipeline trigger "Triggered from gitlab-org/gitlab 'review-docs-deploy' job"
|
|
def trigger_token
|
|
ENV['DOCS_TRIGGER_TOKEN']
|
|
end
|
|
|
|
def extra_variables
|
|
{
|
|
"BRANCH_#{project_slug.upcase}" => ENV['CI_COMMIT_REF_NAME'],
|
|
"REVIEW_SLUG" => review_slug
|
|
}
|
|
end
|
|
|
|
def project_slug
|
|
case ENV['CI_PROJECT_PATH']
|
|
when 'gitlab-org/gitlab-foss'
|
|
'ce'
|
|
when 'gitlab-org/gitlab'
|
|
'ee'
|
|
when 'gitlab-org/gitlab-runner'
|
|
'runner'
|
|
when 'gitlab-org/omnibus-gitlab'
|
|
'omnibus'
|
|
when 'gitlab-org/charts/gitlab'
|
|
'charts'
|
|
end
|
|
end
|
|
|
|
# app_url is the URL of the `gitlab-docs` Review App URL defined in
|
|
# https://gitlab.com/gitlab-org/gitlab-docs/-/blob/b38038132cf82a24271bbb294dead7c2f529e275/.gitlab-ci.yml#L383
|
|
def app_url
|
|
"http://#{ref}#{review_slug}.#{ENV['DOCS_REVIEW_APPS_DOMAIN']}/#{project_slug}"
|
|
end
|
|
|
|
def display_success_message
|
|
puts format(SUCCESS_MESSAGE, app_url: app_url)
|
|
end
|
|
end
|
|
|
|
class DatabaseTesting < Base
|
|
IDENTIFIABLE_NOTE_TAG = 'gitlab-org/database-team/gitlab-com-database-testing:identifiable-note'
|
|
|
|
def self.access_token
|
|
ENV['GITLABCOM_DATABASE_TESTING_ACCESS_TOKEN']
|
|
end
|
|
|
|
def invoke!(post_comment: false, downstream_job_name: nil)
|
|
pipeline = super
|
|
gitlab = gitlab_client(:upstream)
|
|
project_path = base_variables['TOP_UPSTREAM_SOURCE_PROJECT']
|
|
merge_request_id = base_variables['TOP_UPSTREAM_MERGE_REQUEST_IID']
|
|
comment = "<!-- #{IDENTIFIABLE_NOTE_TAG} --> \nStarted database testing [pipeline](https://ops.gitlab.net/#{downstream_project_path}/-/pipelines/#{pipeline.id}) " \
|
|
"(limited access). This comment will be updated once the pipeline has finished running."
|
|
|
|
# Look for an existing note
|
|
db_testing_notes = gitlab.merge_request_notes(project_path, merge_request_id).auto_paginate.select do |note|
|
|
note.body.include?(IDENTIFIABLE_NOTE_TAG)
|
|
end
|
|
|
|
if db_testing_notes.empty?
|
|
# This is the first note
|
|
note = gitlab.create_merge_request_note(project_path, merge_request_id, comment)
|
|
|
|
puts "Posted comment to:\n"
|
|
puts "https://gitlab.com/#{project_path}/-/merge_requests/#{merge_request_id}#note_#{note.id}"
|
|
end
|
|
end
|
|
|
|
private
|
|
|
|
def gitlab_client(type)
|
|
@gitlab_clients ||= {
|
|
downstream: Gitlab.client(
|
|
endpoint: 'https://ops.gitlab.net/api/v4',
|
|
private_token: self.class.access_token
|
|
),
|
|
upstream: Gitlab.client(
|
|
endpoint: 'https://gitlab.com/api/v4',
|
|
private_token: Base.access_token
|
|
)
|
|
}
|
|
|
|
@gitlab_clients[type]
|
|
end
|
|
|
|
def trigger_token
|
|
ENV['GITLABCOM_DATABASE_TESTING_TRIGGER_TOKEN']
|
|
end
|
|
|
|
def downstream_project_path
|
|
ENV.fetch('GITLABCOM_DATABASE_TESTING_PROJECT_PATH', 'gitlab-com/database-team/gitlab-com-database-testing')
|
|
end
|
|
|
|
def extra_variables
|
|
{
|
|
# Use CI_MERGE_REQUEST_SOURCE_BRANCH_SHA for omnibus checkouts due to pipeline for merged results
|
|
# and fallback to CI_COMMIT_SHA for the `detached` pipelines.
|
|
'GITLAB_COMMIT_SHA' => Trigger.non_empty_variable_value('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA') || ENV['CI_COMMIT_SHA'],
|
|
'TRIGGERED_USER_LOGIN' => ENV['GITLAB_USER_LOGIN']
|
|
}
|
|
end
|
|
|
|
def ref
|
|
ENV['GITLABCOM_DATABASE_TESTING_TRIGGER_REF'] || 'master'
|
|
end
|
|
end
|
|
|
|
class CommitComment
|
|
def self.post!(downstream_pipeline, gitlab_client)
|
|
gitlab_client.create_commit_comment(
|
|
ENV['CI_PROJECT_PATH'],
|
|
Trigger.non_empty_variable_value('CI_MERGE_REQUEST_SOURCE_BRANCH_SHA') || ENV['CI_COMMIT_SHA'],
|
|
"The [`#{ENV['CI_JOB_NAME']}`](#{ENV['CI_JOB_URL']}) job from pipeline #{ENV['CI_PIPELINE_URL']} triggered #{downstream_pipeline.web_url} downstream.")
|
|
|
|
rescue Gitlab::Error::Error => error
|
|
puts "Ignoring the following error: #{error}"
|
|
end
|
|
end
|
|
|
|
class Pipeline
|
|
INTERVAL = 60 # seconds
|
|
MAX_DURATION = 3600 * 3 # 3 hours
|
|
|
|
attr_reader :id
|
|
|
|
def self.unscoped_class_name
|
|
name.split('::').last
|
|
end
|
|
|
|
def self.gitlab_api_method_name
|
|
unscoped_class_name.downcase
|
|
end
|
|
|
|
def initialize(project, id, gitlab_client)
|
|
@project = project
|
|
@id = id
|
|
@gitlab_client = gitlab_client
|
|
@start_time = Time.now.to_i
|
|
end
|
|
|
|
def wait!
|
|
(MAX_DURATION / INTERVAL).times do
|
|
case status
|
|
when :created, :pending, :running
|
|
print "."
|
|
sleep INTERVAL
|
|
when :success
|
|
puts "#{self.class.unscoped_class_name} succeeded in #{duration} minutes!"
|
|
return
|
|
else
|
|
raise "#{self.class.unscoped_class_name} did not succeed!"
|
|
end
|
|
|
|
$stdout.flush
|
|
end
|
|
|
|
raise "#{self.class.unscoped_class_name} timed out after waiting for #{duration} minutes!"
|
|
end
|
|
|
|
def duration
|
|
(Time.now.to_i - start_time) / 60
|
|
end
|
|
|
|
def status
|
|
gitlab_client.public_send(self.class.gitlab_api_method_name, project, id).status.to_sym # rubocop:disable GitlabSecurity/PublicSend
|
|
rescue Gitlab::Error::Error => error
|
|
puts "Ignoring the following error: #{error}"
|
|
# Ignore GitLab API hiccups. If GitLab is really down, we'll hit the job
|
|
# timeout anyway.
|
|
:running
|
|
end
|
|
|
|
private
|
|
|
|
attr_reader :project, :gitlab_client, :start_time
|
|
end
|
|
|
|
Job = Class.new(Pipeline)
|
|
end
|
|
|
|
case ARGV[0]
|
|
when 'omnibus'
|
|
Trigger::Omnibus.new.invoke!(post_comment: true, downstream_job_name: 'Trigger:qa-test').wait!
|
|
when 'cng'
|
|
Trigger::CNG.new.invoke!.wait!
|
|
when 'gitlab-com-database-testing'
|
|
Trigger::DatabaseTesting.new.invoke!
|
|
when 'docs'
|
|
docs_trigger = Trigger::Docs.new
|
|
|
|
case ARGV[1]
|
|
when 'deploy'
|
|
docs_trigger.deploy!
|
|
when 'cleanup'
|
|
docs_trigger.cleanup!
|
|
else
|
|
puts 'usage: trigger-build docs <deploy|cleanup>'
|
|
exit 1
|
|
end
|
|
else
|
|
puts "Please provide a valid option:
|
|
omnibus - Triggers a pipeline that builds the omnibus-gitlab package
|
|
cng - Triggers a pipeline that builds images used by the GitLab helm chart
|
|
gitlab-com-database-testing - Triggers a pipeline that tests database changes on GitLab.com data"
|
|
end
|