From 45d8c213fdff6f6040e41f52060c797e1fc57462 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20Coutable?= Date: Tue, 4 Sep 2018 22:56:20 +0200 Subject: [PATCH] Refactor scripts/trigger-build and post a commit note with the downstream pipeline URL for omnibus triggers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Rémy Coutable --- .gitlab-ci.yml | 14 +-- scripts/trigger-build | 220 ++++++++++++++++++++----------------- scripts/trigger-build-docs | 6 +- 3 files changed, 131 insertions(+), 109 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 488d2f261e7..90bc9f74ef3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -130,7 +130,6 @@ stages: .single-script-job: &single-script-job image: ruby:2.4-alpine - before_script: [] stage: test cache: {} dependencies: [] @@ -259,6 +258,7 @@ package-and-qa: SCRIPT_NAME: trigger-build retry: 0 script: + - gem install gitlab --no-document - ./$SCRIPT_NAME omnibus when: manual only: @@ -285,7 +285,7 @@ review-docs-deploy-manual: <<: *review-docs stage: build script: - - gem install gitlab --no-ri --no-rdoc + - gem install gitlab --no-document - ./$SCRIPT_NAME deploy when: manual only: @@ -299,7 +299,7 @@ review-docs-deploy: <<: *review-docs stage: post-test script: - - gem install gitlab --no-ri --no-rdoc + - gem install gitlab --no-document - ./$SCRIPT_NAME deploy only: - /(^docs[\/-].*|.*-docs$)/@gitlab-org/gitlab-ce @@ -314,7 +314,7 @@ review-docs-cleanup: name: review-docs/$CI_COMMIT_REF_SLUG action: stop script: - - gem install gitlab --no-ri --no-rdoc + - gem install gitlab --no-document - ./$SCRIPT_NAME cleanup when: manual only: @@ -333,8 +333,8 @@ cloud-native-image: GIT_DEPTH: "1" cache: {} script: - - gem install gitlab --no-ri --no-rdoc - - BUILD_TRIGGER_TOKEN=$CI_JOB_TOKEN scripts/trigger-build cng + - gem install gitlab --no-document + - CNG_PROJECT_PATH="gitlab-org/build/CNG" BUILD_TRIGGER_TOKEN=$CI_JOB_TOKEN ./scripts/trigger-build cng only: - tags@gitlab-org/gitlab-ce - tags@gitlab-org/gitlab-ee @@ -366,7 +366,7 @@ update-tests-metadata: - rspec_flaky/ policy: push script: - - retry gem install fog-aws mime-types activesupport --no-ri --no-rdoc + - retry gem install fog-aws mime-types activesupport --no-document - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec-pg_node_*.json - scripts/merge-reports ${FLAKY_RSPEC_SUITE_REPORT_PATH} rspec_flaky/all_*_*.json - FLAKY_RSPEC_GENERATE_REPORT=1 scripts/prune-old-flaky-specs ${FLAKY_RSPEC_SUITE_REPORT_PATH} diff --git a/scripts/trigger-build b/scripts/trigger-build index 798bf1e82b7..0b5fd5995dd 100755 --- a/scripts/trigger-build +++ b/scripts/trigger-build @@ -1,132 +1,160 @@ #!/usr/bin/env ruby -require 'net/http' -require 'json' -require 'cgi' +require 'gitlab' + +# +# Configure credentials to be used with gitlab gem +# +Gitlab.configure do |config| + config.endpoint = 'https://gitlab.com/api/v4' + config.private_token = ENV['GITLAB_QA_ACCESS_TOKEN'] # gitlab-qa bot access token +end module Trigger - OMNIBUS_PROJECT_PATH = 'gitlab-org/omnibus-gitlab'.freeze - CNG_PROJECT_PATH = 'gitlab-org/build/CNG'.freeze TOKEN = ENV['BUILD_TRIGGER_TOKEN'] def self.ee? ENV['CI_PROJECT_NAME'] == 'gitlab-ee' || File.exist?('CHANGELOG-EE.md') end - class Omnibus - def initialize - @uri = URI("https://gitlab.com/api/v4/projects/#{CGI.escape(Trigger::OMNIBUS_PROJECT_PATH)}/trigger/pipeline") - @params = env_params.merge(file_params).merge(token: Trigger::TOKEN) + class Base + def initialize(api_token) + Gitlab.private_token = api_token end - def invoke! - res = Net::HTTP.post_form(@uri, @params) - id = JSON.parse(res.body)['id'] - project = Trigger::OMNIBUS_PROJECT_PATH + def invoke!(post_comment: false) + pipeline = Gitlab.run_trigger( + downstream_project_path, + Trigger::TOKEN, + ref, + variables) - if id - puts "Triggered https://gitlab.com/#{project}/pipelines/#{id}" - puts "Waiting for downstream pipeline status" - else - raise "Trigger failed! The response from the trigger is: #{res.body}" + puts "Triggered #{pipeline.web_url}" + puts "Waiting for downstream pipeline status" + + begin + Trigger::CommitComment.post!(downstream_project_path, pipeline) if post_comment + rescue Gitlab::Error::Error => error + puts "Ignoring the following error: #{error}" end - - Trigger::Pipeline.new(project, id) + Trigger::Pipeline.new(downstream_project_path, pipeline.id) end private - def env_params + # Must be overriden + def downstream_project_path + raise NotImplementedError + end + + # Must be overriden + def ref + raise NotImplementedError + end + + # Can be overriden + def extra_variables + {} + end + + # Can be overriden + def version_param_value(version_file) + File.read(version_file).strip + end + + def variables + base_variables.merge(extra_variables).merge(version_file_variables) + end + + def base_variables { - "ref" => ENV["OMNIBUS_BRANCH"] || "master", - "variables[GITLAB_VERSION]" => ENV["CI_COMMIT_SHA"], - "variables[ALTERNATIVE_SOURCES]" => true, - "variables[ee]" => Trigger.ee? ? 'true' : 'false', - "variables[TRIGGERED_USER]" => ENV["GITLAB_USER_NAME"], - "variables[TRIGGER_SOURCE]" => "https://gitlab.com/gitlab-org/#{ENV['CI_PROJECT_NAME']}/-/jobs/#{ENV['CI_JOB_ID']}" + 'TRIGGERED_USER' => ENV['GITLAB_USER_NAME'], + 'TRIGGER_SOURCE' => ENV['CI_JOB_URL'] } end - def file_params - Hash.new.tap do |params| - Dir.glob("*_VERSION").each do |version_file| - params["variables[#{version_file}]"] = File.read(version_file).strip - end + # Read version files from all components + def version_file_variables + Dir.glob("*_VERSION").each_with_object({}) do |version_file, params| + params[version_file] = version_param_value(version_file) end end end - class CNG - def initialize - @uri = URI("https://gitlab.com/api/v4/projects/#{CGI.escape(Trigger::CNG_PROJECT_PATH)}/trigger/pipeline") - @ref_name = ENV['CI_COMMIT_REF_NAME'] - @username = ENV['GITLAB_USER_NAME'] - @project_name = ENV['CI_PROJECT_NAME'] - @job_id = ENV['CI_JOB_ID'] - @params = env_params.merge(file_params).merge(token: Trigger::TOKEN) - end - - # - # Trigger a pipeline - # - def invoke! - res = Net::HTTP.post_form(@uri, @params) - id = JSON.parse(res.body)['id'] - project = Trigger::CNG_PROJECT_PATH - - if id - puts "Triggered https://gitlab.com/#{project}/pipelines/#{id}" - puts "Waiting for downstream pipeline status" - else - raise "Trigger failed! The response from the trigger is: #{res.body}" - end - - Trigger::Pipeline.new(project, id) - end - + class Omnibus < Base private - def env_params - params = { - "ref" => ENV["CNG_BRANCH"] || "master", - "variables[TRIGGERED_USER]" => @username, - "variables[TRIGGER_SOURCE]" => "https://gitlab.com/gitlab-org/#{@project_name}/-/jobs/#{@job_id}" - } - - if Trigger.ee? - params["variables[GITLAB_EE_VERSION]"] = @ref_name - params["variables[EE_PIPELINE]"] = 'true' - else - params["variables[GITLAB_CE_VERSION]"] = @ref_name - params["variables[CE_PIPELINE]"] = 'true' - end - - params + def downstream_project_path + 'gitlab-org/omnibus-gitlab'.freeze end - # Read version files from all components - def file_params - Dir.glob("*_VERSION").each_with_object({}) do |version_file, params| - raw_version = File.read(version_file).strip - # if the version matches semver format, treat it as a tag and prepend `v` - version = if raw_version =~ Regexp.compile(/^\d+\.\d+\.\d+(-rc\d+)?(-ee)?$/) - "v#{raw_version}" - else - raw_version - end + def ref + ENV['OMNIBUS_BRANCH'] || 'master' + end - params["variables[#{version_file}]"] = version + def extra_variables + { + 'GITLAB_VERSION' => ENV['CI_COMMIT_SHA'], + 'ALTERNATIVE_SOURCES' => 'true', + 'ee' => Trigger.ee? ? 'true' : 'false' + } + end + end + + class CNG < Base + private + + def downstream_project_path + ENV['CNG_PROJECT_PATH'] || 'gitlab-org/build/CNG-mirror' + end + + def ref + ENV['CNG_BRANCH'] || 'master' + end + + def extra_variables + edition = Trigger.ee? ? 'EE' : 'CE' + + { + "GITLAB_#{edition}_VERSION" => ENV['CI_COMMIT_REF_NAME'], + "#{edition}_PIPELINE" => 'true' + } + end + + def version_param_value(_version_file) + raw_version = super + + # if the version matches semver format, treat it as a tag and prepend `v` + if raw_version =~ Regexp.compile(/^\d+\.\d+\.\d+(-rc\d+)?(-ee)?$/) + "v#{raw_version}" + else + raw_version end end end + class CommitComment + def self.post!(downstream_project_path, downstream_pipeline) + Gitlab.create_commit_comment( + ENV['CI_PROJECT_PATH'], + ENV['CI_COMMIT_SHA'], + "The [`#{ENV['CI_JOB_NAME']}`](#{ENV['CI_JOB_URL']}) job from pipeline #{ENV['CI_PIPELINE_URL']} triggered #{downstream_pipeline.web_url} downstream.") + end + end + class Pipeline INTERVAL = 60 # seconds MAX_DURATION = 3600 * 3 # 3 hours + attr_reader :project, :id + def initialize(project, id) + @project = project + @id = id @start = Time.now.to_i - @uri = URI("https://gitlab.com/api/v4/projects/#{CGI.escape(project)}/pipelines/#{id}") + + # gitlab-bot's token "GitLab multi-project pipeline polling" + Gitlab.private_token = ENV['GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN'] end def wait! @@ -157,15 +185,9 @@ module Trigger end def status - req = Net::HTTP::Get.new(@uri) - req['PRIVATE-TOKEN'] = ENV['GITLAB_QA_ACCESS_TOKEN'] - - res = Net::HTTP.start(@uri.hostname, @uri.port, use_ssl: true) do |http| - http.request(req) - end - - JSON.parse(res.body)['status'].to_s.to_sym - rescue JSON::ParserError + Gitlab.pipeline(project, id).status.to_sym + rescue Gitlab::Error::Error => error + puts "Ignoring the following error: #{error}" # Ignore GitLab API hiccups. If GitLab is really down, we'll hit the job # timeout anyway. :running @@ -175,9 +197,9 @@ end case ARGV[0] when 'omnibus' - Trigger::Omnibus.new.invoke!.wait! + Trigger::Omnibus.new(ENV['GITLAB_QA_ACCESS_TOKEN']).invoke!(post_comment: true).wait! when 'cng' - Trigger::CNG.new.invoke!.wait! + Trigger::CNG.new(ENV['GITLAB_QA_ACCESS_TOKEN']).invoke!.wait! else puts "Please provide a valid option: omnibus - Triggers a pipeline that builds the omnibus-gitlab package diff --git a/scripts/trigger-build-docs b/scripts/trigger-build-docs index 9ee35684509..dfc8ee6050a 100755 --- a/scripts/trigger-build-docs +++ b/scripts/trigger-build-docs @@ -6,8 +6,8 @@ require 'gitlab' # Configure credentials to be used with gitlab gem # Gitlab.configure do |config| - config.endpoint = 'https://gitlab.com/api/v4' - config.private_token = ENV["DOCS_API_TOKEN"] # GitLab Docs bot access token with Developer access to gitlab-docs + config.endpoint = 'https://gitlab.com/api/v4' + config.private_token = ENV["DOCS_API_TOKEN"] # GitLab Docs bot access token with Developer access to gitlab-docs end # @@ -99,7 +99,7 @@ def trigger_pipeline puts "=> Follow the status of the triggered pipeline:" puts "" - puts "https://gitlab.com/gitlab-com/gitlab-docs/pipelines/#{pipeline.id}" + puts pipeline.web_url puts "" puts "=> In a few minutes, you will be able to preview your changes under the following URL:" puts ""