Refactor scripts/trigger-build and post a commit note with the downstream pipeline URL for omnibus triggers

Signed-off-by: Rémy Coutable <remy@rymai.me>
2018-09-04 22:56:20 +02:00 · 2018-09-04 22:56:20 +02:00 · 45d8c213fd
commit 45d8c213fd
parent bd30b0d511
3 changed files with 131 additions and 109 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -130,7 +130,6 @@ stages:

 .single-script-job: &single-script-job
  image: ruby:2.4-alpine
-  before_script: []
  stage: test
  cache: {}
  dependencies: []
@ -259,6 +258,7 @@ package-and-qa:
    SCRIPT_NAME: trigger-build
  retry: 0
  script:
+    - gem install gitlab --no-document
    - ./$SCRIPT_NAME omnibus
  when: manual
  only:
@ -285,7 +285,7 @@ review-docs-deploy-manual:
  <<: *review-docs
  stage: build
  script:
-    - gem install gitlab --no-ri --no-rdoc
+    - gem install gitlab --no-document
    - ./$SCRIPT_NAME deploy
  when: manual
  only:
@ -299,7 +299,7 @@ review-docs-deploy:
  <<: *review-docs
  stage: post-test
  script:
-    - gem install gitlab --no-ri --no-rdoc
+    - gem install gitlab --no-document
    - ./$SCRIPT_NAME deploy
  only:
    - /(^docs[\/-].*|.*-docs$)/@gitlab-org/gitlab-ce
@ -314,7 +314,7 @@ review-docs-cleanup:
    name: review-docs/$CI_COMMIT_REF_SLUG
    action: stop
  script:
-    - gem install gitlab --no-ri --no-rdoc
+    - gem install gitlab --no-document
    - ./$SCRIPT_NAME cleanup
  when: manual
  only:
@ -333,8 +333,8 @@ cloud-native-image:
    GIT_DEPTH: "1"
  cache: {}
  script:
-    - gem install gitlab --no-ri --no-rdoc
-    - BUILD_TRIGGER_TOKEN=$CI_JOB_TOKEN scripts/trigger-build cng
+    - gem install gitlab --no-document
+    - CNG_PROJECT_PATH="gitlab-org/build/CNG" BUILD_TRIGGER_TOKEN=$CI_JOB_TOKEN ./scripts/trigger-build cng
  only:
    - tags@gitlab-org/gitlab-ce
    - tags@gitlab-org/gitlab-ee
@ -366,7 +366,7 @@ update-tests-metadata:
      - rspec_flaky/
    policy: push
  script:
-    - retry gem install fog-aws mime-types activesupport --no-ri --no-rdoc
+    - retry gem install fog-aws mime-types activesupport --no-document
    - scripts/merge-reports ${KNAPSACK_RSPEC_SUITE_REPORT_PATH} knapsack/${CI_PROJECT_NAME}/rspec-pg_node_*.json
    - scripts/merge-reports ${FLAKY_RSPEC_SUITE_REPORT_PATH} rspec_flaky/all_*_*.json
    - FLAKY_RSPEC_GENERATE_REPORT=1 scripts/prune-old-flaky-specs ${FLAKY_RSPEC_SUITE_REPORT_PATH}
--- a/scripts/trigger-build
+++ b/scripts/trigger-build
@ -1,132 +1,160 @@
 #!/usr/bin/env ruby

-require 'net/http'
-require 'json'
-require 'cgi'
+require 'gitlab'
+
+#
+# Configure credentials to be used with gitlab gem
+#
+Gitlab.configure do |config|
+  config.endpoint      = 'https://gitlab.com/api/v4'
+  config.private_token = ENV['GITLAB_QA_ACCESS_TOKEN'] # gitlab-qa bot access token
+end

 module Trigger
-  OMNIBUS_PROJECT_PATH = 'gitlab-org/omnibus-gitlab'.freeze
-  CNG_PROJECT_PATH = 'gitlab-org/build/CNG'.freeze
  TOKEN = ENV['BUILD_TRIGGER_TOKEN']

  def self.ee?
    ENV['CI_PROJECT_NAME'] == 'gitlab-ee' || File.exist?('CHANGELOG-EE.md')
  end

-  class Omnibus
-    def initialize
-      @uri = URI("https://gitlab.com/api/v4/projects/#{CGI.escape(Trigger::OMNIBUS_PROJECT_PATH)}/trigger/pipeline")
-      @params = env_params.merge(file_params).merge(token: Trigger::TOKEN)
+  class Base
+    def initialize(api_token)
+      Gitlab.private_token = api_token
    end

-    def invoke!
-      res = Net::HTTP.post_form(@uri, @params)
-      id = JSON.parse(res.body)['id']
-      project = Trigger::OMNIBUS_PROJECT_PATH
+    def invoke!(post_comment: false)
+      pipeline = Gitlab.run_trigger(
+        downstream_project_path,
+        Trigger::TOKEN,
+        ref,
+        variables)

-      if id
-        puts "Triggered https://gitlab.com/#{project}/pipelines/#{id}"
-        puts "Waiting for downstream pipeline status"
-      else
-        raise "Trigger failed! The response from the trigger is: #{res.body}"
+      puts "Triggered #{pipeline.web_url}"
+      puts "Waiting for downstream pipeline status"
+
+      begin
+        Trigger::CommitComment.post!(downstream_project_path, pipeline) if post_comment
+      rescue Gitlab::Error::Error => error
+        puts "Ignoring the following error: #{error}"
      end
-
-      Trigger::Pipeline.new(project, id)
+      Trigger::Pipeline.new(downstream_project_path, pipeline.id)
    end

    private

-    def env_params
+    # Must be overriden
+    def downstream_project_path
+      raise NotImplementedError
+    end
+
+    # Must be overriden
+    def ref
+      raise NotImplementedError
+    end
+
+    # Can be overriden
+    def extra_variables
+      {}
+    end
+
+    # Can be overriden
+    def version_param_value(version_file)
+      File.read(version_file).strip
+    end
+
+    def variables
+      base_variables.merge(extra_variables).merge(version_file_variables)
+    end
+
+    def base_variables
      {
-        "ref" => ENV["OMNIBUS_BRANCH"] || "master",
-        "variables[GITLAB_VERSION]" => ENV["CI_COMMIT_SHA"],
-        "variables[ALTERNATIVE_SOURCES]" => true,
-        "variables[ee]" => Trigger.ee? ? 'true' : 'false',
-        "variables[TRIGGERED_USER]" => ENV["GITLAB_USER_NAME"],
-        "variables[TRIGGER_SOURCE]" => "https://gitlab.com/gitlab-org/#{ENV['CI_PROJECT_NAME']}/-/jobs/#{ENV['CI_JOB_ID']}"
+        'TRIGGERED_USER' => ENV['GITLAB_USER_NAME'],
+        'TRIGGER_SOURCE' => ENV['CI_JOB_URL']
      }
    end

-    def file_params
-      Hash.new.tap do |params|
-        Dir.glob("*_VERSION").each do |version_file|
-          params["variables[#{version_file}]"] = File.read(version_file).strip
-        end
+    # Read version files from all components
+    def version_file_variables
+      Dir.glob("*_VERSION").each_with_object({}) do |version_file, params|
+        params[version_file] = version_param_value(version_file)
      end
    end
  end

-  class CNG
-    def initialize
-      @uri = URI("https://gitlab.com/api/v4/projects/#{CGI.escape(Trigger::CNG_PROJECT_PATH)}/trigger/pipeline")
-      @ref_name = ENV['CI_COMMIT_REF_NAME']
-      @username = ENV['GITLAB_USER_NAME']
-      @project_name = ENV['CI_PROJECT_NAME']
-      @job_id = ENV['CI_JOB_ID']
-      @params = env_params.merge(file_params).merge(token: Trigger::TOKEN)
-    end
-
-    #
-    # Trigger a pipeline
-    #
-    def invoke!
-      res = Net::HTTP.post_form(@uri, @params)
-      id = JSON.parse(res.body)['id']
-      project = Trigger::CNG_PROJECT_PATH
-
-      if id
-        puts "Triggered https://gitlab.com/#{project}/pipelines/#{id}"
-        puts "Waiting for downstream pipeline status"
-      else
-        raise "Trigger failed! The response from the trigger is: #{res.body}"
-      end
-
-      Trigger::Pipeline.new(project, id)
-    end
-
+  class Omnibus < Base
    private

-    def env_params
-      params = {
-        "ref" => ENV["CNG_BRANCH"] || "master",
-        "variables[TRIGGERED_USER]" => @username,
-        "variables[TRIGGER_SOURCE]" => "https://gitlab.com/gitlab-org/#{@project_name}/-/jobs/#{@job_id}"
-      }
-
-      if Trigger.ee?
-        params["variables[GITLAB_EE_VERSION]"] = @ref_name
-        params["variables[EE_PIPELINE]"] = 'true'
-      else
-        params["variables[GITLAB_CE_VERSION]"] = @ref_name
-        params["variables[CE_PIPELINE]"] = 'true'
-      end
-
-      params
+    def downstream_project_path
+      'gitlab-org/omnibus-gitlab'.freeze
    end

-    # Read version files from all components
-    def file_params
-      Dir.glob("*_VERSION").each_with_object({}) do |version_file, params|
-        raw_version = File.read(version_file).strip
-        # if the version matches semver format, treat it as a tag and prepend `v`
-        version = if raw_version =~ Regexp.compile(/^\d+\.\d+\.\d+(-rc\d+)?(-ee)?$/)
-                    "v#{raw_version}"
-                  else
-                    raw_version
-                  end
+    def ref
+      ENV['OMNIBUS_BRANCH'] || 'master'
+    end

-        params["variables[#{version_file}]"] = version
+    def extra_variables
+      {
+        'GITLAB_VERSION' => ENV['CI_COMMIT_SHA'],
+        'ALTERNATIVE_SOURCES' => 'true',
+        'ee' => Trigger.ee? ? 'true' : 'false'
+      }
+    end
+  end
+
+  class CNG < Base
+    private
+
+    def downstream_project_path
+      ENV['CNG_PROJECT_PATH'] || 'gitlab-org/build/CNG-mirror'
+    end
+
+    def ref
+      ENV['CNG_BRANCH'] || 'master'
+    end
+
+    def extra_variables
+      edition = Trigger.ee? ? 'EE' : 'CE'
+
+      {
+        "GITLAB_#{edition}_VERSION" => ENV['CI_COMMIT_REF_NAME'],
+        "#{edition}_PIPELINE" => 'true'
+      }
+    end
+
+    def version_param_value(_version_file)
+      raw_version = super
+
+      # if the version matches semver format, treat it as a tag and prepend `v`
+      if raw_version =~ Regexp.compile(/^\d+\.\d+\.\d+(-rc\d+)?(-ee)?$/)
+        "v#{raw_version}"
+      else
+        raw_version
      end
    end
  end

+  class CommitComment
+    def self.post!(downstream_project_path, downstream_pipeline)
+      Gitlab.create_commit_comment(
+        ENV['CI_PROJECT_PATH'],
+        ENV['CI_COMMIT_SHA'],
+        "The [`#{ENV['CI_JOB_NAME']}`](#{ENV['CI_JOB_URL']}) job from pipeline #{ENV['CI_PIPELINE_URL']} triggered #{downstream_pipeline.web_url} downstream.")
+    end
+  end
+
  class Pipeline
    INTERVAL = 60 # seconds
    MAX_DURATION = 3600 * 3 # 3 hours

+    attr_reader :project, :id
+
    def initialize(project, id)
+      @project = project
+      @id = id
      @start = Time.now.to_i
-      @uri = URI("https://gitlab.com/api/v4/projects/#{CGI.escape(project)}/pipelines/#{id}")
+
+      # gitlab-bot's token "GitLab multi-project pipeline polling"
+      Gitlab.private_token = ENV['GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN']
    end

    def wait!
@ -157,15 +185,9 @@ module Trigger
    end

    def status
-      req = Net::HTTP::Get.new(@uri)
-      req['PRIVATE-TOKEN'] = ENV['GITLAB_QA_ACCESS_TOKEN']
-
-      res = Net::HTTP.start(@uri.hostname, @uri.port, use_ssl: true) do |http|
-        http.request(req)
-      end
-
-      JSON.parse(res.body)['status'].to_s.to_sym
-    rescue JSON::ParserError
+      Gitlab.pipeline(project, id).status.to_sym
+    rescue Gitlab::Error::Error => error
+      puts "Ignoring the following error: #{error}"
      # Ignore GitLab API hiccups. If GitLab is really down, we'll hit the job
      # timeout anyway.
      :running
@ -175,9 +197,9 @@ end

 case ARGV[0]
 when 'omnibus'
-  Trigger::Omnibus.new.invoke!.wait!
+  Trigger::Omnibus.new(ENV['GITLAB_QA_ACCESS_TOKEN']).invoke!(post_comment: true).wait!
 when 'cng'
-  Trigger::CNG.new.invoke!.wait!
+  Trigger::CNG.new(ENV['GITLAB_QA_ACCESS_TOKEN']).invoke!.wait!
 else
  puts "Please provide a valid option:
  omnibus - Triggers a pipeline that builds the omnibus-gitlab package
--- a/scripts/trigger-build-docs
+++ b/scripts/trigger-build-docs
@ -6,8 +6,8 @@ require 'gitlab'
 # Configure credentials to be used with gitlab gem
 #
 Gitlab.configure do |config|
-  config.endpoint       = 'https://gitlab.com/api/v4'
-  config.private_token  = ENV["DOCS_API_TOKEN"] # GitLab Docs bot access token with Developer access to gitlab-docs
+  config.endpoint      = 'https://gitlab.com/api/v4'
+  config.private_token = ENV["DOCS_API_TOKEN"] # GitLab Docs bot access token with Developer access to gitlab-docs
 end

 #
@ -99,7 +99,7 @@ def trigger_pipeline

  puts "=> Follow the status of the triggered pipeline:"
  puts ""
-  puts "https://gitlab.com/gitlab-com/gitlab-docs/pipelines/#{pipeline.id}"
+  puts pipeline.web_url
  puts ""
  puts "=> In a few minutes, you will be able to preview your changes under the following URL:"
  puts ""