From d7a4c1d2a5f395ca6ddb2dc2608ac33ccebf8a74 Mon Sep 17 00:00:00 2001 From: Yorick Peterse Date: Mon, 22 Jul 2019 17:11:50 +0200 Subject: [PATCH] Backport EE changes made to the scripts/ directory This backport changes made by EE to the files in the scripts/ directory. This comes with a few changes to some scripts to make them work in the single codebase setup. --- lib/gitlab.rb | 4 +- lib/tasks/gitlab/db.rake | 8 +++ scripts/lint-rugged | 3 + scripts/static-analysis | 5 ++ scripts/utils.sh | 128 +++++++++++++++++++++++++++++++++++++++ spec/lib/gitlab_spec.rb | 6 ++ 6 files changed, 152 insertions(+), 2 deletions(-) diff --git a/lib/gitlab.rb b/lib/gitlab.rb index c62d1071dba..d9d8dcf7900 100644 --- a/lib/gitlab.rb +++ b/lib/gitlab.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -require_dependency File.expand_path('gitlab/popen', __dir__) +require 'pathname' module Gitlab def self.root @@ -61,7 +61,7 @@ module Gitlab def self.ee? @is_ee ||= - if ENV['IS_GITLAB_EE'].present? + if ENV['IS_GITLAB_EE'] && !ENV['IS_GITLAB_EE'].empty? Gitlab::Utils.to_boolean(ENV['IS_GITLAB_EE']) else # We may use this method when the Rails environment is not loaded. This diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index d054959e05e..1961f64659c 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -70,5 +70,13 @@ namespace :gitlab do Gitlab::DowntimeCheck.new.check_and_print(migrations) end + + desc 'Sets up EE specific database functionality' + + if Gitlab.ee? + task setup_ee: %w[geo:db:drop geo:db:create geo:db:schema:load geo:db:migrate] + else + task :setup_ee + end end end diff --git a/scripts/lint-rugged b/scripts/lint-rugged index f40f0489c1a..d862571c1c5 100755 --- a/scripts/lint-rugged +++ b/scripts/lint-rugged @@ -1,6 +1,9 @@ #!/usr/bin/env ruby ALLOWED = [ + # https://gitlab.com/gitlab-org/gitaly/issues/760 + 'lib/elasticsearch/git/repository.rb', + # Needed to handle repositories that are not in any storage 'lib/gitlab/bare_repository_import/repository.rb', diff --git a/scripts/static-analysis b/scripts/static-analysis index 642c50ec0a8..6fd64fbf9da 100755 --- a/scripts/static-analysis +++ b/scripts/static-analysis @@ -1,6 +1,7 @@ #!/usr/bin/env ruby # We don't have auto-loading here +require_relative '../lib/gitlab' require_relative '../lib/gitlab/popen' require_relative '../lib/gitlab/popen/runner' @@ -36,6 +37,10 @@ tasks = [ %w[scripts/lint-rugged] ] +if Gitlab.ee? + tasks.unshift(%w[ruby -rbundler/setup scripts/ee_specific_check/ee_specific_check.rb]) +end + static_analysis = Gitlab::Popen::Runner.new static_analysis.run(tasks) do |cmd, &run| diff --git a/scripts/utils.sh b/scripts/utils.sh index 4a6567b8a62..c1bdef5b847 100644 --- a/scripts/utils.sh +++ b/scripts/utils.sh @@ -29,6 +29,134 @@ function setup_db() { if [ "$GITLAB_DATABASE" = "mysql" ]; then bundle exec rake add_limits_mysql fi + + bundle exec rake gitlab:db:setup_ee +} + +function install_api_client_dependencies_with_apk() { + apk add --update openssl curl jq +} + +function install_api_client_dependencies_with_apt() { + apt update && apt install jq -y +} + +function install_gitlab_gem() { + gem install gitlab --no-document +} + +function echoerr() { + local header="${2}" + + if [ -n "${header}" ]; then + printf "\n\033[0;31m** %s **\n\033[0m" "${1}" >&2; + else + printf "\033[0;31m%s\n\033[0m" "${1}" >&2; + fi +} + +function echoinfo() { + local header="${2}" + + if [ -n "${header}" ]; then + printf "\n\033[0;33m** %s **\n\033[0m" "${1}" >&2; + else + printf "\033[0;33m%s\n\033[0m" "${1}" >&2; + fi +} + +function get_job_id() { + local job_name="${1}" + local query_string="${2:+&${2}}" + local api_token="${API_TOKEN-${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}}" + if [ -z "${api_token}" ]; then + echoerr "Please provide an API token with \$API_TOKEN or \$GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN." + return + fi + + local max_page=3 + local page=1 + + while true; do + local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/pipelines/${CI_PIPELINE_ID}/jobs?per_page=100&page=${page}${query_string}" + echoinfo "GET ${url}" + + local job_id + job_id=$(curl --silent --show-error --header "PRIVATE-TOKEN: ${api_token}" "${url}" | jq "map(select(.name == \"${job_name}\")) | map(.id) | last") + [[ "${job_id}" == "null" && "${page}" -lt "$max_page" ]] || break + + let "page++" + done + + if [[ "${job_id}" == "" ]]; then + echoerr "The '${job_name}' job ID couldn't be retrieved!" + else + echoinfo "The '${job_name}' job ID is ${job_id}" + echo "${job_id}" + fi +} + +function play_job() { + local job_name="${1}" + local job_id + job_id=$(get_job_id "${job_name}" "scope=manual"); + if [ -z "${job_id}" ]; then return; fi + + local api_token="${API_TOKEN-${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}}" + if [ -z "${api_token}" ]; then + echoerr "Please provide an API token with \$API_TOKEN or \$GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN." + return + fi + + local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/jobs/${job_id}/play" + echoinfo "POST ${url}" + + local job_url + job_url=$(curl --silent --show-error --request POST --header "PRIVATE-TOKEN: ${api_token}" "${url}" | jq ".web_url") + echoinfo "Manual job '${job_name}' started at: ${job_url}" +} + +function wait_for_job_to_be_done() { + local job_name="${1}" + local query_string="${2}" + local job_id + job_id=$(get_job_id "${job_name}" "${query_string}") + if [ -z "${job_id}" ]; then return; fi + + local api_token="${API_TOKEN-${GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN}}" + if [ -z "${api_token}" ]; then + echoerr "Please provide an API token with \$API_TOKEN or \$GITLAB_BOT_MULTI_PROJECT_PIPELINE_POLLING_TOKEN." + return + fi + + echoinfo "Waiting for the '${job_name}' job to finish..." + + local url="https://gitlab.com/api/v4/projects/${CI_PROJECT_ID}/jobs/${job_id}" + echoinfo "GET ${url}" + + # In case the job hasn't finished yet. Keep trying until the job times out. + local interval=30 + local elapsed_seconds=0 + while true; do + local job_status + job_status=$(curl --silent --show-error --header "PRIVATE-TOKEN: ${api_token}" "${url}" | jq ".status" | sed -e s/\"//g) + [[ "${job_status}" == "pending" || "${job_status}" == "running" ]] || break + + printf "." + let "elapsed_seconds+=interval" + sleep ${interval} + done + + local elapsed_minutes=$((elapsed_seconds / 60)) + echoinfo "Waited '${job_name}' for ${elapsed_minutes} minutes." + + if [[ "${job_status}" == "failed" ]]; then + echoerr "The '${job_name}' failed." + elif [[ "${job_status}" == "manual" ]]; then + echoinfo "The '${job_name}' is manual." + else + echoinfo "The '${job_name}' passed." + fi } function install_api_client_dependencies_with_apk() { diff --git a/spec/lib/gitlab_spec.rb b/spec/lib/gitlab_spec.rb index 82b0e819063..c293f58c9cb 100644 --- a/spec/lib/gitlab_spec.rb +++ b/spec/lib/gitlab_spec.rb @@ -136,6 +136,12 @@ describe Gitlab do expect(described_class.ee?).to eq(false) end + + it 'returns true when the IS_GITLAB_EE variable is not empty' do + stub_env('IS_GITLAB_EE', '1') + + expect(described_class.ee?).to eq(true) + end end describe '.http_proxy_env?' do