2018-09-25 10:32:53 -04:00
|
|
|
[[ "$TRACE" ]] && set -x
|
|
|
|
export TILLER_NAMESPACE="$KUBE_NAMESPACE"
|
|
|
|
|
2019-05-23 10:37:29 -04:00
|
|
|
function deploy_exists() {
|
2019-03-27 05:36:01 -04:00
|
|
|
local namespace="${1}"
|
|
|
|
local deploy="${2}"
|
|
|
|
echoinfo "Checking if ${deploy} exists in the ${namespace} namespace..." true
|
|
|
|
|
|
|
|
helm status --tiller-namespace "${namespace}" "${deploy}" >/dev/null 2>&1
|
|
|
|
local deploy_exists=$?
|
|
|
|
|
|
|
|
echoinfo "Deployment status for ${deploy} is ${deploy_exists}"
|
|
|
|
return $deploy_exists
|
|
|
|
}
|
|
|
|
|
2019-05-23 10:37:29 -04:00
|
|
|
function previous_deploy_failed() {
|
2019-03-27 05:36:01 -04:00
|
|
|
local deploy="${1}"
|
|
|
|
echoinfo "Checking for previous deployment of ${deploy}" true
|
|
|
|
|
2019-04-09 07:12:49 -04:00
|
|
|
helm status "${deploy}" >/dev/null 2>&1
|
2019-03-27 05:36:01 -04:00
|
|
|
local status=$?
|
|
|
|
|
|
|
|
# if `status` is `0`, deployment exists, has a status
|
|
|
|
if [ $status -eq 0 ]; then
|
|
|
|
echoinfo "Previous deployment found, checking status..."
|
2019-04-09 07:12:49 -04:00
|
|
|
deployment_status=$(helm status "${deploy}" | grep ^STATUS | cut -d' ' -f2)
|
2019-03-27 05:36:01 -04:00
|
|
|
echoinfo "Previous deployment state: ${deployment_status}"
|
|
|
|
if [[ "$deployment_status" == "FAILED" || "$deployment_status" == "PENDING_UPGRADE" || "$deployment_status" == "PENDING_INSTALL" ]]; then
|
|
|
|
status=0;
|
|
|
|
else
|
|
|
|
status=1;
|
|
|
|
fi
|
|
|
|
else
|
|
|
|
echoerr "Previous deployment NOT found."
|
|
|
|
fi
|
|
|
|
return $status
|
|
|
|
}
|
|
|
|
|
2019-08-27 07:01:13 -04:00
|
|
|
function delete_release() {
|
2019-03-27 05:36:01 -04:00
|
|
|
if [ -z "$CI_ENVIRONMENT_SLUG" ]; then
|
|
|
|
echoerr "No release given, aborting the delete!"
|
|
|
|
return
|
|
|
|
fi
|
|
|
|
|
|
|
|
local name="$CI_ENVIRONMENT_SLUG"
|
|
|
|
|
|
|
|
echoinfo "Deleting release '$name'..." true
|
|
|
|
|
2019-04-18 10:05:21 -04:00
|
|
|
helm delete --purge "$name"
|
2019-03-27 05:36:01 -04:00
|
|
|
}
|
|
|
|
|
2019-09-09 15:29:44 -04:00
|
|
|
function delete_failed_release() {
|
|
|
|
if [ -z "$CI_ENVIRONMENT_SLUG" ]; then
|
|
|
|
echoerr "No release given, aborting the delete!"
|
|
|
|
return
|
|
|
|
fi
|
|
|
|
|
|
|
|
if ! deploy_exists "${KUBE_NAMESPACE}" "${CI_ENVIRONMENT_SLUG}"; then
|
|
|
|
echoinfo "No Review App with ${CI_ENVIRONMENT_SLUG} is currently deployed."
|
|
|
|
else
|
|
|
|
# Cleanup and previous installs, as FAILED and PENDING_UPGRADE will cause errors with `upgrade`
|
|
|
|
if previous_deploy_failed "$CI_ENVIRONMENT_SLUG" ; then
|
|
|
|
echoinfo "Review App deployment in bad state, cleaning up $CI_ENVIRONMENT_SLUG"
|
|
|
|
delete_release
|
|
|
|
else
|
|
|
|
echoinfo "Review App deployment in good state"
|
|
|
|
fi
|
|
|
|
fi
|
|
|
|
}
|
|
|
|
|
|
|
|
|
2019-03-27 05:36:01 -04:00
|
|
|
function get_pod() {
|
|
|
|
local app_name="${1}"
|
|
|
|
local status="${2-Running}"
|
2019-07-23 11:11:13 -04:00
|
|
|
get_pod_cmd="kubectl get pods -n ${KUBE_NAMESPACE} --field-selector=status.phase=${status} -lapp=${app_name},release=${CI_ENVIRONMENT_SLUG} --no-headers -o=custom-columns=NAME:.metadata.name | tail -n 1"
|
2019-05-23 10:37:29 -04:00
|
|
|
echoinfo "Waiting till '${app_name}' pod is ready" true
|
|
|
|
echoinfo "Running '${get_pod_cmd}'"
|
2019-03-27 05:36:01 -04:00
|
|
|
|
2019-05-23 10:37:29 -04:00
|
|
|
local interval=5
|
|
|
|
local elapsed_seconds=0
|
|
|
|
local max_seconds=$((2 * 60))
|
2019-03-27 05:36:01 -04:00
|
|
|
while true; do
|
2019-04-09 07:12:49 -04:00
|
|
|
local pod_name
|
|
|
|
pod_name="$(eval "${get_pod_cmd}")"
|
2019-03-27 05:36:01 -04:00
|
|
|
[[ "${pod_name}" == "" ]] || break
|
|
|
|
|
2019-05-23 10:37:29 -04:00
|
|
|
if [[ "${elapsed_seconds}" -gt "${max_seconds}" ]]; then
|
|
|
|
echoerr "The pod name couldn't be found after ${elapsed_seconds} seconds, aborting."
|
2019-06-20 04:59:31 -04:00
|
|
|
break
|
2019-05-23 10:37:29 -04:00
|
|
|
fi
|
|
|
|
|
|
|
|
let "elapsed_seconds+=interval"
|
|
|
|
sleep ${interval}
|
2019-03-27 05:36:01 -04:00
|
|
|
done
|
|
|
|
|
|
|
|
echoinfo "The pod name is '${pod_name}'."
|
|
|
|
echo "${pod_name}"
|
|
|
|
}
|
2018-11-06 17:59:25 -05:00
|
|
|
|
2018-09-25 10:32:53 -04:00
|
|
|
function check_kube_domain() {
|
2019-03-27 05:36:01 -04:00
|
|
|
echoinfo "Checking that Kube domain exists..." true
|
|
|
|
|
2018-09-25 10:32:53 -04:00
|
|
|
if [ -z ${REVIEW_APPS_DOMAIN+x} ]; then
|
|
|
|
echo "In order to deploy or use Review Apps, REVIEW_APPS_DOMAIN variable must be set"
|
|
|
|
echo "You can do it in Auto DevOps project settings or defining a variable at group or project level"
|
|
|
|
echo "You can also manually add it in .gitlab-ci.yml"
|
|
|
|
false
|
|
|
|
else
|
|
|
|
true
|
|
|
|
fi
|
|
|
|
}
|
|
|
|
|
|
|
|
function ensure_namespace() {
|
2019-03-27 05:36:01 -04:00
|
|
|
echoinfo "Ensuring the ${KUBE_NAMESPACE} namespace exists..." true
|
|
|
|
|
2018-09-25 10:32:53 -04:00
|
|
|
kubectl describe namespace "$KUBE_NAMESPACE" || kubectl create namespace "$KUBE_NAMESPACE"
|
|
|
|
}
|
|
|
|
|
|
|
|
function install_tiller() {
|
2019-03-27 05:36:01 -04:00
|
|
|
echoinfo "Checking deployment/tiller-deploy status in the ${TILLER_NAMESPACE} namespace..." true
|
|
|
|
|
|
|
|
echoinfo "Initiating the Helm client..."
|
|
|
|
helm init --client-only
|
|
|
|
|
2019-06-20 04:59:31 -04:00
|
|
|
# Set toleration for Tiller to be installed on a specific node pool
|
2018-12-13 10:45:44 -05:00
|
|
|
helm init \
|
2019-06-20 04:59:31 -04:00
|
|
|
--wait \
|
2018-12-13 10:45:44 -05:00
|
|
|
--upgrade \
|
2019-06-20 04:59:31 -04:00
|
|
|
--node-selectors "app=helm" \
|
|
|
|
--replicas 3 \
|
|
|
|
--override "spec.template.spec.tolerations[0].key"="dedicated" \
|
|
|
|
--override "spec.template.spec.tolerations[0].operator"="Equal" \
|
|
|
|
--override "spec.template.spec.tolerations[0].value"="helm" \
|
|
|
|
--override "spec.template.spec.tolerations[0].effect"="NoSchedule"
|
2019-03-27 05:36:01 -04:00
|
|
|
|
2018-09-25 10:32:53 -04:00
|
|
|
kubectl rollout status -n "$TILLER_NAMESPACE" -w "deployment/tiller-deploy"
|
2019-03-27 05:36:01 -04:00
|
|
|
|
2018-09-25 10:32:53 -04:00
|
|
|
if ! helm version --debug; then
|
|
|
|
echo "Failed to init Tiller."
|
|
|
|
return 1
|
|
|
|
fi
|
2019-03-27 05:36:01 -04:00
|
|
|
}
|
|
|
|
|
|
|
|
function install_external_dns() {
|
|
|
|
local release_name="dns-gitlab-review-app"
|
2019-04-09 07:12:49 -04:00
|
|
|
local domain
|
|
|
|
domain=$(echo "${REVIEW_APPS_DOMAIN}" | awk -F. '{printf "%s.%s", $(NF-1), $NF}')
|
2019-03-27 05:36:01 -04:00
|
|
|
echoinfo "Installing external DNS for domain ${domain}..." true
|
|
|
|
|
2019-05-23 10:37:29 -04:00
|
|
|
if ! deploy_exists "${KUBE_NAMESPACE}" "${release_name}" || previous_deploy_failed "${release_name}" ; then
|
2019-03-27 05:36:01 -04:00
|
|
|
echoinfo "Installing external-dns Helm chart"
|
|
|
|
helm repo update
|
2019-06-28 05:38:38 -04:00
|
|
|
# Default requested: CPU => 0, memory => 0
|
2019-07-10 12:35:42 -04:00
|
|
|
helm install stable/external-dns --version '^2.2.1' \
|
2019-03-27 05:36:01 -04:00
|
|
|
-n "${release_name}" \
|
|
|
|
--namespace "${KUBE_NAMESPACE}" \
|
|
|
|
--set provider="aws" \
|
2019-07-10 12:35:42 -04:00
|
|
|
--set aws.credentials.secretKey="${REVIEW_APPS_AWS_SECRET_KEY}" \
|
|
|
|
--set aws.credentials.accessKey="${REVIEW_APPS_AWS_ACCESS_KEY}" \
|
2019-03-27 05:36:01 -04:00
|
|
|
--set aws.zoneType="public" \
|
2019-07-10 12:35:42 -04:00
|
|
|
--set aws.batchChangeSize=400 \
|
2019-03-27 05:36:01 -04:00
|
|
|
--set domainFilters[0]="${domain}" \
|
|
|
|
--set txtOwnerId="${KUBE_NAMESPACE}" \
|
|
|
|
--set rbac.create="true" \
|
2019-06-28 05:38:38 -04:00
|
|
|
--set policy="sync" \
|
|
|
|
--set resources.requests.cpu=50m \
|
|
|
|
--set resources.limits.cpu=100m \
|
|
|
|
--set resources.requests.memory=100M \
|
|
|
|
--set resources.limits.memory=200M
|
2019-03-27 05:36:01 -04:00
|
|
|
else
|
|
|
|
echoinfo "The external-dns Helm chart is already successfully deployed."
|
|
|
|
fi
|
2018-09-25 10:32:53 -04:00
|
|
|
}
|
|
|
|
|
2019-05-23 10:37:29 -04:00
|
|
|
function create_application_secret() {
|
2019-03-27 05:36:01 -04:00
|
|
|
echoinfo "Creating the ${CI_ENVIRONMENT_SLUG}-gitlab-initial-root-password secret in the ${KUBE_NAMESPACE} namespace..." true
|
2018-09-25 10:32:53 -04:00
|
|
|
|
|
|
|
kubectl create secret generic -n "$KUBE_NAMESPACE" \
|
2019-04-09 07:12:49 -04:00
|
|
|
"${CI_ENVIRONMENT_SLUG}-gitlab-initial-root-password" \
|
|
|
|
--from-literal="password=${REVIEW_APPS_ROOT_PASSWORD}" \
|
2018-09-25 10:32:53 -04:00
|
|
|
--dry-run -o json | kubectl apply -f -
|
2019-11-04 13:06:28 -05:00
|
|
|
|
|
|
|
if [ -z "${REVIEW_APPS_EE_LICENSE}" ]; then echo "License not found" && return; fi
|
|
|
|
|
|
|
|
echoinfo "Creating the ${CI_ENVIRONMENT_SLUG}-gitlab-license secret in the ${KUBE_NAMESPACE} namespace..." true
|
|
|
|
|
|
|
|
echo "${REVIEW_APPS_EE_LICENSE}" > /tmp/license.gitlab
|
|
|
|
|
|
|
|
kubectl create secret generic -n "$KUBE_NAMESPACE" \
|
|
|
|
"${CI_ENVIRONMENT_SLUG}-gitlab-license" \
|
|
|
|
--from-file=license=/tmp/license.gitlab \
|
|
|
|
--dry-run -o json | kubectl apply -f -
|
2018-09-25 10:32:53 -04:00
|
|
|
}
|
|
|
|
|
2019-05-23 10:37:29 -04:00
|
|
|
function download_chart() {
|
2019-03-27 05:36:01 -04:00
|
|
|
echoinfo "Downloading the GitLab chart..." true
|
2018-09-20 10:37:21 -04:00
|
|
|
|
2019-09-03 11:04:00 -04:00
|
|
|
curl --location -o gitlab.tar.bz2 "https://gitlab.com/gitlab-org/charts/gitlab/-/archive/${GITLAB_HELM_CHART_REF}/gitlab-${GITLAB_HELM_CHART_REF}.tar.bz2"
|
2019-03-27 05:36:01 -04:00
|
|
|
tar -xjf gitlab.tar.bz2
|
2019-04-09 07:12:49 -04:00
|
|
|
cd "gitlab-${GITLAB_HELM_CHART_REF}"
|
2019-03-27 05:36:01 -04:00
|
|
|
|
|
|
|
echoinfo "Adding the gitlab repo to Helm..."
|
|
|
|
helm repo add gitlab https://charts.gitlab.io
|
|
|
|
|
|
|
|
echoinfo "Building the gitlab chart's dependencies..."
|
|
|
|
helm dependency build .
|
2018-09-25 10:32:53 -04:00
|
|
|
}
|
|
|
|
|
2019-10-30 11:14:17 -04:00
|
|
|
function base_config_changed() {
|
|
|
|
git fetch origin master --depth=50
|
|
|
|
|
|
|
|
[ -n "$(git diff origin/master... --name-only -- scripts/review_apps/base-config.yaml)" ]
|
|
|
|
}
|
|
|
|
|
2018-09-25 10:32:53 -04:00
|
|
|
function deploy() {
|
2019-03-27 05:36:01 -04:00
|
|
|
local name="$CI_ENVIRONMENT_SLUG"
|
2019-09-16 14:06:05 -04:00
|
|
|
local edition="${GITLAB_EDITION-ce}"
|
2019-10-30 11:14:17 -04:00
|
|
|
local base_config_file_ref="master"
|
|
|
|
echo "REVIEW_APP_CONFIG_CHANGED: ${REVIEW_APP_CONFIG_CHANGED}"
|
|
|
|
if [ -n "${REVIEW_APP_CONFIG_CHANGED}" ]; then
|
|
|
|
base_config_file_ref="$CI_COMMIT_SHA"
|
|
|
|
fi
|
|
|
|
local base_config_file="https://gitlab.com/gitlab-org/gitlab/raw/${base_config_file_ref}/scripts/review_apps/base-config.yaml"
|
|
|
|
|
2019-03-27 05:36:01 -04:00
|
|
|
echoinfo "Deploying ${name}..." true
|
2018-09-25 10:32:53 -04:00
|
|
|
|
2018-11-06 17:59:25 -05:00
|
|
|
IMAGE_REPOSITORY="registry.gitlab.com/gitlab-org/build/cng-mirror"
|
2019-09-16 14:06:05 -04:00
|
|
|
gitlab_migrations_image_repository="${IMAGE_REPOSITORY}/gitlab-rails-${edition}"
|
|
|
|
gitlab_sidekiq_image_repository="${IMAGE_REPOSITORY}/gitlab-sidekiq-${edition}"
|
|
|
|
gitlab_unicorn_image_repository="${IMAGE_REPOSITORY}/gitlab-unicorn-${edition}"
|
|
|
|
gitlab_task_runner_image_repository="${IMAGE_REPOSITORY}/gitlab-task-runner-${edition}"
|
2018-11-06 17:59:25 -05:00
|
|
|
gitlab_gitaly_image_repository="${IMAGE_REPOSITORY}/gitaly"
|
|
|
|
gitlab_shell_image_repository="${IMAGE_REPOSITORY}/gitlab-shell"
|
2019-09-16 14:06:05 -04:00
|
|
|
gitlab_workhorse_image_repository="${IMAGE_REPOSITORY}/gitlab-workhorse-${edition}"
|
2018-09-25 10:32:53 -04:00
|
|
|
|
2019-05-23 10:37:29 -04:00
|
|
|
create_application_secret
|
2018-09-25 10:32:53 -04:00
|
|
|
|
|
|
|
HELM_CMD=$(cat << EOF
|
|
|
|
helm upgrade --install \
|
2019-08-27 07:01:13 -04:00
|
|
|
--wait \
|
2019-07-23 11:53:17 -04:00
|
|
|
--timeout 900 \
|
2019-09-06 17:47:47 -04:00
|
|
|
--set ci.branch="$CI_COMMIT_REF_NAME" \
|
|
|
|
--set ci.commit.sha="$CI_COMMIT_SHORT_SHA" \
|
|
|
|
--set ci.job.url="$CI_JOB_URL" \
|
|
|
|
--set ci.pipeline.url="$CI_PIPELINE_URL" \
|
2018-09-25 10:32:53 -04:00
|
|
|
--set releaseOverride="$CI_ENVIRONMENT_SLUG" \
|
|
|
|
--set global.hosts.hostSuffix="$HOST_SUFFIX" \
|
|
|
|
--set global.hosts.domain="$REVIEW_APPS_DOMAIN" \
|
|
|
|
--set gitlab.migrations.image.repository="$gitlab_migrations_image_repository" \
|
2019-01-17 09:13:31 -05:00
|
|
|
--set gitlab.migrations.image.tag="$CI_COMMIT_REF_SLUG" \
|
2019-04-09 07:12:49 -04:00
|
|
|
--set gitlab.gitaly.image.repository="$gitlab_gitaly_image_repository" \
|
2018-09-25 10:32:53 -04:00
|
|
|
--set gitlab.gitaly.image.tag="v$GITALY_VERSION" \
|
2019-04-09 07:12:49 -04:00
|
|
|
--set gitlab.gitlab-shell.image.repository="$gitlab_shell_image_repository" \
|
2018-09-25 10:32:53 -04:00
|
|
|
--set gitlab.gitlab-shell.image.tag="v$GITLAB_SHELL_VERSION" \
|
2019-06-28 05:38:38 -04:00
|
|
|
--set gitlab.sidekiq.image.repository="$gitlab_sidekiq_image_repository" \
|
|
|
|
--set gitlab.sidekiq.image.tag="$CI_COMMIT_REF_SLUG" \
|
|
|
|
--set gitlab.unicorn.image.repository="$gitlab_unicorn_image_repository" \
|
|
|
|
--set gitlab.unicorn.image.tag="$CI_COMMIT_REF_SLUG" \
|
2018-09-25 10:32:53 -04:00
|
|
|
--set gitlab.unicorn.workhorse.image="$gitlab_workhorse_image_repository" \
|
2019-01-17 09:13:31 -05:00
|
|
|
--set gitlab.unicorn.workhorse.tag="$CI_COMMIT_REF_SLUG" \
|
2019-06-28 05:38:38 -04:00
|
|
|
--set gitlab.task-runner.image.repository="$gitlab_task_runner_image_repository" \
|
|
|
|
--set gitlab.task-runner.image.tag="$CI_COMMIT_REF_SLUG"
|
|
|
|
EOF
|
|
|
|
)
|
|
|
|
|
2019-11-04 13:06:28 -05:00
|
|
|
if [ -n "${REVIEW_APPS_EE_LICENSE}" ]; then
|
|
|
|
HELM_CMD=$(cat << EOF
|
|
|
|
${HELM_CMD} \
|
|
|
|
--set global.gitlab.license.secret="${CI_ENVIRONMENT_SLUG}-gitlab-license"
|
|
|
|
EOF
|
|
|
|
)
|
|
|
|
fi
|
|
|
|
|
2019-06-28 05:38:38 -04:00
|
|
|
HELM_CMD=$(cat << EOF
|
2019-10-30 11:14:17 -04:00
|
|
|
${HELM_CMD} \
|
2019-06-28 05:38:38 -04:00
|
|
|
--namespace="$KUBE_NAMESPACE" \
|
2019-10-30 11:14:17 -04:00
|
|
|
--version="${CI_PIPELINE_ID}-${CI_JOB_ID}" \
|
|
|
|
-f "${base_config_file}" \
|
|
|
|
"${name}" .
|
2018-09-25 10:32:53 -04:00
|
|
|
EOF
|
|
|
|
)
|
|
|
|
|
2019-03-27 05:36:01 -04:00
|
|
|
echoinfo "Deploying with:"
|
|
|
|
echoinfo "${HELM_CMD}"
|
2018-09-25 10:32:53 -04:00
|
|
|
|
2019-05-23 10:37:29 -04:00
|
|
|
eval "${HELM_CMD}"
|
|
|
|
}
|
|
|
|
|
|
|
|
function display_deployment_debug() {
|
2019-10-10 11:06:07 -04:00
|
|
|
# Get all pods for this release
|
|
|
|
echoinfo "Pods for release ${CI_ENVIRONMENT_SLUG}"
|
|
|
|
kubectl get pods -n "$KUBE_NAMESPACE" -lrelease=${CI_ENVIRONMENT_SLUG}
|
2019-09-13 09:26:31 -04:00
|
|
|
|
2019-10-07 08:06:18 -04:00
|
|
|
# Get all non-completed jobs
|
|
|
|
echoinfo "Unsuccessful Jobs for release ${CI_ENVIRONMENT_SLUG}"
|
|
|
|
kubectl get jobs -n "$KUBE_NAMESPACE" -lrelease=${CI_ENVIRONMENT_SLUG} --field-selector=status.successful!=1
|
2019-04-08 13:10:18 -04:00
|
|
|
}
|