Split Security Auto DevOps jobs into templates
Container Scanning, Dependency Scanning, License Management and SAST parts of Auto DevOps now use the preexisting templates. Auto DevOps and the DAST job template will now use a shared job template instead of maintaining two copies of the job. This also allows Auto DevOps to use custom authentication with DAST.
This commit is contained in:
parent
ec5e032a12
commit
3be46bdf08
|
@ -78,217 +78,8 @@ include:
|
|||
- template: Jobs/Code-Quality.gitlab-ci.yml
|
||||
- template: Jobs/Deploy.gitlab-ci.yml
|
||||
- template: Jobs/Browser-Performance-Testing.gitlab-ci.yml
|
||||
|
||||
license_management:
|
||||
stage: test
|
||||
image:
|
||||
name: "registry.gitlab.com/gitlab-org/security-products/license-management:$CI_SERVER_VERSION_MAJOR-$CI_SERVER_VERSION_MINOR-stable"
|
||||
entrypoint: [""]
|
||||
allow_failure: true
|
||||
script:
|
||||
- license_management
|
||||
artifacts:
|
||||
paths: [gl-license-management-report.json]
|
||||
only:
|
||||
refs:
|
||||
- branches
|
||||
- tags
|
||||
variables:
|
||||
- $GITLAB_FEATURES =~ /\blicense_management\b/
|
||||
except:
|
||||
variables:
|
||||
- $LICENSE_MANAGEMENT_DISABLED
|
||||
|
||||
sast:
|
||||
stage: test
|
||||
image: docker:stable
|
||||
allow_failure: true
|
||||
services:
|
||||
- docker:stable-dind
|
||||
script:
|
||||
- setup_docker
|
||||
- sast
|
||||
artifacts:
|
||||
reports:
|
||||
sast: gl-sast-report.json
|
||||
only:
|
||||
refs:
|
||||
- branches
|
||||
- tags
|
||||
variables:
|
||||
- $GITLAB_FEATURES =~ /\bsast\b/
|
||||
except:
|
||||
variables:
|
||||
- $SAST_DISABLED
|
||||
|
||||
dependency_scanning:
|
||||
stage: test
|
||||
image: docker:stable
|
||||
allow_failure: true
|
||||
services:
|
||||
- docker:stable-dind
|
||||
script:
|
||||
- setup_docker
|
||||
- dependency_scanning
|
||||
artifacts:
|
||||
reports:
|
||||
dependency_scanning: gl-dependency-scanning-report.json
|
||||
only:
|
||||
refs:
|
||||
- branches
|
||||
- tags
|
||||
variables:
|
||||
- $GITLAB_FEATURES =~ /\bdependency_scanning\b/
|
||||
except:
|
||||
variables:
|
||||
- $DEPENDENCY_SCANNING_DISABLED
|
||||
|
||||
container_scanning:
|
||||
stage: test
|
||||
image: docker:stable
|
||||
allow_failure: true
|
||||
services:
|
||||
- docker:stable-dind
|
||||
script:
|
||||
- setup_docker
|
||||
- container_scanning
|
||||
artifacts:
|
||||
paths: [gl-container-scanning-report.json]
|
||||
only:
|
||||
refs:
|
||||
- branches
|
||||
- tags
|
||||
variables:
|
||||
- $GITLAB_FEATURES =~ /\bcontainer_scanning\b/
|
||||
except:
|
||||
variables:
|
||||
- $CONTAINER_SCANNING_DISABLED
|
||||
|
||||
dast:
|
||||
stage: dast
|
||||
allow_failure: true
|
||||
image: registry.gitlab.com/gitlab-org/security-products/zaproxy
|
||||
variables:
|
||||
POSTGRES_DB: "false"
|
||||
script:
|
||||
- dast
|
||||
artifacts:
|
||||
paths: [gl-dast-report.json]
|
||||
only:
|
||||
refs:
|
||||
- branches
|
||||
- tags
|
||||
kubernetes: active
|
||||
variables:
|
||||
- $GITLAB_FEATURES =~ /\bdast\b/
|
||||
except:
|
||||
refs:
|
||||
- master
|
||||
variables:
|
||||
- $DAST_DISABLED
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
.auto_devops: &auto_devops |
|
||||
# Auto DevOps variables and functions
|
||||
[[ "$TRACE" ]] && set -x
|
||||
auto_database_url=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${CI_ENVIRONMENT_SLUG}-postgres:5432/${POSTGRES_DB}
|
||||
export DATABASE_URL=${DATABASE_URL-$auto_database_url}
|
||||
if [[ -z "$CI_COMMIT_TAG" ]]; then
|
||||
export CI_APPLICATION_REPOSITORY=$CI_REGISTRY_IMAGE/$CI_COMMIT_REF_SLUG
|
||||
export CI_APPLICATION_TAG=$CI_COMMIT_SHA
|
||||
else
|
||||
export CI_APPLICATION_REPOSITORY=$CI_REGISTRY_IMAGE
|
||||
export CI_APPLICATION_TAG=$CI_COMMIT_TAG
|
||||
fi
|
||||
export TILLER_NAMESPACE=$KUBE_NAMESPACE
|
||||
# Extract "MAJOR.MINOR" from CI_SERVER_VERSION and generate "MAJOR-MINOR-stable" for Security Products
|
||||
export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')
|
||||
|
||||
function registry_login() {
|
||||
if [[ -n "$CI_REGISTRY_USER" ]]; then
|
||||
echo "Logging to GitLab Container Registry with CI credentials..."
|
||||
docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY"
|
||||
echo ""
|
||||
fi
|
||||
}
|
||||
|
||||
function container_scanning() {
|
||||
registry_login
|
||||
|
||||
docker run -d --name db arminc/clair-db:latest
|
||||
docker run -p 6060:6060 --link db:postgres -d --name clair --restart on-failure arminc/clair-local-scan:v2.0.6
|
||||
apk add -U wget ca-certificates
|
||||
docker pull ${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG}
|
||||
wget https://github.com/arminc/clair-scanner/releases/download/v8/clair-scanner_linux_amd64
|
||||
mv clair-scanner_linux_amd64 clair-scanner
|
||||
chmod +x clair-scanner
|
||||
touch clair-whitelist.yml
|
||||
retries=0
|
||||
echo "Waiting for clair daemon to start"
|
||||
while( ! wget -T 10 -q -O /dev/null http://${DOCKER_SERVICE}:6060/v1/namespaces ) ; do sleep 1 ; echo -n "." ; if [ $retries -eq 10 ] ; then echo " Timeout, aborting." ; exit 1 ; fi ; retries=$(($retries+1)) ; done
|
||||
./clair-scanner -c http://${DOCKER_SERVICE}:6060 --ip $(hostname -i) -r gl-container-scanning-report.json -l clair.log -w clair-whitelist.yml ${CI_APPLICATION_REPOSITORY}:${CI_APPLICATION_TAG} || true
|
||||
}
|
||||
|
||||
function license_management() {
|
||||
/run.sh analyze .
|
||||
}
|
||||
|
||||
function sast() {
|
||||
case "$CI_SERVER_VERSION" in
|
||||
*-ee)
|
||||
|
||||
# Deprecation notice for CONFIDENCE_LEVEL variable
|
||||
if [ -z "$SAST_CONFIDENCE_LEVEL" -a "$CONFIDENCE_LEVEL" ]; then
|
||||
SAST_CONFIDENCE_LEVEL="$CONFIDENCE_LEVEL"
|
||||
echo "WARNING: CONFIDENCE_LEVEL is deprecated and MUST be replaced with SAST_CONFIDENCE_LEVEL"
|
||||
fi
|
||||
|
||||
docker run --env SAST_CONFIDENCE_LEVEL="${SAST_CONFIDENCE_LEVEL:-3}" \
|
||||
--volume "$PWD:/code" \
|
||||
--volume /var/run/docker.sock:/var/run/docker.sock \
|
||||
"registry.gitlab.com/gitlab-org/security-products/sast:$SP_VERSION" /app/bin/run /code
|
||||
;;
|
||||
*)
|
||||
echo "GitLab EE is required"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
function dependency_scanning() {
|
||||
case "$CI_SERVER_VERSION" in
|
||||
*-ee)
|
||||
docker run --env DEP_SCAN_DISABLE_REMOTE_CHECKS="${DEP_SCAN_DISABLE_REMOTE_CHECKS:-false}" \
|
||||
--volume "$PWD:/code" \
|
||||
--volume /var/run/docker.sock:/var/run/docker.sock \
|
||||
"registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$SP_VERSION" /code
|
||||
;;
|
||||
*)
|
||||
echo "GitLab EE is required"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# With the Kubernetes executor, 'localhost' must be used instead
|
||||
# https://docs.gitlab.com/runner/executors/kubernetes.html
|
||||
function setup_docker() {
|
||||
if ! docker info &>/dev/null; then
|
||||
if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
|
||||
export DOCKER_HOST='tcp://localhost:2375'
|
||||
export DOCKER_SERVICE="localhost"
|
||||
else
|
||||
export DOCKER_SERVICE="docker"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
function dast() {
|
||||
export CI_ENVIRONMENT_URL=$(cat environment_url.txt)
|
||||
|
||||
mkdir /zap/wrk/
|
||||
/zap/zap-baseline.py -J gl-dast-report.json -t "$CI_ENVIRONMENT_URL" || true
|
||||
cp /zap/wrk/gl-dast-report.json .
|
||||
}
|
||||
|
||||
before_script:
|
||||
- *auto_devops
|
||||
- template: Jobs/DAST.gitlab-ci.yml
|
||||
- template: Security/Container-Scanning.gitlab-ci.yml
|
||||
- template: Security/Dependency-Scanning.gitlab-ci.yml
|
||||
- template: Security/License-Management.gitlab-ci.yml
|
||||
- template: Security/SAST.gitlab-ci.yml
|
||||
|
|
|
@ -0,0 +1,54 @@
|
|||
dast:
|
||||
stage: dast
|
||||
image: docker:stable
|
||||
variables:
|
||||
DOCKER_DRIVER: overlay2
|
||||
allow_failure: true
|
||||
services:
|
||||
- docker:stable-dind
|
||||
script:
|
||||
- export DAST_WEBSITE=${DAST_WEBSITE:-$(cat environment_url.txt)}
|
||||
- export DAST_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')}
|
||||
- |
|
||||
if ! docker info &>/dev/null; then
|
||||
if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
|
||||
export DOCKER_HOST='tcp://localhost:2375'
|
||||
fi
|
||||
fi
|
||||
- |
|
||||
function dast_run() {
|
||||
docker run \
|
||||
--env DAST_TARGET_AVAILABILITY_TIMEOUT \
|
||||
--volume "$PWD:/output" \
|
||||
--volume /var/run/docker.sock:/var/run/docker.sock \
|
||||
-w /output \
|
||||
"registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION" \
|
||||
/analyze -t $DAST_WEBSITE \
|
||||
"$@"
|
||||
}
|
||||
- |
|
||||
if [ -n "$DAST_AUTH_URL" ]
|
||||
then
|
||||
dast_run \
|
||||
--auth-url $DAST_AUTH_URL \
|
||||
--auth-username $DAST_USERNAME \
|
||||
--auth-password $DAST_PASSWORD \
|
||||
--auth-username-field $DAST_USERNAME_FIELD \
|
||||
--auth-password-field $DAST_PASSWORD_FIELD
|
||||
else
|
||||
dast_run
|
||||
fi
|
||||
artifacts:
|
||||
reports:
|
||||
dast: gl-dast-report.json
|
||||
only:
|
||||
refs:
|
||||
- branches
|
||||
- tags
|
||||
variables:
|
||||
- $GITLAB_FEATURES =~ /\bdast\b/
|
||||
except:
|
||||
refs:
|
||||
- master
|
||||
variables:
|
||||
- $DAST_DISABLED
|
|
@ -28,6 +28,12 @@ container_scanning:
|
|||
- docker:stable-dind
|
||||
script:
|
||||
- if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then { export DOCKER_SERVICE="localhost" ; export DOCKER_HOST="tcp://${DOCKER_SERVICE}:2375" ; } fi
|
||||
- |
|
||||
if [[ -n "$CI_REGISTRY_USER" ]]; then
|
||||
echo "Logging to GitLab Container Registry with CI credentials..."
|
||||
docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" "$CI_REGISTRY"
|
||||
echo ""
|
||||
fi
|
||||
- docker run -d --name db arminc/clair-db:latest
|
||||
- docker run -p 6060:6060 --link db:postgres -d --name clair --restart on-failure arminc/clair-local-scan:${CLAIR_LOCAL_SCAN_VERSION}
|
||||
- apk add -U wget ca-certificates
|
||||
|
@ -36,7 +42,6 @@ container_scanning:
|
|||
- mv clair-scanner_linux_amd64 clair-scanner
|
||||
- chmod +x clair-scanner
|
||||
- touch clair-whitelist.yml
|
||||
- while( ! wget -q -O /dev/null http://${DOCKER_SERVICE}:6060/v1/namespaces ) ; do sleep 1 ; done
|
||||
- retries=0
|
||||
- echo "Waiting for clair daemon to start"
|
||||
- while( ! wget -T 10 -q -O /dev/null http://${DOCKER_SERVICE}:6060/v1/namespaces ) ; do sleep 1 ; echo -n "." ; if [ $retries -eq 10 ] ; then echo " Timeout, aborting." ; exit 1 ; fi ; retries=$(($retries+1)) ; done
|
||||
|
|
|
@ -4,6 +4,9 @@
|
|||
# List of the variables: https://gitlab.com/gitlab-org/security-products/dast#settings
|
||||
# How to set: https://docs.gitlab.com/ee/ci/yaml/#variables
|
||||
|
||||
include:
|
||||
- template: Jobs/DAST.gitlab-ci.yml
|
||||
|
||||
variables:
|
||||
DAST_WEBSITE: http://example.com # Please edit to be your website to scan for vulnerabilities
|
||||
|
||||
|
@ -14,46 +17,10 @@ stages:
|
|||
- dast
|
||||
|
||||
dast:
|
||||
stage: dast
|
||||
image: docker:stable
|
||||
variables:
|
||||
DOCKER_DRIVER: overlay2
|
||||
allow_failure: true
|
||||
services:
|
||||
- docker:stable-dind
|
||||
script:
|
||||
- export DAST_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')}
|
||||
- |
|
||||
function dast_run() {
|
||||
docker run \
|
||||
--env DAST_TARGET_AVAILABILITY_TIMEOUT \
|
||||
--volume "$PWD:/output" \
|
||||
--volume /var/run/docker.sock:/var/run/docker.sock \
|
||||
-w /output \
|
||||
"registry.gitlab.com/gitlab-org/security-products/dast:$DAST_VERSION" \
|
||||
/analyze -t $DAST_WEBSITE \
|
||||
"$@"
|
||||
}
|
||||
- |
|
||||
if [ -n "$DAST_AUTH_URL" ]
|
||||
then
|
||||
dast_run \
|
||||
--auth-url $DAST_AUTH_URL \
|
||||
--auth-username $DAST_USERNAME \
|
||||
--auth-password $DAST_PASSWORD \
|
||||
--auth-username-field $DAST_USERNAME_FIELD \
|
||||
--auth-password-field $DAST_PASSWORD_FIELD
|
||||
else
|
||||
dast_run
|
||||
fi
|
||||
artifacts:
|
||||
reports:
|
||||
dast: gl-dast-report.json
|
||||
only:
|
||||
refs:
|
||||
- branches
|
||||
variables:
|
||||
- $GITLAB_FEATURES =~ /\bdast\b/
|
||||
except:
|
||||
refs: [] # Override default from template
|
||||
variables:
|
||||
- $DAST_DISABLED
|
||||
|
|
|
@ -14,6 +14,12 @@ dependency_scanning:
|
|||
- docker:stable-dind
|
||||
script:
|
||||
- export DS_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')}
|
||||
- |
|
||||
if ! docker info &>/dev/null; then
|
||||
if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
|
||||
export DOCKER_HOST='tcp://localhost:2375'
|
||||
fi
|
||||
fi
|
||||
- |
|
||||
docker run \
|
||||
--env DS_ANALYZER_IMAGES \
|
||||
|
|
|
@ -14,6 +14,12 @@ sast:
|
|||
- docker:stable-dind
|
||||
script:
|
||||
- export SAST_VERSION=${SP_VERSION:-$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')}
|
||||
- |
|
||||
if ! docker info &>/dev/null; then
|
||||
if [ -z "$DOCKER_HOST" -a "$KUBERNETES_PORT" ]; then
|
||||
export DOCKER_HOST='tcp://localhost:2375'
|
||||
fi
|
||||
fi
|
||||
- |
|
||||
docker run \
|
||||
--env SAST_ANALYZER_IMAGES \
|
||||
|
|
|
@ -4,6 +4,7 @@ require 'spec_helper'
|
|||
|
||||
describe "CI YML Templates" do
|
||||
ABSTRACT_TEMPLATES = %w[Serverless].freeze
|
||||
PROJECT_DEPENDENT_TEMPLATES = %w[Auto-DevOps].freeze
|
||||
|
||||
def self.concrete_templates
|
||||
Gitlab::Template::GitlabCiYmlTemplate.all.reject do |template|
|
||||
|
@ -20,7 +21,10 @@ describe "CI YML Templates" do
|
|||
describe 'concrete templates with CI/CD jobs' do
|
||||
concrete_templates.each do |template|
|
||||
it "#{template.name} template should be valid" do
|
||||
expect { Gitlab::Ci::YamlProcessor.new(template.content) }
|
||||
# Trigger processing of included files
|
||||
project = create(:project, :test_repo) if PROJECT_DEPENDENT_TEMPLATES.include?(template.name)
|
||||
|
||||
expect { Gitlab::Ci::YamlProcessor.new(template.content, project: project) }
|
||||
.not_to raise_error
|
||||
end
|
||||
end
|
||||
|
|
Loading…
Reference in New Issue