From ab696cdf28064ec088de0292290461b57fd900f5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Philippe=20Lafoucri=C3=A8re?= <plafoucriere@gitlab.com>
Date: Tue, 25 Jun 2019 08:57:33 +0000
Subject: [PATCH] Fix reports jobs timing out because of cache

Note: This commit relies on
https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/26801
which brings support to multiple extends.

While `cache: {}` can disable a cache being set globally, it doesn't
work with extends, which will perform a reverse deep merge based on the
keys.

The cache defined in the base `.default-cache` job
won't be disabled in the report jobs.

As a side effect, the `code_quality`, `sast`, and `dependency_scanning`
jobs are running on a larger code base than expected, leading to
timeouts.

fixes https://gitlab.com/gitlab-org/gitlab-ee/issues/11303
fixes https://gitlab.com/gitlab-org/gitlab-ce/issues/60879
fixes https://gitlab.com/gitlab-org/gitlab-ee/issues/12021
---
 .gitlab/ci/global.gitlab-ci.yml               | 14 ++-
 .gitlab/ci/reports.gitlab-ci.yml              | 88 ++-----------------
 .../60879-fix-reports-timing-out.yml          |  5 ++
 3 files changed, 25 insertions(+), 82 deletions(-)
 create mode 100644 changelogs/unreleased/60879-fix-reports-timing-out.yml

diff --git a/.gitlab/ci/global.gitlab-ci.yml b/.gitlab/ci/global.gitlab-ci.yml
index b7ef4b31743..eb50f08c1a7 100644
--- a/.gitlab/ci/global.gitlab-ci.yml
+++ b/.gitlab/ci/global.gitlab-ci.yml
@@ -28,11 +28,15 @@
     policy: pull
   stage: test
 
-.dedicated-no-docs-pull-cache-job:
-  extends: .dedicated-pull-cache-job
+.no-docs:
   except:
     - /(^docs[\/-].*|.*-docs$)/
 
+.dedicated-no-docs-pull-cache-job:
+  extends:
+    - .dedicated-pull-cache-job
+    - .no-docs
+
 .dedicated-no-docs-and-no-qa-pull-cache-job:
   extends: .dedicated-pull-cache-job
   except:
@@ -45,6 +49,12 @@
   variables:
     SETUP_DB: "false"
 
+# Jobs that need a dedicated runner, with no cache
+.dedicated-no-docs:
+  extends:
+    - .dedicated-runner
+    - .no-docs
+
 .single-script-job-dedicated-runner:
   extends: .dedicated-runner
   image: ruby:2.6-alpine
diff --git a/.gitlab/ci/reports.gitlab-ci.yml b/.gitlab/ci/reports.gitlab-ci.yml
index d0e09dbf2f8..89b5ae38072 100644
--- a/.gitlab/ci/reports.gitlab-ci.yml
+++ b/.gitlab/ci/reports.gitlab-ci.yml
@@ -1,98 +1,26 @@
 include:
   - template: Code-Quality.gitlab-ci.yml
+  - template: Security/SAST.gitlab-ci.yml
+  - template: Security/Dependency-Scanning.gitlab-ci.yml
 
 code_quality:
-  extends: .dedicated-no-docs-no-db-pull-cache-job
+  extends: .dedicated-no-docs
   # gitlab-org runners set `privileged: false` but we need to have it set to true
   # since we're using Docker in Docker
   tags: []
   before_script: []
   cache: {}
   dependencies: []
-  variables:
-    SETUP_DB: "false"
 
 sast:
-  extends: .dedicated-no-docs-no-db-pull-cache-job
-  image: docker:stable
+  extends: .dedicated-no-docs
+  before_script: []
+  tags: []
   variables:
     SAST_CONFIDENCE_LEVEL: 2
     DOCKER_DRIVER: overlay2
-  allow_failure: true
-  tags: []
-  before_script: []
-  cache: {}
-  dependencies: []
-  services:
-    - docker:stable-dind
-  script:
-    - | # this is required to avoid undesirable reset of Docker image ENV variables being set on build stage
-      function propagate_env_vars() {
-        CURRENT_ENV=$(printenv)
-
-        for VAR_NAME; do
-          echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME "
-        done
-      }
-    - export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')
-    - |
-      docker run \
-        $(propagate_env_vars \
-          SAST_ANALYZER_IMAGES \
-          SAST_ANALYZER_IMAGE_PREFIX \
-          SAST_ANALYZER_IMAGE_TAG \
-          SAST_DEFAULT_ANALYZERS \
-          SAST_BRAKEMAN_LEVEL \
-          SAST_GOSEC_LEVEL \
-          SAST_FLAWFINDER_LEVEL \
-          SAST_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \
-          SAST_PULL_ANALYZER_IMAGE_TIMEOUT \
-          SAST_RUN_ANALYZER_TIMEOUT \
-        ) \
-        --volume "$PWD:/code" \
-        --volume /var/run/docker.sock:/var/run/docker.sock \
-        "registry.gitlab.com/gitlab-org/security-products/sast:$SP_VERSION" /app/bin/run /code
-  artifacts:
-    reports:
-      sast: gl-sast-report.json
 
 dependency_scanning:
-  extends: .dedicated-no-docs-no-db-pull-cache-job
-  image: docker:stable
-  variables:
-    DOCKER_DRIVER: overlay2
-  allow_failure: true
-  tags: []
   before_script: []
-  cache: {}
-  dependencies: []
-  services:
-    - docker:stable-dind
-  script:
-    - export SP_VERSION=$(echo "$CI_SERVER_VERSION" | sed 's/^\([0-9]*\)\.\([0-9]*\).*/\1-\2-stable/')
-    - | # this is required to avoid undesirable reset of Docker image ENV variables being set on build stage
-      function propagate_env_vars() {
-        CURRENT_ENV=$(printenv)
-
-        for VAR_NAME; do
-          echo $CURRENT_ENV | grep "${VAR_NAME}=" > /dev/null && echo "--env $VAR_NAME "
-        done
-      }
-    - |
-      docker run \
-        $(propagate_env_vars \
-          DS_ANALYZER_IMAGES \
-          DS_ANALYZER_IMAGE_PREFIX \
-          DS_ANALYZER_IMAGE_TAG \
-          DS_DEFAULT_ANALYZERS \
-          DEP_SCAN_DISABLE_REMOTE_CHECKS \
-          DS_DOCKER_CLIENT_NEGOTIATION_TIMEOUT \
-          DS_PULL_ANALYZER_IMAGE_TIMEOUT \
-          DS_RUN_ANALYZER_TIMEOUT \
-        ) \
-        --volume "$PWD:/code" \
-        --volume /var/run/docker.sock:/var/run/docker.sock \
-        "registry.gitlab.com/gitlab-org/security-products/dependency-scanning:$SP_VERSION" /code
-  artifacts:
-    reports:
-      dependency_scanning: gl-dependency-scanning-report.json
+  tags: []
+  extends: .dedicated-no-docs
diff --git a/changelogs/unreleased/60879-fix-reports-timing-out.yml b/changelogs/unreleased/60879-fix-reports-timing-out.yml
new file mode 100644
index 00000000000..845162fe10f
--- /dev/null
+++ b/changelogs/unreleased/60879-fix-reports-timing-out.yml
@@ -0,0 +1,5 @@
+---
+title: Fix reports jobs timing out because of cache
+merge_request: 29780
+author:
+type: fixed