{{ importInitiatorText }}
diff --git a/app/assets/javascripts/jira_import/components/jira_import_setup.vue b/app/assets/javascripts/jira_import/components/jira_import_setup.vue index 44773a773d5..285c5c815ac 100644 --- a/app/assets/javascripts/jira_import/components/jira_import_setup.vue +++ b/app/assets/javascripts/jira_import/components/jira_import_setup.vue @@ -11,6 +11,10 @@ export default { type: String, required: true, }, + jiraIntegrationPath: { + type: String, + required: true, + }, }, }; @@ -21,6 +25,6 @@ export default { title="" :description="__('You will first need to set up Jira Integration to use this feature.')" :primary-button-text="__('Set up Jira Integration')" - primary-button-link="../services/jira/edit" + :primary-button-link="jiraIntegrationPath" /> diff --git a/app/assets/javascripts/jira_import/index.js b/app/assets/javascripts/jira_import/index.js index 8bd70e4e277..b576668fe7c 100644 --- a/app/assets/javascripts/jira_import/index.js +++ b/app/assets/javascripts/jira_import/index.js @@ -27,6 +27,7 @@ export default function mountJiraImportApp() { inProgressIllustration: el.dataset.inProgressIllustration, isJiraConfigured: parseBoolean(el.dataset.isJiraConfigured), issuesPath: el.dataset.issuesPath, + jiraIntegrationPath: el.dataset.jiraIntegrationPath, jiraProjects: el.dataset.jiraProjects ? JSON.parse(el.dataset.jiraProjects) : [], projectPath: el.dataset.projectPath, setupIllustration: el.dataset.setupIllustration, diff --git a/app/assets/javascripts/monitoring/components/dashboard_with_alerts.vue b/app/assets/javascripts/monitoring/components/dashboard_with_alerts.vue deleted file mode 100644 index be92414fd56..00000000000 --- a/app/assets/javascripts/monitoring/components/dashboard_with_alerts.vue +++ /dev/null @@ -1,25 +0,0 @@ - diff --git a/app/assets/javascripts/monitoring/monitoring_bundle.js b/app/assets/javascripts/monitoring/monitoring_bundle.js index 99af8ccaf05..2bbf9ef9d78 100644 --- a/app/assets/javascripts/monitoring/monitoring_bundle.js +++ b/app/assets/javascripts/monitoring/monitoring_bundle.js @@ -1,6 +1,6 @@ import Vue from 'vue'; import { GlToast } from '@gitlab/ui'; -import Dashboard from '~/monitoring/components/dashboard_with_alerts.vue'; +import Dashboard from '~/monitoring/components/dashboard.vue'; import { parseBoolean } from '~/lib/utils/common_utils'; import { getParameterValues } from '~/lib/utils/url_utility'; import store from './stores'; diff --git a/app/views/projects/import/jira/show.html.haml b/app/views/projects/import/jira/show.html.haml index 4106bcc2e5a..cddd97cbc84 100644 --- a/app/views/projects/import/jira/show.html.haml +++ b/app/views/projects/import/jira/show.html.haml @@ -1,6 +1,7 @@ - if Feature.enabled?(:jira_issue_import_vue, @project, default_enabled: true) .js-jira-import-root{ data: { project_path: @project.full_path, issues_path: project_issues_path(@project), + jira_integration_path: edit_project_service_path(@project, :jira), is_jira_configured: @project.jira_service.present?.to_s, jira_projects: @jira_projects.to_json, in_progress_illustration: image_path('illustrations/export-import.svg'), diff --git a/changelogs/unreleased/28566-add-sec-binaries-template.yml b/changelogs/unreleased/28566-add-sec-binaries-template.yml new file mode 100644 index 00000000000..e9e00911f20 --- /dev/null +++ b/changelogs/unreleased/28566-add-sec-binaries-template.yml @@ -0,0 +1,5 @@ +--- +title: Add secure binaries template +merge_request: 28566 +author: +type: added diff --git a/changelogs/unreleased/cngo-fix-jira-importer-urls.yml b/changelogs/unreleased/cngo-fix-jira-importer-urls.yml new file mode 100644 index 00000000000..e4b4f17ee99 --- /dev/null +++ b/changelogs/unreleased/cngo-fix-jira-importer-urls.yml @@ -0,0 +1,5 @@ +--- +title: Fix Jira importer URLs +merge_request: 30155 +author: +type: added diff --git a/changelogs/unreleased/dz-scope-pipeline-routes.yml b/changelogs/unreleased/dz-scope-pipeline-routes.yml new file mode 100644 index 00000000000..ed735200c43 --- /dev/null +++ b/changelogs/unreleased/dz-scope-pipeline-routes.yml @@ -0,0 +1,5 @@ +--- +title: Copy pipelines routing under - scope +merge_request: 30159 +author: +type: changed diff --git a/changelogs/unreleased/sh-log-api-errors.yml b/changelogs/unreleased/sh-log-api-errors.yml new file mode 100644 index 00000000000..eeeca6c4de6 --- /dev/null +++ b/changelogs/unreleased/sh-log-api-errors.yml @@ -0,0 +1,5 @@ +--- +title: Log server responses of API bad requests in api_json.log +merge_request: 29839 +author: +type: other diff --git a/config/routes/pipelines.rb b/config/routes/pipelines.rb new file mode 100644 index 00000000000..9b236a8ce17 --- /dev/null +++ b/config/routes/pipelines.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +resources :pipelines, only: [:index, :new, :create, :show, :destroy] do + collection do + resource :pipelines_settings, path: 'settings', only: [:show, :update] + get :charts + scope '(*ref)', constraints: { ref: Gitlab::PathRegex.git_reference_regex } do + get :latest, action: :show, defaults: { latest: true } + end + end + + member do + get :stage + get :stage_ajax + post :cancel + post :retry + get :builds + get :failures + get :status + get :test_report + get :test_reports_count + end + + member do + resources :stages, only: [], param: :name do + post :play_manual + end + end +end + +resources :pipeline_schedules, except: [:show] do + member do + post :play + post :take_ownership + end +end diff --git a/config/routes/project.rb b/config/routes/project.rb index 6e5784a8f92..67afbe5e7f5 100644 --- a/config/routes/project.rb +++ b/config/routes/project.rb @@ -365,39 +365,15 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do post 'alerts/notify', to: 'alerting/notifications#create' - resources :pipelines, only: [:index, :new, :create, :show, :destroy] do - collection do - resource :pipelines_settings, path: 'settings', only: [:show, :update] - get :charts - scope '(*ref)', constraints: { ref: Gitlab::PathRegex.git_reference_regex } do - get :latest, action: :show, defaults: { latest: true } - end - end + # Unscoped route. It will be replaced with redirect to /-/pipelines/ + # Issue https://gitlab.com/gitlab-org/gitlab/issues/118849 + draw :pipelines - member do - get :stage - get :stage_ajax - post :cancel - post :retry - get :builds - get :failures - get :status - get :test_report - get :test_reports_count - end - - member do - resources :stages, only: [], param: :name do - post :play_manual - end - end - end - - resources :pipeline_schedules, except: [:show] do - member do - post :play - post :take_ownership - end + # To ensure an old unscoped routing is used for the UI we need to + # add prefix 'as' to the scope routing and place it below original routing. + # Issue https://gitlab.com/gitlab-org/gitlab/issues/118849 + scope '-', as: 'scoped' do + draw :pipelines end draw :legacy_builds diff --git a/doc/administration/gitaly/praefect.md b/doc/administration/gitaly/praefect.md index 4d3bc498e3e..6810d3ff60d 100644 --- a/doc/administration/gitaly/praefect.md +++ b/doc/administration/gitaly/praefect.md @@ -13,7 +13,7 @@ unavailable, Praefect will automatically route traffic to a warm Gitaly replica. The current version supports: - Eventual consistency of the secondary replicas. -- Automatic fail over from the primary to the secondary. +- Automatic failover from the primary to the secondary. - Reporting of possible data loss if replication queue is non empty. Follow the [HA Gitaly epic](https://gitlab.com/groups/gitlab-org/-/epics/1489) @@ -266,7 +266,7 @@ application server, or a Gitaly node. NOTE: **Note:** The `gitaly-1` node is currently denoted the primary. This can be used to manually fail from one node to another. This will be removed - in the future to allow for automatic failover. + in the [future](https://gitlab.com/gitlab-org/gitaly/-/issues/2634). ```ruby # Name of storage hash must match storage name in git_data_dirs on GitLab @@ -290,11 +290,36 @@ application server, or a Gitaly node. } ``` -1. Enable the replication queue: +1. Enable the database replication queue: - ```ruby - praefect['postgres_queue_enabled'] = true - ``` + ```ruby + praefect['postgres_queue_enabled'] = true + ``` + + In the next release, database replication queue will be enabled by default. + See [issue #2615](https://gitlab.com/gitlab-org/gitaly/-/issues/2615). + +1. Enable automatic failover by editing `/etc/gitlab/gitlab.rb`: + + ```ruby + praefect['failover_enabled'] = true + praefect['failover_election_strategy'] = 'sql' + ``` + + When automatic failover is enabled, Praefect checks the health of internal + Gitaly nodes. If the primary has a certain amount of health checks fail, it + will promote one of the secondaries to be primary, and demote the primary to + be a secondary. + + NOTE: **Note:** Database leader election will be [enabled by default in the + future](https://gitlab.com/gitlab-org/gitaly/-/issues/2682). + + Caution, **automatic failover** favors availability over consistency and will + cause data loss if changes have not been replicated to the newly elected + primary. In the next release, leader election will [prefer to promote up to + date replicas](https://gitlab.com/gitlab-org/gitaly/-/issues/2642), and it + will be an option to favor consistency by marking [out-of-date repositories + read-only](https://gitlab.com/gitlab-org/gitaly/-/issues/2630). 1. Save the changes to `/etc/gitlab/gitlab.rb` and [reconfigure Praefect](../restart_gitlab.md#omnibus-gitlab-reconfigure): @@ -312,60 +337,6 @@ application server, or a Gitaly node. edit `/etc/gitlab/gitlab.rb`, remember to run `sudo gitlab-ctl reconfigure` again before trying the `sql-ping` command. -#### Automatic failover - -When automatic failover is enabled, Praefect will do automatic detection of the health of internal Gitaly nodes. If the -primary has a certain amount of health checks fail, it will decide to promote one of the secondaries to be primary, and -demote the primary to be a secondary. - -1. To enable automatic failover, edit `/etc/gitlab/gitlab.rb`: - - ```ruby - # failover_enabled turns on automatic failover - praefect['failover_enabled'] = true - praefect['virtual_storages'] = { - 'storage-1' => { - 'gitaly-1' => { - 'address' => 'tcp://GITALY_HOST:8075', - 'token' => 'PRAEFECT_INTERNAL_TOKEN', - 'primary' => true - }, - 'gitaly-2' => { - 'address' => 'tcp://GITALY_HOST:8075', - 'token' => 'PRAEFECT_INTERNAL_TOKEN' - }, - 'gitaly-3' => { - 'address' => 'tcp://GITALY_HOST:8075', - 'token' => 'PRAEFECT_INTERNAL_TOKEN' - } - } - } - ``` - -1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure). - -Below is the picture when Praefect starts up with the config.toml above: - -```mermaid -graph TD - A[Praefect] -->|Mutator RPC| B(internal_storage_0) - B --> |Replication|C[internal_storage_1] -``` - -Let's say suddenly `internal_storage_0` goes down. Praefect will detect this and -automatically switch over to `internal_storage_1`, and `internal_storage_0` will serve as a secondary: - -```mermaid -graph TD - A[Praefect] -->|Mutator RPC| B(internal_storage_1) - B --> |Replication|C[internal_storage_0] -``` - -NOTE: **Note:**: Currently this feature is supported for setups that only have 1 Praefect instance. Praefect instances running, -for example behind a load balancer, `failover_enabled` should be disabled. The reason is The reason is because there -is no coordination that currently happens across different Praefect instances, so there could be a situation where -two Praefect instances think two different Gitaly nodes are the primary. - ### Gitaly NOTE: **Note:** Complete these steps for **each** Gitaly node. @@ -520,38 +491,6 @@ config. sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml dial-nodes ``` -1. Enable automatic failover by editing `/etc/gitlab/gitlab.rb`: - - ```ruby - praefect['failover_enabled'] = true - ``` - - When automatic failover is enabled, Praefect checks the health of internal - Gitaly nodes. If the primary has a certain amount of health checks fail, it - will promote one of the secondaries to be primary, and demote the primary to - be a secondary. - - Manual failover is possible by updating `praefect['virtual_storages']` and - nominating a new primary node. - -1. By default, Praefect will nominate a primary Gitaly node for each - shard and store the state of the primary in local memory. This state - does not persist across restarts and will cause a split brain - if multiple Praefect nodes are used for redundancy. - - To avoid this limitation, enable the SQL election strategy: - - ```ruby - praefect['failover_election_strategy'] = 'sql' - ``` - -1. Save the changes to `/etc/gitlab/gitlab.rb` and [reconfigure - Praefect](../restart_gitlab.md#omnibus-gitlab-reconfigure): - - ```shell - gitlab-ctl reconfigure - ``` - ### Load Balancer In a highly available Gitaly configuration, a load balancer is needed to route @@ -756,6 +695,13 @@ Praefect regularly checks the health of each backend Gitaly node. This information can be used to automatically failover to a new primary node if the current primary node is found to be unhealthy. +- **PostgreSQL (recommended):** Enabled by setting + `praefect['failover_election_strategy'] = sql`. This configuration + option will allow multiple Praefect nodes to coordinate via the + PostgreSQL database to elect a primary Gitaly node. This configuration + will cause Praefect nodes to elect a new primary, monitor its health, + and elect a new primary if the current one has not been reachable in + 10 seconds by a majority of the Praefect nodes. - **Manual:** Automatic failover is disabled. The primary node can be reconfigured in `/etc/gitlab/gitlab.rb` on the Praefect node. Modify the `praefect['virtual_storages']` field by moving the `primary = true` to promote @@ -766,13 +712,6 @@ current primary node is found to be unhealthy. checks fail for the current primary backend Gitaly node, and new primary will be elected. **Do not use with multiple Praefect nodes!** Using with multiple Praefect nodes is likely to result in a split brain. -- **PostgreSQL:** Enabled by setting - `praefect['failover_election_strategy'] = sql`. This configuration - option will allow multiple Praefect nodes to coordinate via the - PostgreSQL database to elect a primary Gitaly node. This configuration - will cause Praefect nodes to elect a new primary, monitor its health, - and elect a new primary if the current one has not been reachable in - 10 seconds by a majority of the Praefect nodes. NOTE: **Note:**: Praefect does not yet account for replication lag on the secondaries during the election process, so data loss can occur diff --git a/doc/topics/airgap/index.md b/doc/topics/airgap/index.md index e712e3bb6b5..29722a734c5 100644 --- a/doc/topics/airgap/index.md +++ b/doc/topics/airgap/index.md @@ -30,7 +30,73 @@ example of such a transfer: 1. Transfer images to offline environment. 1. Load transferred images into offline Docker registry. -### Example image packager script +### Using the official GitLab template + +GitLab provides a [vendored template](../../ci/yaml/README.md#includetemplate) +to ease this process. + +This template should be used in a new, empty project, with a `gitlab-ci.yml` file containing: + +```yaml +include: + - template: Secure-Binaries.gitlab-ci.yml +``` + +The pipeline downloads the Docker images needed for the Security Scanners and saves them as +[job artifacts](../../ci/pipelines/job_artifacts.md) or pushes them to the [Container Registry](../../user/packages/container_registry/index.md) +of the project where the pipeline is executed. These archives can be transferred to another location +and [loaded](https://docs.docker.com/engine/reference/commandline/load/) in a Docker daemon. +This method requires a GitLab Runner with access to both `gitlab.com` (including +`registry.gitlab.com`) and the local offline instance. This runner must run in +[privileged mode](https://docs.gitlab.com/runner/executors/docker.html#use-docker-in-docker-with-privileged-mode) +to be able to use the `docker` command inside the jobs. This runner can be installed in a DMZ or on +a bastion, and used only for this specific project. + +#### Scheduling the updates + +By default, this project's pipeline will run only once, when the `.gitlab-ci.yml` is added to the +repo. To update the GitLab security scanners and signatures, it's necessary to run this pipeline +regularly. GitLab provides a way to [schedule pipelines](../../ci/pipelines/schedules.md). For +example, you can set this up to download and store the Docker images every week. + +Some images can be updated more frequently than others. For example, the [vulnerability database](https://hub.docker.com/r/arminc/clair-db/tags) +for Container Scanning is updated daily. To update this single image, create a new Scheduled +Pipeline that runs daily and set `SECURE_BINARIES_ANALYZERS` to `clair-vulnerabilities-db`. Only +this job will be triggered, and the image will be updated daily and made available in the project +registry. + +#### Using the secure bundle created + +The project using the `Secure-Binaries.gitlab-ci.yml` template should now host all the required +images and resources needed to run GitLab Security features. + +The next step is to tell the offline instance to use these resources instead of the default ones on +`gitlab.com`. This can be done by setting the right environment variables: +`SAST_ANALYZER_IMAGE_PREFIX` for SAST analyzers, `DS_ANALYZER_IMAGE_PREFIX` for Dependency Scanning, +and so on. + +You can set these variables in the project's `.gitlab-ci.yml` files by using the bundle directly, or +in the GitLab UI at the project or group level. See the [GitLab CI/CD environment variables page](../../ci/variables/README.md#creating-a-custom-environment-variable) +for more information. + +#### Variables + +The following table shows which variables you can use with the `Secure-Binaries.gitlab-ci.yml` +template: + +| VARIABLE | Description | Default value | +|-------------------------------------------|-----------------------------------------------|-----------------------------------| +| `SECURE_BINARIES_ANALYZERS` | Comma-separated list of analyzers to download | `"bandit, brakeman, gosec, and so on..."` | +| `SECURE_BINARIES_DOWNLOAD_IMAGES` | Used to disable jobs | `"true"` | +| `SECURE_BINARIES_PUSH_IMAGES` | Push files to the project registry | `"true"` | +| `SECURE_BINARIES_SAVE_ARTIFACTS` | Also save image archives as artifacts | `"false"` | +| `SECURE_BINARIES_ANALYZER_VERSION` | Default analyzer version (docker tag) | `"2"` | + +### Alternate way without the official template + +If it's not possible to follow the above method, the images can be transferred manually instead: + +#### Example image packager script ```sh #!/bin/bash @@ -49,7 +115,7 @@ do done ``` -### Example image loader script +#### Example image loader script This example loads the images from a bastion host to an offline host. In certain configurations, physical media may be needed for such a transfer: diff --git a/lib/api/helpers.rb b/lib/api/helpers.rb index 42b82aac1c4..5ce3353b734 100644 --- a/lib/api/helpers.rb +++ b/lib/api/helpers.rb @@ -11,6 +11,7 @@ module API SUDO_PARAM = :sudo API_USER_ENV = 'gitlab.api.user' API_EXCEPTION_ENV = 'gitlab.api.exception' + API_RESPONSE_STATUS_CODE = 'gitlab.api.response_status_code' def declared_params(options = {}) options = { include_parent_namespaces: false }.merge(options) @@ -416,6 +417,11 @@ module API end def render_api_error!(message, status) + # grape-logging doesn't pass the status code, so this is a + # workaround for getting that information in the loggers: + # https://github.com/aserafin/grape_logging/issues/71 + env[API_RESPONSE_STATUS_CODE] = Rack::Utils.status_code(status) + error!({ 'message' => message }, status, header) end diff --git a/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml new file mode 100644 index 00000000000..a7945e31a5d --- /dev/null +++ b/lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml @@ -0,0 +1,252 @@ +# This template should be used when Security Products (https://about.gitlab.com/handbook/engineering/development/secure/#security-products) +# have to be downloaded and stored locally. +# +# Usage: +# +# ``` +# include: +# - template: Secure-Binaries.gitlab-ci.yml +# ``` +# +# Docs: https://docs.gitlab.com/ee/topics/airgap/ + + +variables: + SECURE_BINARIES_ANALYZERS: >- + bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, tslint, secrets, sobelow, pmd-apex, kubesec, + bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python, + klar, clair-vulnerabilities-db, + license-management, + dast + + SECURE_BINARIES_DOWNLOAD_IMAGES: "true" + SECURE_BINARIES_PUSH_IMAGES: "true" + SECURE_BINARIES_SAVE_ARTIFACTS: "false" + + SECURE_BINARIES_ANALYZER_VERSION: "2" + +.download_images: + allow_failure: true + image: docker:stable + only: + refs: + - branches + variables: + DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "" + services: + - docker:stable-dind + script: + - docker info + - env + - if [ -z "$SECURE_BINARIES_IMAGE" ]; then export SECURE_BINARIES_IMAGE=${SECURE_BINARIES_IMAGE:-"registry.gitlab.com/gitlab-org/security-products/${CI_JOB_NAME}:${SECURE_BINARIES_ANALYZER_VERSION}"}; fi + - docker pull ${SECURE_BINARIES_IMAGE} + - mkdir -p output/$(dirname ${CI_JOB_NAME}) + - | + if [ "$SECURE_BINARIES_SAVE_ARTIFACTS" = "true" ]; then + docker save ${SECURE_BINARIES_IMAGE} -o output/${CI_JOB_NAME}_${SECURE_BINARIES_ANALYZER_VERSION}.tar + gzip output/${CI_JOB_NAME}_${SECURE_BINARIES_ANALYZER_VERSION}.tar + sha256sum output/${CI_JOB_NAME}_${SECURE_BINARIES_ANALYZER_VERSION}.tar.gz > output/${CI_JOB_NAME}_${SECURE_BINARIES_ANALYZER_VERSION}.tag.gz.sha256sum + fi + - | + if [ "$SECURE_BINARIES_PUSH_IMAGES" = "true" ]; then + docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY + docker tag ${SECURE_BINARIES_IMAGE} ${CI_REGISTRY_IMAGE}/${CI_JOB_NAME}:${SECURE_BINARIES_ANALYZER_VERSION} + docker push ${CI_REGISTRY_IMAGE}/${CI_JOB_NAME}:${SECURE_BINARIES_ANALYZER_VERSION} + fi + + artifacts: + paths: + - output/ + +# +# SAST jobs +# + +analyzers/bandit: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bbandit\b/ + +analyzers/brakeman: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bbrakeman\b/ + +analyzers/gosec: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bgosec\b/ + +analyzers/spotbugs: + extends: .download_images + variables: + # TODO: Spotbugs is > 1GB, disabling for now + SECURE_BINARIES_SAVE_ARTIFACTS: "false" + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bspotbugs\b/ + +analyzers/flawfinder: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bflawfinder\b/ + +analyzers/phpcs-security-audit: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bphpcs-security-audit\b/ + +analyzers/security-code-scan: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bsecurity-code-scan\b/ + +analyzers/nodejs-scan: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bnodejs-scan\b/ + +analyzers/eslint: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\beslint\b/ + +analyzers/tslint: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\btslint\b/ + +analyzers/secrets: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bsecrets\b/ + +analyzers/sobelow: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bsobelow\b/ + +analyzers/pmd-apex: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bsecrets\b/ + +analyzers/kubesec: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bkubesec\b/ +# +# Container Scanning jobs +# + +analyzers/klar: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bklar\b/ + +analyzers/clair-vulnerabilities-db: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bclair-vulnerabilities-db\b/ + variables: + SECURE_BINARIES_IMAGE: arminc/clair-db + SECURE_BINARIES_ANALYZER_VERSION: latest + +# +# Dependency Scanning jobs +# + +analyzers/bundler-audit: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bbundler-audit\b/ + +analyzers/retire.js: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bretire\.js\b/ + +analyzers/gemnasium: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bgemnasium\b/ + +analyzers/gemnasium-maven: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bgemnasium-maven\b/ + +analyzers/gemnasium-python: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bgemnasium-python\b/ + +# +# License Scanning +# + +license-management: + extends: .download_images + variables: + SECURE_BINARIES_ANALYZER_VERSION: "${CI_SERVER_VERSION_MAJOR}-${CI_SERVER_VERSION_MINOR}-stable" + # TODO: license-management is > 1GB, disabling for now + SECURE_BINARIES_SAVE_ARTIFACTS: "false" + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\blicense-management\b/ + +# +# DAST +# + +dast: + extends: .download_images + only: + variables: + - $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" && + $SECURE_BINARIES_ANALYZERS =~ /\bdast\b/ + variables: + SECURE_BINARIES_ANALYZER_VERSION: 1 diff --git a/lib/gitlab/grape_logging/loggers/exception_logger.rb b/lib/gitlab/grape_logging/loggers/exception_logger.rb index 606b7c0dbce..14147769422 100644 --- a/lib/gitlab/grape_logging/loggers/exception_logger.rb +++ b/lib/gitlab/grape_logging/loggers/exception_logger.rb @@ -4,14 +4,16 @@ module Gitlab module GrapeLogging module Loggers class ExceptionLogger < ::GrapeLogging::Loggers::Base - def parameters(request, _) + def parameters(request, response_body) + data = {} + data[:api_error] = format_body(response_body) if bad_request?(request) + # grape-logging attempts to pass the logger the exception # (https://github.com/aserafin/grape_logging/blob/v1.7.0/lib/grape_logging/middleware/request_logger.rb#L63), # but it appears that the rescue_all in api.rb takes # precedence so the logger never sees it. We need to # store and retrieve the exception from the environment. exception = request.env[::API::Helpers::API_EXCEPTION_ENV] - data = {} return data unless exception.is_a?(Exception) @@ -19,6 +21,28 @@ module Gitlab data end + + private + + def format_body(response_body) + # https://github.com/rack/rack/blob/master/SPEC.rdoc#label-The+Body: + # The response_body must respond to each, but just in case we + # guard against errors here. + response_body = Array(response_body) unless response_body.respond_to?(:each) + + # To avoid conflicting types in Elasticsearch, convert every + # element into an Array of strings. A response body is usually + # an array of Strings so that the response can be sent in + # chunks. + body = [] + # each_with_object doesn't work with Rack::BodyProxy + response_body.each { |chunk| body << chunk.to_s } + body + end + + def bad_request?(request) + request.env[::API::Helpers::API_RESPONSE_STATUS_CODE] == 400 + end end end end diff --git a/lib/gitlab/metrics/exporter/sidekiq_exporter.rb b/lib/gitlab/metrics/exporter/sidekiq_exporter.rb index 5ba7b29734b..054b4949dd6 100644 --- a/lib/gitlab/metrics/exporter/sidekiq_exporter.rb +++ b/lib/gitlab/metrics/exporter/sidekiq_exporter.rb @@ -32,7 +32,7 @@ module Gitlab Sidekiq.logger.error( class: self.class.to_s, message: 'Cannot start sidekiq_exporter', - exception: e.message + 'exception.message' => e.message ) false diff --git a/spec/frontend/jira_import/components/jira_import_app_spec.js b/spec/frontend/jira_import/components/jira_import_app_spec.js index ce32559d5c9..70713ca3cd7 100644 --- a/spec/frontend/jira_import/components/jira_import_app_spec.js +++ b/spec/frontend/jira_import/components/jira_import_app_spec.js @@ -26,6 +26,7 @@ const mountComponent = ({ ['My Second Jira Project', 'MSJP'], ['Migrate to GitLab', 'MTG'], ], + jiraIntegrationPath: 'gitlab-org/gitlab-test/-/services/jira/edit', projectPath: 'gitlab-org/gitlab-test', setupIllustration: 'setup-illustration.svg', }, diff --git a/spec/frontend/jira_import/components/jira_import_progress_spec.js b/spec/frontend/jira_import/components/jira_import_progress_spec.js index 9a6fc3b5925..ace497295c0 100644 --- a/spec/frontend/jira_import/components/jira_import_progress_spec.js +++ b/spec/frontend/jira_import/components/jira_import_progress_spec.js @@ -2,6 +2,10 @@ import { GlEmptyState } from '@gitlab/ui'; import { mount, shallowMount } from '@vue/test-utils'; import JiraImportProgress from '~/jira_import/components/jira_import_progress.vue'; +const illustration = 'illustration.svg'; +const importProject = 'JIRAPROJECT'; +const issuesPath = 'gitlab-org/gitlab-test/-/issues'; + describe('JiraImportProgress', () => { let wrapper; @@ -13,11 +17,11 @@ describe('JiraImportProgress', () => { const mountFunction = mountType === 'shallowMount' ? shallowMount : mount; return mountFunction(JiraImportProgress, { propsData: { - illustration: 'illustration.svg', + illustration, importInitiator: 'Jane Doe', - importProject: 'JIRAPROJECT', + importProject, importTime: '2020-04-08T12:17:25+00:00', - issuesPath: 'gitlab-org/gitlab-test/-/issues', + issuesPath, }, }); }; @@ -33,7 +37,7 @@ describe('JiraImportProgress', () => { }); it('contains illustration', () => { - expect(getGlEmptyStateAttribute('svgpath')).toBe('illustration.svg'); + expect(getGlEmptyStateAttribute('svgpath')).toBe(illustration); }); it('contains a title', () => { @@ -46,7 +50,8 @@ describe('JiraImportProgress', () => { }); it('contains button url', () => { - expect(getGlEmptyStateAttribute('primarybuttonlink')).toBe('gitlab-org/gitlab-test/-/issues'); + const expected = `${issuesPath}?search=${importProject}`; + expect(getGlEmptyStateAttribute('primarybuttonlink')).toBe(expected); }); }); diff --git a/spec/frontend/jira_import/components/jira_import_setup_spec.js b/spec/frontend/jira_import/components/jira_import_setup_spec.js index 834c14b512e..4c8cba70d8a 100644 --- a/spec/frontend/jira_import/components/jira_import_setup_spec.js +++ b/spec/frontend/jira_import/components/jira_import_setup_spec.js @@ -2,6 +2,9 @@ import { GlEmptyState } from '@gitlab/ui'; import { shallowMount } from '@vue/test-utils'; import JiraImportSetup from '~/jira_import/components/jira_import_setup.vue'; +const illustration = 'illustration.svg'; +const jiraIntegrationPath = 'gitlab-org/gitlab-test/-/services/jira/edit'; + describe('JiraImportSetup', () => { let wrapper; @@ -10,7 +13,8 @@ describe('JiraImportSetup', () => { beforeEach(() => { wrapper = shallowMount(JiraImportSetup, { propsData: { - illustration: 'illustration.svg', + illustration, + jiraIntegrationPath, }, }); }); @@ -21,7 +25,7 @@ describe('JiraImportSetup', () => { }); it('contains illustration', () => { - expect(getGlEmptyStateAttribute('svgpath')).toBe('illustration.svg'); + expect(getGlEmptyStateAttribute('svgpath')).toBe(illustration); }); it('contains a description', () => { @@ -32,4 +36,8 @@ describe('JiraImportSetup', () => { it('contains button text', () => { expect(getGlEmptyStateAttribute('primarybuttontext')).toBe('Set up Jira Integration'); }); + + it('contains button link', () => { + expect(getGlEmptyStateAttribute('primarybuttonlink')).toBe(jiraIntegrationPath); + }); }); diff --git a/spec/frontend/monitoring/components/dashboard_spec.js b/spec/frontend/monitoring/components/dashboard_spec.js index d8c9af59f90..0ee0a2e6e41 100644 --- a/spec/frontend/monitoring/components/dashboard_spec.js +++ b/spec/frontend/monitoring/components/dashboard_spec.js @@ -6,7 +6,7 @@ import MockAdapter from 'axios-mock-adapter'; import axios from '~/lib/utils/axios_utils'; import statusCodes from '~/lib/utils/http_status'; import { metricStates } from '~/monitoring/constants'; -import Dashboard from '~/monitoring/components/dashboard_with_alerts.vue'; +import Dashboard from '~/monitoring/components/dashboard.vue'; import DateTimePicker from '~/vue_shared/components/date_time_picker/date_time_picker.vue'; import CustomMetricsFormFields from '~/custom_metrics/components/custom_metrics_form_fields.vue'; diff --git a/spec/lib/gitlab/grape_logging/loggers/exception_logger_spec.rb b/spec/lib/gitlab/grape_logging/loggers/exception_logger_spec.rb index c9021e2f436..cc9535d4d2c 100644 --- a/spec/lib/gitlab/grape_logging/loggers/exception_logger_spec.rb +++ b/spec/lib/gitlab/grape_logging/loggers/exception_logger_spec.rb @@ -3,14 +3,73 @@ require 'spec_helper' describe Gitlab::GrapeLogging::Loggers::ExceptionLogger do - subject { described_class.new } - let(:mock_request) { OpenStruct.new(env: {}) } + let(:response_body) { nil } describe ".parameters" do + subject { described_class.new.parameters(mock_request, response_body) } + describe 'when no exception is available' do it 'returns an empty hash' do - expect(subject.parameters(mock_request, nil)).to eq({}) + expect(subject).to eq({}) + end + end + + describe 'with a response' do + before do + mock_request.env[::API::Helpers::API_RESPONSE_STATUS_CODE] = code + end + + context 'with a String response' do + let(:response_body) { { message: "something went wrong" }.to_json } + let(:code) { 400 } + let(:expected) { { api_error: [response_body.to_s] } } + + it 'logs the response body' do + expect(subject).to eq(expected) + end + end + + context 'with an Array response' do + let(:response_body) { ["hello world", 1] } + let(:code) { 400 } + let(:expected) { { api_error: ["hello world", "1"] } } + + it 'casts all elements to strings' do + expect(subject).to eq(expected) + end + end + + # Rack v2.0.9 can return a BodyProxy. This was changed in later versions: + # https://github.com/rack/rack/blob/2.0.9/lib/rack/response.rb#L69 + context 'with a Rack BodyProxy response' do + let(:message) { { message: "something went wrong" }.to_json } + let(:response) { Rack::Response.new(message, code, {}) } + let(:response_body) { Rack::BodyProxy.new(response) } + let(:code) { 400 } + let(:expected) { { api_error: [message] } } + + it 'logs the response body' do + expect(subject).to eq(expected) + end + end + + context 'unauthorized error' do + let(:response_body) { 'unauthorized' } + let(:code) { 401 } + + it 'does not log an api_error field' do + expect(subject).not_to have_key(:api_error) + end + end + + context 'HTTP success' do + let(:response_body) { 'success' } + let(:code) { 200 } + + it 'does not log an api_error field' do + expect(subject).not_to have_key(:api_error) + end end end @@ -32,7 +91,7 @@ describe Gitlab::GrapeLogging::Loggers::ExceptionLogger do end it 'returns the correct fields' do - expect(subject.parameters(mock_request, nil)).to eq(expected) + expect(subject).to eq(expected) end context 'with backtrace' do @@ -43,7 +102,7 @@ describe Gitlab::GrapeLogging::Loggers::ExceptionLogger do end it 'includes the backtrace' do - expect(subject.parameters(mock_request, nil)).to eq(expected) + expect(subject).to eq(expected) end end end diff --git a/spec/lib/gitlab/metrics/exporter/sidekiq_exporter_spec.rb b/spec/lib/gitlab/metrics/exporter/sidekiq_exporter_spec.rb index a415b6407d5..0b820fdbde9 100644 --- a/spec/lib/gitlab/metrics/exporter/sidekiq_exporter_spec.rb +++ b/spec/lib/gitlab/metrics/exporter/sidekiq_exporter_spec.rb @@ -53,7 +53,7 @@ describe Gitlab::Metrics::Exporter::SidekiqExporter do .with( class: described_class.to_s, message: 'Cannot start sidekiq_exporter', - exception: anything) + 'exception.message' => anything) exporter.start end diff --git a/spec/requests/api/helpers_spec.rb b/spec/requests/api/helpers_spec.rb index 98904a4d79f..d65c89f48ea 100644 --- a/spec/requests/api/helpers_spec.rb +++ b/spec/requests/api/helpers_spec.rb @@ -328,6 +328,8 @@ describe API::Helpers do it 'returns a 401 response' do expect { authenticate! }.to raise_error /401/ + + expect(env[described_class::API_RESPONSE_STATUS_CODE]).to eq(401) end end @@ -340,6 +342,8 @@ describe API::Helpers do it 'does not raise an error' do expect { authenticate! }.not_to raise_error + + expect(env[described_class::API_RESPONSE_STATUS_CODE]).to be_nil end end end