kotovalexarian-likes-gitlab/gitlab-org--gitlab-foss
1
0
Fork 0
Code Releases Activity

Add latest changes from gitlab-org/gitlab@master

Browse source
This commit is contained in:
GitLab Bot 2020-04-23 12:09:46 +00:00
parent bd5d5791c5
commit c6acc1681a
28 changed files with 572 additions and 182 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
.gitlab/ci/rules.gitlab-ci.yml
View file

@ -557,11 +557,13 @@
.test-metadata:rules:update-tests-metadata:
rules:
- <<: *if-dot-com-ee-schedule
changes: *code-backstage-patterns
- <<: *if-not-ee
when: never
- changes:
- ".gitlab/ci/test-metadata.gitlab-ci.yml"
- "scripts/rspec_helpers.sh"
- <<: *if-dot-com-ee-schedule
changes: *code-backstage-patterns
##############
# YAML rules #

2
GITALY_SERVER_VERSION
View file

@ -1 +1 @@
12.10.0-rc1
12.10.0

10
app/assets/javascripts/jira_import/components/jira_import_app.vue
View file

@ -30,6 +30,10 @@ export default {
type: String,
required: true,
},
jiraIntegrationPath: {
type: String,
required: true,
},
jiraProjects: {
type: Array,
required: true,
@ -133,7 +137,11 @@ export default {
{{ errorMessage }}
</gl-alert>
<jira-import-setup v-if="!isJiraConfigured" :illustration="setupIllustration" />
<jira-import-setup
v-if="!isJiraConfigured"
:illustration="setupIllustration"
:jira-integration-path="jiraIntegrationPath"
/>
<gl-loading-icon v-else-if="$apollo.loading" size="md" class="mt-3" />
<jira-import-progress
v-else-if="isImportInProgress"

5
app/assets/javascripts/jira_import/components/jira_import_progress.vue
View file

@ -46,6 +46,9 @@ export default {
importTime: formatDate(this.importTime),
});
},
issuesLink() {
return `${this.issuesPath}?search=${this.importProject}`;
},
},
};
</script>
@ -55,7 +58,7 @@ export default {
:svg-path="illustration"
:title="__('Import in progress')"
:primary-button-text="__('View issues')"
:primary-button-link="issuesPath"
:primary-button-link="issuesLink"
>
<template #description>
<p class="mb-0">{{ importInitiatorText }}</p>

6
app/assets/javascripts/jira_import/components/jira_import_setup.vue
View file

@ -11,6 +11,10 @@ export default {
type: String,
required: true,
},
jiraIntegrationPath: {
type: String,
required: true,
},
},
};
</script>
@ -21,6 +25,6 @@ export default {
title=""
:description="__('You will first need to set up Jira Integration to use this feature.')"
:primary-button-text="__('Set up Jira Integration')"
primary-button-link="../services/jira/edit"
:primary-button-link="jiraIntegrationPath"
/>
</template>

1
app/assets/javascripts/jira_import/index.js
View file

@ -27,6 +27,7 @@ export default function mountJiraImportApp() {
inProgressIllustration: el.dataset.inProgressIllustration,
isJiraConfigured: parseBoolean(el.dataset.isJiraConfigured),
issuesPath: el.dataset.issuesPath,
jiraIntegrationPath: el.dataset.jiraIntegrationPath,
jiraProjects: el.dataset.jiraProjects ? JSON.parse(el.dataset.jiraProjects) : [],
projectPath: el.dataset.projectPath,
setupIllustration: el.dataset.setupIllustration,

25
app/assets/javascripts/monitoring/components/dashboard_with_alerts.vue
View file

@ -1,25 +0,0 @@
<script>
import CeDashboard from '~/monitoring/components/dashboard.vue';
import AlertWidget from './alert_widget.vue';
export default {
components: {
AlertWidget,
},
extends: CeDashboard,
data() {
return {
allAlerts: {},
};
},
methods: {
setAlerts(alertPath, alertAttributes) {
if (alertAttributes) {
this.$set(this.allAlerts, alertPath, alertAttributes);
} else {
this.$delete(this.allAlerts, alertPath);
}
},
},
};
</script>

2
app/assets/javascripts/monitoring/monitoring_bundle.js
View file

@ -1,6 +1,6 @@
import Vue from 'vue';
import { GlToast } from '@gitlab/ui';
import Dashboard from '~/monitoring/components/dashboard_with_alerts.vue';
import Dashboard from '~/monitoring/components/dashboard.vue';
import { parseBoolean } from '~/lib/utils/common_utils';
import { getParameterValues } from '~/lib/utils/url_utility';
import store from './stores';

1
app/views/projects/import/jira/show.html.haml
View file

@ -1,6 +1,7 @@
- if Feature.enabled?(:jira_issue_import_vue, @project, default_enabled: true)
.js-jira-import-root{ data: { project_path: @project.full_path,
issues_path: project_issues_path(@project),
jira_integration_path: edit_project_service_path(@project, :jira),
is_jira_configured: @project.jira_service.present?.to_s,
jira_projects: @jira_projects.to_json,
in_progress_illustration: image_path('illustrations/export-import.svg'),

5
changelogs/unreleased/28566-add-sec-binaries-template.yml Normal file
View file

@ -0,0 +1,5 @@
---
title: Add secure binaries template
merge_request: 28566
author:
type: added

5
changelogs/unreleased/cngo-fix-jira-importer-urls.yml Normal file
View file

@ -0,0 +1,5 @@
---
title: Fix Jira importer URLs
merge_request: 30155
author:
type: added

5
changelogs/unreleased/dz-scope-pipeline-routes.yml Normal file
View file

@ -0,0 +1,5 @@
---
title: Copy pipelines routing under - scope
merge_request: 30159
author:
type: changed

5
changelogs/unreleased/sh-log-api-errors.yml Normal file
View file

@ -0,0 +1,5 @@
---
title: Log server responses of API bad requests in api_json.log
merge_request: 29839
author:
type: other

36
config/routes/pipelines.rb Normal file
View file

@ -0,0 +1,36 @@
# frozen_string_literal: true
resources :pipelines, only: [:index, :new, :create, :show, :destroy] do
collection do
resource :pipelines_settings, path: 'settings', only: [:show, :update]
get :charts
scope '(*ref)', constraints: { ref: Gitlab::PathRegex.git_reference_regex } do
get :latest, action: :show, defaults: { latest: true }
end
end
member do
get :stage
get :stage_ajax
post :cancel
post :retry
get :builds
get :failures
get :status
get :test_report
get :test_reports_count
end
member do
resources :stages, only: [], param: :name do
post :play_manual
end
end
end
resources :pipeline_schedules, except: [:show] do
member do
post :play
post :take_ownership
end
end

40
config/routes/project.rb
View file

@ -365,39 +365,15 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
post 'alerts/notify', to: 'alerting/notifications#create'
resources :pipelines, only: [:index, :new, :create, :show, :destroy] do
collection do
resource :pipelines_settings, path: 'settings', only: [:show, :update]
get :charts
scope '(*ref)', constraints: { ref: Gitlab::PathRegex.git_reference_regex } do
get :latest, action: :show, defaults: { latest: true }
end
end
# Unscoped route. It will be replaced with redirect to /-/pipelines/
# Issue https://gitlab.com/gitlab-org/gitlab/issues/118849
draw :pipelines
member do
get :stage
get :stage_ajax
post :cancel
post :retry
get :builds
get :failures
get :status
get :test_report
get :test_reports_count
end
member do
resources :stages, only: [], param: :name do
post :play_manual
end
end
end
resources :pipeline_schedules, except: [:show] do
member do
post :play
post :take_ownership
end
# To ensure an old unscoped routing is used for the UI we need to
# add prefix 'as' to the scope routing and place it below original routing.
# Issue https://gitlab.com/gitlab-org/gitlab/issues/118849
scope '-', as: 'scoped' do
draw :pipelines
end
draw :legacy_builds

137
doc/administration/gitaly/praefect.md
View file

@ -13,7 +13,7 @@ unavailable, Praefect will automatically route traffic to a warm Gitaly replica.
The current version supports:
- Eventual consistency of the secondary replicas.
- Automatic fail over from the primary to the secondary.
- Automatic failover from the primary to the secondary.
- Reporting of possible data loss if replication queue is non empty.
Follow the [HA Gitaly epic](https://gitlab.com/groups/gitlab-org/-/epics/1489)
@ -266,7 +266,7 @@ application server, or a Gitaly node.
NOTE: **Note:** The `gitaly-1` node is currently denoted the primary. This
can be used to manually fail from one node to another. This will be removed
in the future to allow for automatic failover.
in the [future](https://gitlab.com/gitlab-org/gitaly/-/issues/2634).
```ruby
# Name of storage hash must match storage name in git_data_dirs on GitLab
@ -290,11 +290,36 @@ application server, or a Gitaly node.
}
```
1. Enable the replication queue:
1. Enable the database replication queue:
```ruby
praefect['postgres_queue_enabled'] = true
```
```ruby
praefect['postgres_queue_enabled'] = true
```
In the next release, database replication queue will be enabled by default.
See [issue #2615](https://gitlab.com/gitlab-org/gitaly/-/issues/2615).
1. Enable automatic failover by editing `/etc/gitlab/gitlab.rb`:
```ruby
praefect['failover_enabled'] = true
praefect['failover_election_strategy'] = 'sql'
```
When automatic failover is enabled, Praefect checks the health of internal
Gitaly nodes. If the primary has a certain amount of health checks fail, it
will promote one of the secondaries to be primary, and demote the primary to
be a secondary.
NOTE: **Note:** Database leader election will be [enabled by default in the
future](https://gitlab.com/gitlab-org/gitaly/-/issues/2682).
Caution, **automatic failover** favors availability over consistency and will
cause data loss if changes have not been replicated to the newly elected
primary. In the next release, leader election will [prefer to promote up to
date replicas](https://gitlab.com/gitlab-org/gitaly/-/issues/2642), and it
will be an option to favor consistency by marking [out-of-date repositories
read-only](https://gitlab.com/gitlab-org/gitaly/-/issues/2630).
1. Save the changes to `/etc/gitlab/gitlab.rb` and [reconfigure Praefect](../restart_gitlab.md#omnibus-gitlab-reconfigure):
@ -312,60 +337,6 @@ application server, or a Gitaly node.
edit `/etc/gitlab/gitlab.rb`, remember to run `sudo gitlab-ctl reconfigure`
again before trying the `sql-ping` command.
#### Automatic failover
When automatic failover is enabled, Praefect will do automatic detection of the health of internal Gitaly nodes. If the
primary has a certain amount of health checks fail, it will decide to promote one of the secondaries to be primary, and
demote the primary to be a secondary.
1. To enable automatic failover, edit `/etc/gitlab/gitlab.rb`:
```ruby
# failover_enabled turns on automatic failover
praefect['failover_enabled'] = true
praefect['virtual_storages'] = {
'storage-1' => {
'gitaly-1' => {
'address' => 'tcp://GITALY_HOST:8075',
'token' => 'PRAEFECT_INTERNAL_TOKEN',
'primary' => true
},
'gitaly-2' => {
'address' => 'tcp://GITALY_HOST:8075',
'token' => 'PRAEFECT_INTERNAL_TOKEN'
},
'gitaly-3' => {
'address' => 'tcp://GITALY_HOST:8075',
'token' => 'PRAEFECT_INTERNAL_TOKEN'
}
}
}
```
1. Save the file and [reconfigure GitLab](../restart_gitlab.md#omnibus-gitlab-reconfigure).
Below is the picture when Praefect starts up with the config.toml above:
```mermaid
graph TD
A[Praefect] -->|Mutator RPC| B(internal_storage_0)
B --> |Replication|C[internal_storage_1]
```
Let's say suddenly `internal_storage_0` goes down. Praefect will detect this and
automatically switch over to `internal_storage_1`, and `internal_storage_0` will serve as a secondary:
```mermaid
graph TD
A[Praefect] -->|Mutator RPC| B(internal_storage_1)
B --> |Replication|C[internal_storage_0]
```
NOTE: **Note:**: Currently this feature is supported for setups that only have 1 Praefect instance. Praefect instances running,
for example behind a load balancer, `failover_enabled` should be disabled. The reason is The reason is because there
is no coordination that currently happens across different Praefect instances, so there could be a situation where
two Praefect instances think two different Gitaly nodes are the primary.
### Gitaly
NOTE: **Note:** Complete these steps for **each** Gitaly node.
@ -520,38 +491,6 @@ config.
sudo /opt/gitlab/embedded/bin/praefect -config /var/opt/gitlab/praefect/config.toml dial-nodes
```
1. Enable automatic failover by editing `/etc/gitlab/gitlab.rb`:
```ruby
praefect['failover_enabled'] = true
```
When automatic failover is enabled, Praefect checks the health of internal
Gitaly nodes. If the primary has a certain amount of health checks fail, it
will promote one of the secondaries to be primary, and demote the primary to
be a secondary.
Manual failover is possible by updating `praefect['virtual_storages']` and
nominating a new primary node.
1. By default, Praefect will nominate a primary Gitaly node for each
shard and store the state of the primary in local memory. This state
does not persist across restarts and will cause a split brain
if multiple Praefect nodes are used for redundancy.
To avoid this limitation, enable the SQL election strategy:
```ruby
praefect['failover_election_strategy'] = 'sql'
```
1. Save the changes to `/etc/gitlab/gitlab.rb` and [reconfigure
Praefect](../restart_gitlab.md#omnibus-gitlab-reconfigure):
```shell
gitlab-ctl reconfigure
```
### Load Balancer
In a highly available Gitaly configuration, a load balancer is needed to route
@ -756,6 +695,13 @@ Praefect regularly checks the health of each backend Gitaly node. This
information can be used to automatically failover to a new primary node if the
current primary node is found to be unhealthy.
- **PostgreSQL (recommended):** Enabled by setting
`praefect['failover_election_strategy'] = sql`. This configuration
option will allow multiple Praefect nodes to coordinate via the
PostgreSQL database to elect a primary Gitaly node. This configuration
will cause Praefect nodes to elect a new primary, monitor its health,
and elect a new primary if the current one has not been reachable in
10 seconds by a majority of the Praefect nodes.
- **Manual:** Automatic failover is disabled. The primary node can be
reconfigured in `/etc/gitlab/gitlab.rb` on the Praefect node. Modify the
`praefect['virtual_storages']` field by moving the `primary = true` to promote
@ -766,13 +712,6 @@ current primary node is found to be unhealthy.
checks fail for the current primary backend Gitaly node, and new primary will
be elected. **Do not use with multiple Praefect nodes!** Using with multiple
Praefect nodes is likely to result in a split brain.
- **PostgreSQL:** Enabled by setting
`praefect['failover_election_strategy'] = sql`. This configuration
option will allow multiple Praefect nodes to coordinate via the
PostgreSQL database to elect a primary Gitaly node. This configuration
will cause Praefect nodes to elect a new primary, monitor its health,
and elect a new primary if the current one has not been reachable in
10 seconds by a majority of the Praefect nodes.
NOTE: **Note:**: Praefect does not yet account for replication lag on
the secondaries during the election process, so data loss can occur

70
doc/topics/airgap/index.md
View file

@ -30,7 +30,73 @@ example of such a transfer:
1. Transfer images to offline environment.
1. Load transferred images into offline Docker registry.
### Example image packager script
### Using the official GitLab template
GitLab provides a [vendored template](../../ci/yaml/README.md#includetemplate)
to ease this process.
This template should be used in a new, empty project, with a `gitlab-ci.yml` file containing:
```yaml
include:
- template: Secure-Binaries.gitlab-ci.yml
```
The pipeline downloads the Docker images needed for the Security Scanners and saves them as
[job artifacts](../../ci/pipelines/job_artifacts.md) or pushes them to the [Container Registry](../../user/packages/container_registry/index.md)
of the project where the pipeline is executed. These archives can be transferred to another location
and [loaded](https://docs.docker.com/engine/reference/commandline/load/) in a Docker daemon.
This method requires a GitLab Runner with access to both `gitlab.com` (including
`registry.gitlab.com`) and the local offline instance. This runner must run in
[privileged mode](https://docs.gitlab.com/runner/executors/docker.html#use-docker-in-docker-with-privileged-mode)
to be able to use the `docker` command inside the jobs. This runner can be installed in a DMZ or on
a bastion, and used only for this specific project.
#### Scheduling the updates
By default, this project's pipeline will run only once, when the `.gitlab-ci.yml` is added to the
repo. To update the GitLab security scanners and signatures, it's necessary to run this pipeline
regularly. GitLab provides a way to [schedule pipelines](../../ci/pipelines/schedules.md). For
example, you can set this up to download and store the Docker images every week.
Some images can be updated more frequently than others. For example, the [vulnerability database](https://hub.docker.com/r/arminc/clair-db/tags)
for Container Scanning is updated daily. To update this single image, create a new Scheduled
Pipeline that runs daily and set `SECURE_BINARIES_ANALYZERS` to `clair-vulnerabilities-db`. Only
this job will be triggered, and the image will be updated daily and made available in the project
registry.
#### Using the secure bundle created
The project using the `Secure-Binaries.gitlab-ci.yml` template should now host all the required
images and resources needed to run GitLab Security features.
The next step is to tell the offline instance to use these resources instead of the default ones on
`gitlab.com`. This can be done by setting the right environment variables:
`SAST_ANALYZER_IMAGE_PREFIX` for SAST analyzers, `DS_ANALYZER_IMAGE_PREFIX` for Dependency Scanning,
and so on.
You can set these variables in the project's `.gitlab-ci.yml` files by using the bundle directly, or
in the GitLab UI at the project or group level. See the [GitLab CI/CD environment variables page](../../ci/variables/README.md#creating-a-custom-environment-variable)
for more information.
#### Variables
The following table shows which variables you can use with the `Secure-Binaries.gitlab-ci.yml`
template:
| VARIABLE | Description | Default value |
|-------------------------------------------|-----------------------------------------------|-----------------------------------|
| `SECURE_BINARIES_ANALYZERS` | Comma-separated list of analyzers to download | `"bandit, brakeman, gosec, and so on..."` |
| `SECURE_BINARIES_DOWNLOAD_IMAGES` | Used to disable jobs | `"true"` |
| `SECURE_BINARIES_PUSH_IMAGES` | Push files to the project registry | `"true"` |
| `SECURE_BINARIES_SAVE_ARTIFACTS` | Also save image archives as artifacts | `"false"` |
| `SECURE_BINARIES_ANALYZER_VERSION` | Default analyzer version (docker tag) | `"2"` |
### Alternate way without the official template
If it's not possible to follow the above method, the images can be transferred manually instead:
#### Example image packager script
```sh
#!/bin/bash
@ -49,7 +115,7 @@ do
done
```
### Example image loader script
#### Example image loader script
This example loads the images from a bastion host to an offline host. In certain configurations,
physical media may be needed for such a transfer:

6
lib/api/helpers.rb
View file

@ -11,6 +11,7 @@ module API
SUDO_PARAM = :sudo
API_USER_ENV = 'gitlab.api.user'
API_EXCEPTION_ENV = 'gitlab.api.exception'
API_RESPONSE_STATUS_CODE = 'gitlab.api.response_status_code'
def declared_params(options = {})
options = { include_parent_namespaces: false }.merge(options)
@ -416,6 +417,11 @@ module API
end
def render_api_error!(message, status)
# grape-logging doesn't pass the status code, so this is a
# workaround for getting that information in the loggers:
# https://github.com/aserafin/grape_logging/issues/71
env[API_RESPONSE_STATUS_CODE] = Rack::Utils.status_code(status)
error!({ 'message' => message }, status, header)
end

252
lib/gitlab/ci/templates/Security/Secure-Binaries.gitlab-ci.yml Normal file
View file

@ -0,0 +1,252 @@
# This template should be used when Security Products (https://about.gitlab.com/handbook/engineering/development/secure/#security-products)
# have to be downloaded and stored locally.
#
# Usage:
#
# ```
# include:
# - template: Secure-Binaries.gitlab-ci.yml
# ```
#
# Docs: https://docs.gitlab.com/ee/topics/airgap/
variables:
SECURE_BINARIES_ANALYZERS: >-
bandit, brakeman, gosec, spotbugs, flawfinder, phpcs-security-audit, security-code-scan, nodejs-scan, eslint, tslint, secrets, sobelow, pmd-apex, kubesec,
bundler-audit, retire.js, gemnasium, gemnasium-maven, gemnasium-python,
klar, clair-vulnerabilities-db,
license-management,
dast
SECURE_BINARIES_DOWNLOAD_IMAGES: "true"
SECURE_BINARIES_PUSH_IMAGES: "true"
SECURE_BINARIES_SAVE_ARTIFACTS: "false"
SECURE_BINARIES_ANALYZER_VERSION: "2"
.download_images:
allow_failure: true
image: docker:stable
only:
refs:
- branches
variables:
DOCKER_DRIVER: overlay2
DOCKER_TLS_CERTDIR: ""
services:
- docker:stable-dind
script:
- docker info
- env
- if [ -z "$SECURE_BINARIES_IMAGE" ]; then export SECURE_BINARIES_IMAGE=${SECURE_BINARIES_IMAGE:-"registry.gitlab.com/gitlab-org/security-products/${CI_JOB_NAME}:${SECURE_BINARIES_ANALYZER_VERSION}"}; fi
- docker pull ${SECURE_BINARIES_IMAGE}
- mkdir -p output/$(dirname ${CI_JOB_NAME})
- |
if [ "$SECURE_BINARIES_SAVE_ARTIFACTS" = "true" ]; then
docker save ${SECURE_BINARIES_IMAGE} -o output/${CI_JOB_NAME}_${SECURE_BINARIES_ANALYZER_VERSION}.tar
gzip output/${CI_JOB_NAME}_${SECURE_BINARIES_ANALYZER_VERSION}.tar
sha256sum output/${CI_JOB_NAME}_${SECURE_BINARIES_ANALYZER_VERSION}.tar.gz > output/${CI_JOB_NAME}_${SECURE_BINARIES_ANALYZER_VERSION}.tag.gz.sha256sum
fi
- |
if [ "$SECURE_BINARIES_PUSH_IMAGES" = "true" ]; then
docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
docker tag ${SECURE_BINARIES_IMAGE} ${CI_REGISTRY_IMAGE}/${CI_JOB_NAME}:${SECURE_BINARIES_ANALYZER_VERSION}
docker push ${CI_REGISTRY_IMAGE}/${CI_JOB_NAME}:${SECURE_BINARIES_ANALYZER_VERSION}
fi
artifacts:
paths:
- output/
#
# SAST jobs
#
analyzers/bandit:
extends: .download_images
only:
variables:
- $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
$SECURE_BINARIES_ANALYZERS =~ /\bbandit\b/
analyzers/brakeman:
extends: .download_images
only:
variables:
- $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
$SECURE_BINARIES_ANALYZERS =~ /\bbrakeman\b/
analyzers/gosec:
extends: .download_images
only:
variables:
- $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
$SECURE_BINARIES_ANALYZERS =~ /\bgosec\b/
analyzers/spotbugs:
extends: .download_images
variables:
# TODO: Spotbugs is > 1GB, disabling for now
SECURE_BINARIES_SAVE_ARTIFACTS: "false"
only:
variables:
- $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
$SECURE_BINARIES_ANALYZERS =~ /\bspotbugs\b/
analyzers/flawfinder:
extends: .download_images
only:
variables:
- $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
$SECURE_BINARIES_ANALYZERS =~ /\bflawfinder\b/
analyzers/phpcs-security-audit:
extends: .download_images
only:
variables:
- $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
$SECURE_BINARIES_ANALYZERS =~ /\bphpcs-security-audit\b/
analyzers/security-code-scan:
extends: .download_images
only:
variables:
- $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
$SECURE_BINARIES_ANALYZERS =~ /\bsecurity-code-scan\b/
analyzers/nodejs-scan:
extends: .download_images
only:
variables:
- $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
$SECURE_BINARIES_ANALYZERS =~ /\bnodejs-scan\b/
analyzers/eslint:
extends: .download_images
only:
variables:
- $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
$SECURE_BINARIES_ANALYZERS =~ /\beslint\b/
analyzers/tslint:
extends: .download_images
only:
variables:
- $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
$SECURE_BINARIES_ANALYZERS =~ /\btslint\b/
analyzers/secrets:
extends: .download_images
only:
variables:
- $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
$SECURE_BINARIES_ANALYZERS =~ /\bsecrets\b/
analyzers/sobelow:
extends: .download_images
only:
variables:
- $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
$SECURE_BINARIES_ANALYZERS =~ /\bsobelow\b/
analyzers/pmd-apex:
extends: .download_images
only:
variables:
- $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
$SECURE_BINARIES_ANALYZERS =~ /\bsecrets\b/
analyzers/kubesec:
extends: .download_images
only:
variables:
- $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
$SECURE_BINARIES_ANALYZERS =~ /\bkubesec\b/
#
# Container Scanning jobs
#
analyzers/klar:
extends: .download_images
only:
variables:
- $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
$SECURE_BINARIES_ANALYZERS =~ /\bklar\b/
analyzers/clair-vulnerabilities-db:
extends: .download_images
only:
variables:
- $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
$SECURE_BINARIES_ANALYZERS =~ /\bclair-vulnerabilities-db\b/
variables:
SECURE_BINARIES_IMAGE: arminc/clair-db
SECURE_BINARIES_ANALYZER_VERSION: latest
#
# Dependency Scanning jobs
#
analyzers/bundler-audit:
extends: .download_images
only:
variables:
- $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
$SECURE_BINARIES_ANALYZERS =~ /\bbundler-audit\b/
analyzers/retire.js:
extends: .download_images
only:
variables:
- $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
$SECURE_BINARIES_ANALYZERS =~ /\bretire\.js\b/
analyzers/gemnasium:
extends: .download_images
only:
variables:
- $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
$SECURE_BINARIES_ANALYZERS =~ /\bgemnasium\b/
analyzers/gemnasium-maven:
extends: .download_images
only:
variables:
- $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
$SECURE_BINARIES_ANALYZERS =~ /\bgemnasium-maven\b/
analyzers/gemnasium-python:
extends: .download_images
only:
variables:
- $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
$SECURE_BINARIES_ANALYZERS =~ /\bgemnasium-python\b/
#
# License Scanning
#
license-management:
extends: .download_images
variables:
SECURE_BINARIES_ANALYZER_VERSION: "${CI_SERVER_VERSION_MAJOR}-${CI_SERVER_VERSION_MINOR}-stable"
# TODO: license-management is > 1GB, disabling for now
SECURE_BINARIES_SAVE_ARTIFACTS: "false"
only:
variables:
- $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
$SECURE_BINARIES_ANALYZERS =~ /\blicense-management\b/
#
# DAST
#
dast:
extends: .download_images
only:
variables:
- $SECURE_BINARIES_DOWNLOAD_IMAGES == "true" &&
$SECURE_BINARIES_ANALYZERS =~ /\bdast\b/
variables:
SECURE_BINARIES_ANALYZER_VERSION: 1

28
lib/gitlab/grape_logging/loggers/exception_logger.rb
View file

@ -4,14 +4,16 @@ module Gitlab
module GrapeLogging
module Loggers
class ExceptionLogger < ::GrapeLogging::Loggers::Base
def parameters(request, _)
def parameters(request, response_body)
data = {}
data[:api_error] = format_body(response_body) if bad_request?(request)
# grape-logging attempts to pass the logger the exception
# (https://github.com/aserafin/grape_logging/blob/v1.7.0/lib/grape_logging/middleware/request_logger.rb#L63),
# but it appears that the rescue_all in api.rb takes
# precedence so the logger never sees it. We need to
# store and retrieve the exception from the environment.
exception = request.env[::API::Helpers::API_EXCEPTION_ENV]
data = {}
return data unless exception.is_a?(Exception)
@ -19,6 +21,28 @@ module Gitlab
data
end
private
def format_body(response_body)
# https://github.com/rack/rack/blob/master/SPEC.rdoc#label-The+Body:
# The response_body must respond to each, but just in case we
# guard against errors here.
response_body = Array(response_body) unless response_body.respond_to?(:each)
# To avoid conflicting types in Elasticsearch, convert every
# element into an Array of strings. A response body is usually
# an array of Strings so that the response can be sent in
# chunks.
body = []
# each_with_object doesn't work with Rack::BodyProxy
response_body.each { |chunk| body << chunk.to_s }
body
end
def bad_request?(request)
request.env[::API::Helpers::API_RESPONSE_STATUS_CODE] == 400
end
end
end
end

2
lib/gitlab/metrics/exporter/sidekiq_exporter.rb
View file

@ -32,7 +32,7 @@ module Gitlab
Sidekiq.logger.error(
class: self.class.to_s,
message: 'Cannot start sidekiq_exporter',
exception: e.message
'exception.message' => e.message
)
false

1
spec/frontend/jira_import/components/jira_import_app_spec.js
View file

@ -26,6 +26,7 @@ const mountComponent = ({
['My Second Jira Project', 'MSJP'],
['Migrate to GitLab', 'MTG'],
],
jiraIntegrationPath: 'gitlab-org/gitlab-test/-/services/jira/edit',
projectPath: 'gitlab-org/gitlab-test',
setupIllustration: 'setup-illustration.svg',
},

15
spec/frontend/jira_import/components/jira_import_progress_spec.js
View file

@ -2,6 +2,10 @@ import { GlEmptyState } from '@gitlab/ui';
import { mount, shallowMount } from '@vue/test-utils';
import JiraImportProgress from '~/jira_import/components/jira_import_progress.vue';
const illustration = 'illustration.svg';
const importProject = 'JIRAPROJECT';
const issuesPath = 'gitlab-org/gitlab-test/-/issues';
describe('JiraImportProgress', () => {
let wrapper;
@ -13,11 +17,11 @@ describe('JiraImportProgress', () => {
const mountFunction = mountType === 'shallowMount' ? shallowMount : mount;
return mountFunction(JiraImportProgress, {
propsData: {
illustration: 'illustration.svg',
illustration,
importInitiator: 'Jane Doe',
importProject: 'JIRAPROJECT',
importProject,
importTime: '2020-04-08T12:17:25+00:00',
issuesPath: 'gitlab-org/gitlab-test/-/issues',
issuesPath,
},
});
};
@ -33,7 +37,7 @@ describe('JiraImportProgress', () => {
});
it('contains illustration', () => {
expect(getGlEmptyStateAttribute('svgpath')).toBe('illustration.svg');
expect(getGlEmptyStateAttribute('svgpath')).toBe(illustration);
});
it('contains a title', () => {
@ -46,7 +50,8 @@ describe('JiraImportProgress', () => {
});
it('contains button url', () => {
expect(getGlEmptyStateAttribute('primarybuttonlink')).toBe('gitlab-org/gitlab-test/-/issues');
const expected = `${issuesPath}?search=${importProject}`;
expect(getGlEmptyStateAttribute('primarybuttonlink')).toBe(expected);
});
});

12
spec/frontend/jira_import/components/jira_import_setup_spec.js
View file

@ -2,6 +2,9 @@ import { GlEmptyState } from '@gitlab/ui';
import { shallowMount } from '@vue/test-utils';
import JiraImportSetup from '~/jira_import/components/jira_import_setup.vue';
const illustration = 'illustration.svg';
const jiraIntegrationPath = 'gitlab-org/gitlab-test/-/services/jira/edit';
describe('JiraImportSetup', () => {
let wrapper;
@ -10,7 +13,8 @@ describe('JiraImportSetup', () => {
beforeEach(() => {
wrapper = shallowMount(JiraImportSetup, {
propsData: {
illustration: 'illustration.svg',
illustration,
jiraIntegrationPath,
},
});
});
@ -21,7 +25,7 @@ describe('JiraImportSetup', () => {
});
it('contains illustration', () => {
expect(getGlEmptyStateAttribute('svgpath')).toBe('illustration.svg');
expect(getGlEmptyStateAttribute('svgpath')).toBe(illustration);
});
it('contains a description', () => {
@ -32,4 +36,8 @@ describe('JiraImportSetup', () => {
it('contains button text', () => {
expect(getGlEmptyStateAttribute('primarybuttontext')).toBe('Set up Jira Integration');
});
it('contains button link', () => {
expect(getGlEmptyStateAttribute('primarybuttonlink')).toBe(jiraIntegrationPath);
});
});

2
spec/frontend/monitoring/components/dashboard_spec.js
View file

@ -6,7 +6,7 @@ import MockAdapter from 'axios-mock-adapter';
import axios from '~/lib/utils/axios_utils';
import statusCodes from '~/lib/utils/http_status';
import { metricStates } from '~/monitoring/constants';
import Dashboard from '~/monitoring/components/dashboard_with_alerts.vue';
import Dashboard from '~/monitoring/components/dashboard.vue';
import DateTimePicker from '~/vue_shared/components/date_time_picker/date_time_picker.vue';
import CustomMetricsFormFields from '~/custom_metrics/components/custom_metrics_form_fields.vue';

69
spec/lib/gitlab/grape_logging/loggers/exception_logger_spec.rb
View file

@ -3,14 +3,73 @@
require 'spec_helper'
describe Gitlab::GrapeLogging::Loggers::ExceptionLogger do
subject { described_class.new }
let(:mock_request) { OpenStruct.new(env: {}) }
let(:response_body) { nil }
describe ".parameters" do
subject { described_class.new.parameters(mock_request, response_body) }
describe 'when no exception is available' do
it 'returns an empty hash' do
expect(subject.parameters(mock_request, nil)).to eq({})
expect(subject).to eq({})
end
end
describe 'with a response' do
before do
mock_request.env[::API::Helpers::API_RESPONSE_STATUS_CODE] = code
end
context 'with a String response' do
let(:response_body) { { message: "something went wrong" }.to_json }
let(:code) { 400 }
let(:expected) { { api_error: [response_body.to_s] } }
it 'logs the response body' do
expect(subject).to eq(expected)
end
end
context 'with an Array response' do
let(:response_body) { ["hello world", 1] }
let(:code) { 400 }
let(:expected) { { api_error: ["hello world", "1"] } }
it 'casts all elements to strings' do
expect(subject).to eq(expected)
end
end
# Rack v2.0.9 can return a BodyProxy. This was changed in later versions:
# https://github.com/rack/rack/blob/2.0.9/lib/rack/response.rb#L69
context 'with a Rack BodyProxy response' do
let(:message) { { message: "something went wrong" }.to_json }
let(:response) { Rack::Response.new(message, code, {}) }
let(:response_body) { Rack::BodyProxy.new(response) }
let(:code) { 400 }
let(:expected) { { api_error: [message] } }
it 'logs the response body' do
expect(subject).to eq(expected)
end
end
context 'unauthorized error' do
let(:response_body) { 'unauthorized' }
let(:code) { 401 }
it 'does not log an api_error field' do
expect(subject).not_to have_key(:api_error)
end
end
context 'HTTP success' do
let(:response_body) { 'success' }
let(:code) { 200 }
it 'does not log an api_error field' do
expect(subject).not_to have_key(:api_error)
end
end
end
@ -32,7 +91,7 @@ describe Gitlab::GrapeLogging::Loggers::ExceptionLogger do
end
it 'returns the correct fields' do
expect(subject.parameters(mock_request, nil)).to eq(expected)
expect(subject).to eq(expected)
end
context 'with backtrace' do
@ -43,7 +102,7 @@ describe Gitlab::GrapeLogging::Loggers::ExceptionLogger do
end
it 'includes the backtrace' do
expect(subject.parameters(mock_request, nil)).to eq(expected)
expect(subject).to eq(expected)
end
end
end

2
spec/lib/gitlab/metrics/exporter/sidekiq_exporter_spec.rb
View file

@ -53,7 +53,7 @@ describe Gitlab::Metrics::Exporter::SidekiqExporter do
.with(
class: described_class.to_s,
message: 'Cannot start sidekiq_exporter',
exception: anything)
'exception.message' => anything)
exporter.start
end

4
spec/requests/api/helpers_spec.rb
View file

@ -328,6 +328,8 @@ describe API::Helpers do
it 'returns a 401 response' do
expect { authenticate! }.to raise_error /401/
expect(env[described_class::API_RESPONSE_STATUS_CODE]).to eq(401)
end
end
@ -340,6 +342,8 @@ describe API::Helpers do
it 'does not raise an error' do
expect { authenticate! }.not_to raise_error
expect(env[described_class::API_RESPONSE_STATUS_CODE]).to be_nil
end
end
end